diff --git a/.cargo/config b/.cargo/config deleted file mode 100644 index a4dc4b423fe..00000000000 --- a/.cargo/config +++ /dev/null @@ -1,13 +0,0 @@ -[target.x86_64-unknown-redox] -linker = "x86_64-unknown-redox-gcc" - -[target.'cfg(feature = "cargo-clippy")'] -rustflags = [ - "-Wclippy::use_self", - "-Wclippy::needless_pass_by_value", - "-Wclippy::semicolon_if_nothing_returned", - "-Wclippy::single_char_pattern", - "-Wclippy::explicit_iter_loop", - "-Wclippy::if_not_else", -] - diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000000..c6aa207614f --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +[target.x86_64-unknown-redox] +linker = "x86_64-unknown-redox-gcc" + +[env] +PROJECT_NAME_FOR_VERSION_STRING = "uutils coreutils" diff --git a/.clippy.toml b/.clippy.toml index bee70857bd1..b6f9a360f69 100644 --- a/.clippy.toml +++ b/.clippy.toml @@ -1,2 +1,5 @@ -msrv = "1.64.0" -cognitive-complexity-threshold = 10 +msrv = "1.85.0" +avoid-breaking-exported-api = false +check-private-items = true +cognitive-complexity-threshold = 24 +missing-docs-in-crate-items = true diff --git a/.config/nextest.toml b/.config/nextest.toml index 3ba8bb393a4..473c461402a 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -4,3 +4,10 @@ status-level = "all" final-status-level = "skip" failure-output = "immediate-final" fail-fast = false + +[profile.coverage] +retries = 0 +status-level = "all" +final-status-level = "skip" +failure-output = "immediate-final" +fail-fast = false diff --git a/.editorconfig b/.editorconfig index 53ccc4f9a15..9df8cbbbf98 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,4 +1,5 @@ # EditorConfig (is awesome!; ref: http://EditorConfig.org; v2022.02.11 [rivy]) +# spell-checker:ignore akefile shellcheck vcproj # * top-most EditorConfig file root = true @@ -52,7 +53,7 @@ indent_style = space switch_case_indent = true [*.{sln,vc{,x}proj{,.*},[Ss][Ln][Nn],[Vv][Cc]{,[Xx]}[Pp][Rr][Oo][Jj]{,.*}}] -# MSVC sln/vcproj/vcxproj files, when used, will persistantly revert to CRLF EOLNs and eat final EOLs +# MSVC sln/vcproj/vcxproj files, when used, will persistently revert to CRLF EOLNs and eat final EOLs end_of_line = crlf insert_final_newline = false diff --git a/.envrc b/.envrc new file mode 100644 index 00000000000..cbf4a76e2de --- /dev/null +++ b/.envrc @@ -0,0 +1,7 @@ +# spell-checker:ignore direnv + +if ! has nix_direnv_version || ! nix_direnv_version 3.0.6; then + source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.6/direnvrc" "sha256-RYcUJaRMf8oF5LznDrlCXbkOQrywm0HDv1VjYGaJGdM=" +fi + +use flake diff --git a/.github/workflows/CICD.yml b/.github/workflows/CICD.yml index fe72de11067..c9dcb2e3552 100644 --- a/.github/workflows/CICD.yml +++ b/.github/workflows/CICD.yml @@ -2,20 +2,26 @@ name: CICD # spell-checker:ignore (abbrev/names) CICD CodeCOV MacOS MinGW MSVC musl taiki # spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic CARGOFLAGS -# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers -# spell-checker:ignore (people) Peltoche rivy dtolnay -# spell-checker:ignore (shell/tools) choco clippy dmake dpkg esac fakeroot fdesc fdescfs gmake grcov halium lcov libssl mkdir popd printf pushd rsync rustc rustfmt rustup shopt utmpdump xargs -# spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos gnueabihf issuecomment maint multisize nullglob onexitbegin onexitend pell runtest Swatinem tempfile testsuite toybox uutils +# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers dedupe devel profdata +# spell-checker:ignore (people) Peltoche rivy dtolnay Anson dawidd +# spell-checker:ignore (shell/tools) binutils choco clippy dmake dpkg esac fakeroot fdesc fdescfs gmake grcov halium lcov libclang libfuse libssl limactl mkdir nextest nocross pacman popd printf pushd redoxer rsync rustc rustfmt rustup shopt sccache utmpdump xargs +# spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos getenforce gnueabihf issuecomment maint manpages msys multisize noconfirm nullglob onexitbegin onexitend pell runtest Swatinem tempfile testsuite toybox uutils env: PROJECT_NAME: coreutils PROJECT_DESC: "Core universal (cross-platform) utilities" PROJECT_AUTH: "uutils" - RUST_MIN_SRV: "1.64.0" + RUST_MIN_SRV: "1.85.0" # * style job configuration STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis -on: [push, pull_request] +on: + pull_request: + push: + tags: + - '*' + branches: + - '*' permissions: contents: read # to fetch code (actions/checkout) @@ -30,8 +36,10 @@ jobs: name: Style/cargo-deny runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: EmbarkStudios/cargo-deny-action@v1 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: EmbarkStudios/cargo-deny-action@v2 style_deps: ## ToDO: [2021-11-10; rivy] 'Style/deps' needs more informative output and better integration of results into the GHA dashboard @@ -47,7 +55,9 @@ jobs: - { os: macos-latest , features: "feat_Tier1,feat_require_unix,feat_require_unix_utmpx" } - { os: windows-latest , features: feat_os_windows } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@nightly ## note: requires 'nightly' toolchain b/c `cargo-udeps` uses the `rustc` '-Z save-analysis' option ## * ... ref: @@ -82,199 +92,6 @@ jobs: grep --ignore-case "all deps seem to have been used" udeps.log || { printf "%s\n" "::${fault_type} ::${fault_prefix}: \`cargo udeps\`: style violation (unused dependency found)" ; fault=true ; } if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi - style_format: - name: Style/format - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: rustfmt - - uses: Swatinem/rust-cache@v2 - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # failure mode - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - outputs FAIL_ON_FAULT FAULT_TYPE - # target-specific options - # * CARGO_FEATURES_OPTION - CARGO_FEATURES_OPTION='' ; - if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features "${{ matrix.job.features }}"' ; fi - outputs CARGO_FEATURES_OPTION - - name: "`cargo fmt` testing" - shell: bash - run: | - ## `cargo fmt` testing - unset fault - fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" - fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') - # * convert any errors/warnings to GHA UI annotations; ref: - S=$(cargo fmt -- --check) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s\n" "$S" | sed -E -n -e "s/^Diff[[:space:]]+in[[:space:]]+${PWD//\//\\/}\/(.*)[[:space:]]+at[[:space:]]+[^0-9]+([0-9]+).*$/::${fault_type} file=\1,line=\2::${fault_prefix}: \`cargo fmt\`: style violation (file:'\1', line:\2; use \`cargo fmt -- \"\1\"\`)/p" ; fault=true ; } - if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi - - fuzz: - name: Run the fuzzers - runs-on: ubuntu-latest - env: - RUN_FOR: 60 - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@nightly - - name: Install `cargo-fuzz` - run: cargo install cargo-fuzz - - uses: Swatinem/rust-cache@v2 - - name: Run fuzz_date for XX seconds - shell: bash - run: | - ## Run it - cd fuzz - cargo +nightly fuzz run fuzz_date -- -max_total_time=${{ env.RUN_FOR }} -detect_leaks=0 - - name: Run fuzz_test for XX seconds - continue-on-error: true - shell: bash - run: | - ## Run it - cd fuzz - cargo +nightly fuzz run fuzz_test -- -max_total_time=${{ env.RUN_FOR }} -detect_leaks=0 - - name: Run fuzz_parse_glob for XX seconds - shell: bash - run: | - ## Run it - cd fuzz - cargo +nightly fuzz run fuzz_parse_glob -- -max_total_time=${{ env.RUN_FOR }} -detect_leaks=0 - - name: Run fuzz_parse_size for XX seconds - shell: bash - run: | - ## Run it - cd fuzz - cargo +nightly fuzz run fuzz_parse_size -- -max_total_time=${{ env.RUN_FOR }} -detect_leaks=0 - - name: Run fuzz_parse_time for XX seconds - shell: bash - run: | - ## Run it - cd fuzz - cargo +nightly fuzz run fuzz_parse_time -- -max_total_time=${{ env.RUN_FOR }} -detect_leaks=0 - - style_lint: - name: Style/lint - runs-on: ${{ matrix.job.os }} - env: - SCCACHE_GHA_ENABLED: "true" - RUSTC_WRAPPER: "sccache" - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - - { os: macos-latest , features: feat_os_macos } - - { os: windows-latest , features: feat_os_windows } - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # failure mode - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - outputs FAIL_ON_FAULT FAULT_TYPE - # target-specific options - # * CARGO_FEATURES_OPTION - CARGO_FEATURES_OPTION='--all-features' ; - if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features ${{ matrix.job.features }}' ; fi - outputs CARGO_FEATURES_OPTION - # * determine sub-crate utility list - UTILITY_LIST="$(./util/show-utils.sh ${CARGO_FEATURES_OPTION})" - echo UTILITY_LIST=${UTILITY_LIST} - CARGO_UTILITY_LIST_OPTIONS="$(for u in ${UTILITY_LIST}; do echo -n "-puu_${u} "; done;)" - outputs CARGO_UTILITY_LIST_OPTIONS - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites - case '${{ matrix.job.os }}' in - macos-latest) brew install coreutils ;; # needed for show-utils.sh - esac - - name: "`cargo clippy` lint testing" - shell: bash - run: | - ## `cargo clippy` lint testing - unset fault - fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" - fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') - # * convert any warnings to GHA UI annotations; ref: - S=$(cargo clippy --all-targets ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} -- -W clippy::default_trait_access -W clippy::manual_string_new -D warnings 2>&1) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*$/::${fault_type} file=\2,line=\3,col=\4::${fault_prefix}: \`cargo clippy\`: \1 (file:'\2', line:\3)/p;" -e '}' ; fault=true ; } - if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi - - style_spellcheck: - name: Style/spelling - runs-on: ${{ matrix.job.os }} - strategy: - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - steps: - - uses: actions/checkout@v3 - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # failure mode - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - outputs FAIL_ON_FAULT FAULT_TYPE - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites - # * pin installed cspell to v4.2.8 (cspell v5+ is broken for NodeJS < v12) - ## maint: [2021-11-10; rivy] `cspell` version may be advanced to v5 when used with NodeJS >= v12 - sudo apt-get -y update ; sudo apt-get -y install npm ; sudo npm install cspell@4.2.8 -g ; - - name: Run `cspell` - shell: bash - run: | - ## Run `cspell` - unset fault - fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" - fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') - # * find cspell configuration ; note: avoid quotes around ${cfg_file} b/c `cspell` (v4) doesn't correctly dequote the config argument (or perhaps a subshell expansion issue?) - cfg_files=($(shopt -s nullglob ; echo {.vscode,.}/{,.}c[sS]pell{.json,.config{.js,.cjs,.json,.yaml,.yml},.yaml,.yml} ;)) - cfg_file=${cfg_files[0]} - unset CSPELL_CFG_OPTION ; if [ -n "$cfg_file" ]; then CSPELL_CFG_OPTION="--config $cfg_file" ; fi - # * `cspell` - ## maint: [2021-11-10; rivy] the `--no-progress` option for `cspell` is a `cspell` v5+ option - # S=$(cspell ${CSPELL_CFG_OPTION} --no-summary --no-progress "**/*") && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n "s/${PWD//\//\\/}\/(.*):(.*):(.*) - (.*)/::${fault_type} file=\1,line=\2,col=\3::${fault_type^^}: \4 (file:'\1', line:\2)/p" ; fault=true ; true ; } - S=$(cspell ${CSPELL_CFG_OPTION} --no-summary "**/*") && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n "s/${PWD//\//\\/}\/(.*):(.*):(.*) - (.*)/::${fault_type} file=\1,line=\2,col=\3::${fault_type^^}: \4 (file:'\1', line:\2)/p" ; fault=true ; true ; } - if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi - doc_warnings: name: Documentation/warnings runs-on: ${{ matrix.job.os }} @@ -292,14 +109,20 @@ jobs: # - { os: macos-latest , features: feat_os_macos } # - { os: windows-latest , features: feat_os_windows } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: stable components: clippy - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install/setup prerequisites + shell: bash + run: | + sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - name: Initialize workflow variables id: vars shell: bash @@ -326,9 +149,9 @@ jobs: shell: bash run: | RUSTDOCFLAGS="-Dwarnings" cargo doc ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --no-deps --workspace --document-private-items - - uses: DavidAnson/markdownlint-cli2-action@v12 + - uses: DavidAnson/markdownlint-cli2-action@v20 with: - command: fix + fix: "true" globs: | *.md docs/src/*.md @@ -345,7 +168,9 @@ jobs: job: - { os: ubuntu-latest , features: feat_os_unix } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.RUST_MIN_SRV }} @@ -353,7 +178,7 @@ jobs: - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Initialize workflow variables id: vars shell: bash @@ -365,19 +190,19 @@ jobs: unset CARGO_FEATURES_OPTION if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features "${{ matrix.job.features }}"' ; fi outputs CARGO_FEATURES_OPTION - - name: Confirm MinSRV compatible 'Cargo.lock' + - name: Confirm MinSRV compatible '*/Cargo.lock' shell: bash run: | - ## Confirm MinSRV compatible 'Cargo.lock' - # * 'Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) - cargo fetch --locked --quiet || { echo "::error file=Cargo.lock::Incompatible (or out-of-date) 'Cargo.lock' file; update using \`cargo +${{ env.RUST_MIN_SRV }} update\`" ; exit 1 ; } - - name: Confirm MinSRV equivalence for '.clippy.toml' + ## Confirm MinSRV compatible '*/Cargo.lock' + # * '*/Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) + for dir in "." "fuzz"; do + ( cd "$dir" && cargo fetch --locked --quiet ) || { echo "::error file=$dir/Cargo.lock::Incompatible (or out-of-date) '$dir/Cargo.lock' file; update using \`cd '$dir' && cargo +${{ env.RUST_MIN_SRV }} update\`" ; exit 1 ; } + done + - name: Install/setup prerequisites shell: bash run: | - ## Confirm MinSRV equivalence for '.clippy.toml' - # * ensure '.clippy.toml' MSRV configuration setting is equal to ${{ env.RUST_MIN_SRV }} - CLIPPY_MSRV=$(grep -P "(?i)^\s*msrv\s*=\s*" .clippy.toml | grep -oP "\d+([.]\d+)+") - if [ "${CLIPPY_MSRV}" != "${{ env.RUST_MIN_SRV }}" ]; then { echo "::error file=.clippy.toml::Incorrect MSRV configuration for clippy (found '${CLIPPY_MSRV}'; should be '${{ env.RUST_MIN_SRV }}'); update '.clippy.toml' with 'msrv = \"${{ env.RUST_MIN_SRV }}\"'" ; exit 1 ; } ; fi + # Install a package for one of the tests + sudo apt-get -y update ; sudo apt-get -y install attr - name: Info shell: bash run: | @@ -397,7 +222,7 @@ jobs: echo "## dependency list" ## * using the 'stable' toolchain is necessary to avoid "unexpected '--filter-platform'" errors RUSTUP_TOOLCHAIN=stable cargo fetch --locked --quiet - RUSTUP_TOOLCHAIN=stable cargo tree --all --locked --no-dev-dependencies --no-indent ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} | grep -vE "$PWD" | sort --unique + RUSTUP_TOOLCHAIN=stable cargo tree --no-dedupe --locked -e=no-dev --prefix=none ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} | grep -vE "$PWD" | sort --unique - name: Test run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} -p uucore -p coreutils env: @@ -413,7 +238,9 @@ jobs: job: - { os: ubuntu-latest , features: feat_os_unix } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - name: "`cargo update` testing" @@ -421,7 +248,9 @@ jobs: run: | ## `cargo update` testing # * convert any errors/warnings to GHA UI annotations; ref: - cargo fetch --locked --quiet || { echo "::error file=Cargo.lock::'Cargo.lock' file requires update (use \`cargo +${{ env.RUST_MIN_SRV }} update\`)" ; exit 1 ; } + for dir in "." "fuzz"; do + ( cd "$dir" && cargo fetch --locked --quiet ) || { echo "::error file=$dir/Cargo.lock::'$dir/Cargo.lock' file requires update (use \`cd '$dir' && cargo +${{ env.RUST_MIN_SRV }} update\`)" ; exit 1 ; } + done build_makefile: name: Build/Makefile @@ -436,12 +265,18 @@ jobs: job: - { os: ubuntu-latest , features: feat_os_unix } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 + - name: Install/setup prerequisites + shell: bash + run: | + sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: "`make build`" shell: bash run: | @@ -451,10 +286,26 @@ jobs: run: make nextest CARGOFLAGS="--profile ci --hide-progress-bar" env: RUST_BACKTRACE: "1" + - name: "`make install COMPLETIONS=n MANPAGES=n`" + shell: bash + run: | + DESTDIR=/tmp/ make PROFILE=release COMPLETIONS=n MANPAGES=n install + # Check that the utils are present + test -f /tmp/usr/local/bin/tty + # Check that the manpage is not present + ! test -f /tmp/usr/local/share/man/man1/whoami.1 + # Check that the completion is not present + ! test -f /tmp/usr/local/share/zsh/site-functions/_install + ! test -f /tmp/usr/local/share/bash-completion/completions/head + ! test -f /tmp/usr/local/share/fish/vendor_completions.d/cat.fish + env: + RUST_BACKTRACE: "1" - name: "`make install`" shell: bash run: | DESTDIR=/tmp/ make PROFILE=release install + # Check that the utils are present + test -f /tmp/usr/local/bin/tty # Check that the manpage is present test -f /tmp/usr/local/share/man/man1/whoami.1 # Check that the completion is present @@ -467,6 +318,8 @@ jobs: shell: bash run: | DESTDIR=/tmp/ make uninstall + # Check that the utils are not present + ! test -f /tmp/usr/local/bin/tty # Check that the manpage is not present ! test -f /tmp/usr/local/share/man/man1/whoami.1 # Check that the completion is not present @@ -490,14 +343,16 @@ jobs: - { os: macos-latest , features: feat_os_macos } - { os: windows-latest , features: feat_os_windows } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Test - run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + run: cargo nextest run --hide-progress-bar --profile ci --features ${{ matrix.job.features }} env: RUST_BACKTRACE: "1" @@ -517,14 +372,16 @@ jobs: - { os: macos-latest , features: feat_os_macos } - { os: windows-latest , features: feat_os_windows } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@nightly - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Test - run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + run: cargo nextest run --hide-progress-bar --profile ci --features ${{ matrix.job.features }} env: RUST_BACKTRACE: "1" @@ -541,17 +398,19 @@ jobs: job: - { os: ubuntu-latest , features: feat_os_unix } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Install dependencies shell: bash run: | ## Install dependencies sudo apt-get update - sudo apt-get install jq + sudo apt-get install jq libselinux1-dev - name: "`make install`" shell: bash run: | @@ -584,14 +443,14 @@ jobs: --arg multisize "$SIZE_MULTI" \ '{($date): { sha: $sha, size: $size, multisize: $multisize, }}' > size-result.json - name: Download the previous individual size result - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v9 with: workflow: CICD.yml name: individual-size-result repo: uutils/coreutils path: dl - name: Download the previous size result - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v9 with: workflow: CICD.yml name: size-result @@ -603,6 +462,12 @@ jobs: check() { # Warn if the size increases by more than 5% threshold='1.05' + + if [[ "$2" -eq 0 || "$3" -eq 0 ]]; then + echo "::warning file=$4::Invalid size for $1. Sizes cannot be 0." + return + fi + ratio=$(jq -n "$2 / $3") echo "$1: size=$2, previous_size=$3, ratio=$ratio, threshold=$threshold" if [[ "$(jq -n "$ratio > $threshold")" == 'true' ]]; then @@ -622,12 +487,12 @@ jobs: previous_multisize=$(cat dl/size-result.json | jq -r '.[] | .multisize') check 'multicall binary' "$multisize" "$previous_multisize" 'size-result.json' - name: Upload the individual size result - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: individual-size-result path: individual-size-result.json - name: Upload the size result - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: size-result path: size-result.json @@ -648,20 +513,28 @@ jobs: fail-fast: false matrix: job: - # { os , target , cargo-options , features , use-cross , toolchain } - - { os: ubuntu-latest , target: arm-unknown-linux-gnueabihf, features: feat_os_unix_gnueabihf, use-cross: use-cross, } - - { os: ubuntu-latest , target: aarch64-unknown-linux-gnu , features: feat_os_unix_gnueabihf , use-cross: use-cross } + # - { os , target , cargo-options , default-features, features , use-cross , toolchain, skip-tests, workspace-tests, skip-package, skip-publish } + - { os: ubuntu-latest , target: arm-unknown-linux-gnueabihf , features: feat_os_unix_gnueabihf , use-cross: use-cross , skip-tests: true } + - { os: ubuntu-24.04-arm , target: aarch64-unknown-linux-gnu , features: feat_os_unix_gnueabihf } + - { os: ubuntu-latest , target: aarch64-unknown-linux-musl , features: feat_os_unix , use-cross: use-cross , skip-tests: true } # - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: feat_selinux , use-cross: use-cross } - - { os: ubuntu-latest , target: i686-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } - - { os: ubuntu-latest , target: i686-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } - - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross } - - { os: ubuntu-latest , target: x86_64-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } - - { os: macos-latest , target: x86_64-apple-darwin , features: feat_os_macos } + - { os: ubuntu-latest , target: i686-unknown-linux-gnu , features: "feat_os_unix,test_risky_names", use-cross: use-cross } + - { os: ubuntu-latest , target: i686-unknown-linux-musl , features: feat_os_unix , use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: "feat_os_unix,test_risky_names", use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: "feat_os_unix,uudoc" , use-cross: no, workspace-tests: true } + - { os: ubuntu-latest , target: x86_64-unknown-linux-musl , features: feat_os_unix , use-cross: use-cross } + - { os: ubuntu-latest , target: x86_64-unknown-redox , features: feat_os_unix_redox , use-cross: redoxer , skip-tests: true } + - { os: ubuntu-latest , target: wasm32-unknown-unknown , default-features: false, features: uucore/format, skip-tests: true, skip-package: true, skip-publish: true } + - { os: macos-latest , target: aarch64-apple-darwin , features: feat_os_macos, workspace-tests: true } # M1 CPU + - { os: macos-13 , target: x86_64-apple-darwin , features: feat_os_macos, workspace-tests: true } - { os: windows-latest , target: i686-pc-windows-msvc , features: feat_os_windows } - - { os: windows-latest , target: x86_64-pc-windows-gnu , features: feat_os_windows } ## note: requires rust >= 1.43.0 to link correctly + - { os: windows-latest , target: x86_64-pc-windows-gnu , features: feat_os_windows } - { os: windows-latest , target: x86_64-pc-windows-msvc , features: feat_os_windows } + - { os: windows-latest , target: aarch64-pc-windows-msvc , features: feat_os_windows, use-cross: use-cross , skip-tests: true } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.RUST_MIN_SRV }} @@ -670,7 +543,7 @@ jobs: with: key: "${{ matrix.job.os }}_${{ matrix.job.target }}" - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Initialize workflow variables id: vars shell: bash @@ -707,7 +580,13 @@ jobs: i686-*) TARGET_ARCH=i686 ;; x86_64-*) TARGET_ARCH=x86_64 ;; esac; - unset TARGET_OS ; case '${{ matrix.job.target }}' in *-linux-*) TARGET_OS=linux ;; *-apple-*) TARGET_OS=macos ;; *-windows-*) TARGET_OS=windows ;; esac; + unset TARGET_OS + case '${{ matrix.job.target }}' in + *-linux-*) TARGET_OS=linux ;; + *-apple-*) TARGET_OS=macos ;; + *-windows-*) TARGET_OS=windows ;; + *-redox*) TARGET_OS=redox ;; + esac outputs TARGET_ARCH TARGET_OS # package name PKG_suffix=".tar.gz" ; case '${{ matrix.job.target }}' in *-pc-windows-*) PKG_suffix=".zip" ;; esac; @@ -741,20 +620,47 @@ jobs: CARGO_FEATURES_OPTION='' ; if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features=${{ matrix.job.features }}' ; fi outputs CARGO_FEATURES_OPTION + # * CARGO_DEFAULT_FEATURES_OPTION + CARGO_DEFAULT_FEATURES_OPTION='' ; + if [ "${{ matrix.job.default-features }}" == "false" ]; then CARGO_DEFAULT_FEATURES_OPTION='--no-default-features' ; fi + outputs CARGO_DEFAULT_FEATURES_OPTION # * CARGO_CMD - CARGO_CMD='cross' ; case '${{ matrix.job.use-cross }}' in ''|0|f|false|n|no) CARGO_CMD='cargo' ;; esac; + CARGO_CMD='cross' + CARGO_CMD_OPTIONS='+${{ env.RUST_MIN_SRV }}' + # Added suffix for artifacts, needed when multiple jobs use the same target. + ARTIFACTS_SUFFIX='' + case '${{ matrix.job.use-cross }}' in + ''|0|f|false|n|no) + CARGO_CMD='cargo' + ARTIFACTS_SUFFIX='-nocross' + ;; + redoxer) + CARGO_CMD='redoxer' + CARGO_CMD_OPTIONS='' + ;; + esac outputs CARGO_CMD + outputs CARGO_CMD_OPTIONS + outputs ARTIFACTS_SUFFIX + CARGO_TEST_OPTIONS='' + case '${{ matrix.job.workspace-tests }}' in + 1|t|true|y|yes) + # This also runs tests in other packages in the source directory (e.g. uucore). + # We cannot enable this everywhere as some platforms are currently broken, and + # we cannot use `cross` as its Docker image is ancient (Ubuntu 16.04) and is + # missing required system dependencies (e.g. recent libclang-dev). + CARGO_TEST_OPTIONS='--workspace' + ;; + esac + outputs CARGO_TEST_OPTIONS # ** pass needed environment into `cross` container (iff `cross` not already configured via "Cross.toml") if [ "${CARGO_CMD}" = 'cross' ] && [ ! -e "Cross.toml" ] ; then printf "[build.env]\npassthrough = [\"CI\", \"RUST_BACKTRACE\", \"CARGO_TERM_COLOR\"]\n" > Cross.toml fi - # * test only library and/or binaries for arm-type targets - unset CARGO_TEST_OPTIONS ; case '${{ matrix.job.target }}' in aarch64-* | arm-*) CARGO_TEST_OPTIONS="--bins" ;; esac; - outputs CARGO_TEST_OPTIONS # * executable for `strip`? STRIP="strip" case ${{ matrix.job.target }} in - aarch64-*-linux-gnu) STRIP="aarch64-linux-gnu-strip" ;; + aarch64-*-linux-*) STRIP="aarch64-linux-gnu-strip" ;; arm-*-linux-gnueabihf) STRIP="arm-linux-gnueabihf-strip" ;; *-pc-windows-msvc) STRIP="" ;; esac; @@ -762,7 +668,7 @@ jobs: - uses: taiki-e/install-action@v2 if: steps.vars.outputs.CARGO_CMD == 'cross' with: - tool: cross@0.2.1 + tool: cross@0.2.5 - name: Create all needed build/work directories shell: bash run: | @@ -775,14 +681,27 @@ jobs: run: | ## Install/setup prerequisites case '${{ matrix.job.target }}' in - arm-unknown-linux-gnueabihf) sudo apt-get -y update ; sudo apt-get -y install gcc-arm-linux-gnueabihf ;; - aarch64-unknown-linux-gnu) sudo apt-get -y update ; sudo apt-get -y install gcc-aarch64-linux-gnu ;; + arm-unknown-linux-gnueabihf) + sudo apt-get -y update + sudo apt-get -y install gcc-arm-linux-gnueabihf + ;; + aarch64-unknown-linux-*) + sudo apt-get -y update + sudo apt-get -y install gcc-aarch64-linux-gnu + ;; + *-redox*) + sudo apt-get -y update + sudo apt-get -y install fuse3 libfuse-dev + ;; esac case '${{ matrix.job.os }}' in macos-latest) brew install coreutils ;; # needed for testing esac case '${{ matrix.job.os }}' in ubuntu-*) + # selinux headers needed to build tests + sudo apt-get -y update + sudo apt-get -y install libselinux1-dev # pinky is a tool to show logged-in users from utmp, and gecos fields from /etc/passwd. # In GitHub Action *nix VMs, no accounts log in, even the "runner" account that runs the commands. The account also has empty gecos fields. # To work around this for pinky tests, we create a fake login entry for the GH runner account... @@ -796,6 +715,10 @@ jobs: echo "foo" > /home/runner/.plan ;; esac + - uses: taiki-e/install-action@v2 + if: steps.vars.outputs.CARGO_CMD == 'redoxer' + with: + tool: redoxer@0.2.37 - name: Initialize toolchain-dependent workflow variables id: dep_vars shell: bash @@ -829,35 +752,38 @@ jobs: # dependencies echo "## dependency list" cargo fetch --locked --quiet - cargo tree --locked --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --all --no-dev-dependencies --no-indent | grep -vE "$PWD" | sort --unique + cargo tree --locked --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} --no-dedupe -e=no-dev --prefix=none | grep -vE "$PWD" | sort --unique - name: Build shell: bash run: | ## Build - ${{ steps.vars.outputs.CARGO_CMD }} +${{ env.RUST_MIN_SRV }} build --release \ - --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} build --release \ + --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} - name: Test + if: matrix.job.skip-tests != true shell: bash run: | ## Test - ${{ steps.vars.outputs.CARGO_CMD }} +${{ env.RUST_MIN_SRV }} test --target=${{ matrix.job.target }} \ - ${{ steps.vars.outputs.CARGO_TEST_OPTIONS}} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} + ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} test --target=${{ matrix.job.target }} \ + ${{ steps.vars.outputs.CARGO_TEST_OPTIONS}} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} env: RUST_BACKTRACE: "1" - name: Test individual utilities + if: matrix.job.skip-tests != true shell: bash run: | ## Test individual utilities - ${{ steps.vars.outputs.CARGO_CMD }} +${{ env.RUST_MIN_SRV }} test --target=${{ matrix.job.target }} \ - ${{ steps.vars.outputs.CARGO_TEST_OPTIONS}} ${{ matrix.job.cargo-options }} ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} + ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} test --target=${{ matrix.job.target }} \ + ${{ matrix.job.cargo-options }} ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} env: RUST_BACKTRACE: "1" - name: Archive executable artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: ${{ env.PROJECT_NAME }}-${{ matrix.job.target }} + name: ${{ env.PROJECT_NAME }}-${{ matrix.job.target }}${{ steps.vars.outputs.ARTIFACTS_SUFFIX }} path: target/${{ matrix.job.target }}/release/${{ env.PROJECT_NAME }}${{ steps.vars.outputs.EXE_suffix }} - name: Package + if: matrix.job.skip-package != true shell: bash run: | ## Package artifact(s) @@ -892,9 +818,10 @@ jobs: fakeroot dpkg-deb --build "${DPKG_DIR}" "${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.DPKG_NAME }}" fi - name: Publish - uses: softprops/action-gh-release@v1 - if: steps.vars.outputs.DEPLOY + uses: softprops/action-gh-release@v2 + if: steps.vars.outputs.DEPLOY && matrix.job.skip-publish != true with: + draft: true files: | ${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_NAME }} ${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.DPKG_NAME }} @@ -920,13 +847,16 @@ jobs: run: | ## VARs setup echo "TEST_SUMMARY_FILE=busybox-result.json" >> $GITHUB_OUTPUT - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Install/setup prerequisites shell: bash run: | + sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev ## Install/setup prerequisites make prepare-busytest - name: Run BusyBox test suite @@ -964,17 +894,17 @@ jobs: HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) echo "HASH=${HASH}" >> $GITHUB_OUTPUT - name: Reserve SHA1/ID of 'test-summary' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "${{ steps.summary.outputs.HASH }}" path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Reserve test results summary - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: test-summary + name: busybox-test-summary path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Upload json results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: busybox-result.json path: ${{ steps.vars.outputs.TEST_SUMMARY_FILE }} @@ -1000,24 +930,29 @@ jobs: outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } TEST_SUMMARY_FILE="toybox-result.json" outputs TEST_SUMMARY_FILE - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.RUST_MIN_SRV }} components: rustfmt - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install/setup prerequisites + shell: bash + run: | + sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - name: Build coreutils as multiple binaries shell: bash run: | ## Build individual uutil binaries set -v make - - name: Install/setup prerequisites + - name: Run toybox src shell: bash run: | - ## Install/setup prerequisites make toybox-src - name: Run Toybox test suite id: summary @@ -1052,30 +987,21 @@ jobs: HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) echo "HASH=${HASH}" >> $GITHUB_OUTPUT - name: Reserve SHA1/ID of 'test-summary' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "${{ steps.summary.outputs.HASH }}" path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Reserve test results summary - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: test-summary + name: toybox-test-summary path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Upload json results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: toybox-result.json path: ${{ steps.vars.outputs.TEST_SUMMARY_FILE }} - toml_format: - runs-on: ubuntu-latest - steps: - - name: Clone repository - uses: actions/checkout@v3 - - - name: Check - run: npx --yes @taplo/cli fmt --check - coverage: name: Code Coverage runs-on: ${{ matrix.job.os }} @@ -1088,45 +1014,55 @@ jobs: matrix: job: - { os: ubuntu-latest , features: unix, toolchain: nightly } - - { os: macos-latest , features: macos, toolchain: nightly } - - { os: windows-latest , features: windows, toolchain: nightly-x86_64-pc-windows-gnu } + # FIXME: Re-enable macos code coverage + # - { os: macos-latest , features: macos, toolchain: nightly } + # FIXME: Re-enable Code Coverage on windows, which currently fails due to "profiler_builtins". See #6686. + # - { os: windows-latest , features: windows, toolchain: nightly-x86_64-pc-windows-gnu } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.job.toolchain }} components: rustfmt - - uses: taiki-e/install-action@nextest - - uses: taiki-e/install-action@grcov + - uses: taiki-e/install-action@v2 + with: + tool: nextest,grcov@0.8.24 - uses: Swatinem/rust-cache@v2 + - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 + # - name: Reattach HEAD ## may be needed for accurate code coverage info # run: git checkout ${{ github.head_ref }} + - name: Initialize workflow variables id: vars shell: bash run: | ## VARs setup outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + # toolchain TOOLCHAIN="nightly" ## default to "nightly" toolchain (required for certain required unstable compiler flags) ## !maint: refactor when stable channel has needed support + # * specify gnu-type TOOLCHAIN for windows; `grcov` requires gnu-style code coverage data files case ${{ matrix.job.os }} in windows-*) TOOLCHAIN="$TOOLCHAIN-x86_64-pc-windows-gnu" ;; esac; + # * use requested TOOLCHAIN if specified if [ -n "${{ matrix.job.toolchain }}" ]; then TOOLCHAIN="${{ matrix.job.toolchain }}" ; fi outputs TOOLCHAIN - # staging directory - STAGING='_staging' - outputs STAGING + # target-specific options + # * CARGO_FEATURES_OPTION CARGO_FEATURES_OPTION='--all-features' ; ## default to '--all-features' for code coverage if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features=${{ matrix.job.features }}' ; fi outputs CARGO_FEATURES_OPTION + # * CODECOV_FLAGS CODECOV_FLAGS=$( echo "${{ matrix.job.os }}" | sed 's/[^[:alnum:]]/_/g' ) outputs CODECOV_FLAGS + - name: Install/setup prerequisites shell: bash run: | @@ -1134,8 +1070,10 @@ jobs: case '${{ matrix.job.os }}' in macos-latest) brew install coreutils ;; # needed for testing esac + case '${{ matrix.job.os }}' in ubuntu-latest) + sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev # pinky is a tool to show logged-in users from utmp, and gecos fields from /etc/passwd. # In GitHub Action *nix VMs, no accounts log in, even the "runner" account that runs the commands. The account also has empty gecos fields. # To work around this for pinky tests, we create a fake login entry for the GH runner account... @@ -1149,63 +1087,118 @@ jobs: echo "foo" > /home/runner/.plan ;; esac - - name: Initialize toolchain-dependent workflow variables - id: dep_vars - shell: bash + + case '${{ matrix.job.os }}' in + # Update binutils if MinGW due to https://github.com/rust-lang/rust/issues/112368 + windows-latest) C:/msys64/usr/bin/pacman.exe -Sy --needed mingw-w64-x86_64-gcc --noconfirm ; echo "C:\msys64\mingw64\bin" >> $GITHUB_PATH ;; + esac + + ## Install the llvm-tools component to get access to `llvm-profdata` + rustup component add llvm-tools + + - name: Run test and coverage + id: run_test_cov run: | - ## Dependent VARs setup outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # * determine sub-crate utility list - UTILITY_LIST="$(./util/show-utils.sh ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }})" - CARGO_UTILITY_LIST_OPTIONS="$(for u in ${UTILITY_LIST}; do echo -n "-puu_${u} "; done;)" - outputs CARGO_UTILITY_LIST_OPTIONS - - name: Test uucore - run: cargo nextest run --profile ci --hide-progress-bar -p uucore - env: - RUSTC_WRAPPER: "" - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - RUSTDOCFLAGS: "-Cpanic=abort" - RUST_BACKTRACE: "1" - # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }} - - name: Test - run: cargo nextest run --profile ci --hide-progress-bar ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} - env: - RUSTC_WRAPPER: "" - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - RUSTDOCFLAGS: "-Cpanic=abort" - RUST_BACKTRACE: "1" - # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }} - - name: Test individual utilities - run: cargo nextest run --profile ci --hide-progress-bar ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} + + # Run the coverage script + ./util/build-run-test-coverage-linux.sh + + outputs REPORT_FILE env: - RUSTC_WRAPPER: "" - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - RUSTDOCFLAGS: "-Cpanic=abort" - RUST_BACKTRACE: "1" + COVERAGE_DIR: ${{ github.workspace }}/coverage + FEATURES_OPTION: ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }} - - name: Generate coverage data (via `grcov`) - id: coverage - shell: bash - run: | - ## Generate coverage data - COVERAGE_REPORT_DIR="target/debug" - COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" - # GRCOV_IGNORE_OPTION='--ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*"' ## `grcov` ignores these params when passed as an environment variable (why?) - # GRCOV_EXCLUDE_OPTION='--excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()"' ## `grcov` ignores these params when passed as an environment variable (why?) - mkdir -p "${COVERAGE_REPORT_DIR}" - # display coverage files - ~/.cargo/bin/grcov . --output-type files --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique - # generate coverage report - ~/.cargo/bin/grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" - echo "report=${COVERAGE_REPORT_FILE}" >> $GITHUB_OUTPUT + - name: Upload coverage results (to Codecov.io) - uses: codecov/codecov-action@v3 - # if: steps.vars.outputs.HAS_CODECOV_TOKEN + uses: codecov/codecov-action@v5 with: - # token: ${{ secrets.CODECOV_TOKEN }} - file: ${{ steps.coverage.outputs.report }} + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ steps.run_test_cov.outputs.report }} ## flags: IntegrationTests, UnitTests, ${{ steps.vars.outputs.CODECOV_FLAGS }} flags: ${{ steps.vars.outputs.CODECOV_FLAGS }} name: codecov-umbrella fail_ci_if_error: false + test_separately: + name: Separate Builds + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: build and test all programs individually + shell: bash + run: | + for f in $(util/show-utils.sh) + do + echo "Building and testing $f" + cargo test -p "uu_$f" || exit 1 + done + + test_all_features: + name: Test all features separately + needs: [ min_version, deps ] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + # windows-latest - https://github.com/uutils/coreutils/issues/7044 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: build and test all features individually + shell: bash + run: | + for f in $(util/show-utils.sh) + do + echo "Running tests with --features=$f and --no-default-features" + cargo test --features=$f --no-default-features + done + + test_selinux: + name: Build/SELinux + needs: [ min_version, deps ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - name: Setup Lima + uses: lima-vm/lima-actions/setup@v1 + id: lima-actions-setup + - name: Cache ~/.cache/lima + uses: actions/cache@v4 + with: + path: ~/.cache/lima + key: lima-${{ steps.lima-actions-setup.outputs.version }} + - name: Start Fedora VM with SELinux + run: limactl start --plain --name=default --cpus=1 --disk=30 --memory=4 --network=lima:user-v2 template://fedora + - name: Setup SSH + uses: lima-vm/lima-actions/ssh@v1 + - run: rsync -v -a -e ssh . lima-default:~/work/ + - name: Setup Rust and other build deps in VM + run: | + lima sudo dnf install gcc g++ git rustup libselinux-devel clang-devel attr -y + lima rustup-init -y --default-toolchain stable + - name: Verify SELinux Status + run: | + lima getenforce + lima ls -laZ /etc/selinux + - name: Build and Test with SELinux + run: | + lima ls + lima bash -c "cd work && cargo test --features 'feat_selinux'" + - name: Lint with SELinux + run: lima bash -c "cd work && cargo clippy --all-targets --features 'feat_selinux' -- -D warnings" diff --git a/.github/workflows/CheckScripts.yml b/.github/workflows/CheckScripts.yml index e1b3b24d205..78a4656fcde 100644 --- a/.github/workflows/CheckScripts.yml +++ b/.github/workflows/CheckScripts.yml @@ -1,6 +1,6 @@ name: CheckScripts -# spell-checker:ignore ludeeus mfinelli +# spell-checker:ignore ludeeus mfinelli shellcheck scandir shfmt env: SCRIPT_DIR: 'util' @@ -8,7 +8,7 @@ env: on: push: branches: - - main + - '*' paths: - 'util/**/*.sh' pull_request: @@ -29,7 +29,9 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Run ShellCheck uses: ludeeus/action-shellcheck@master env: @@ -41,34 +43,18 @@ jobs: shell_fmt: name: ShellScript/Format - # no need to run in pr events - # shfmt will be performed on main branch when the PR is merged - if: github.event_name != 'pull_request' runs-on: ubuntu-latest - needs: [ shell_check ] permissions: - contents: write - pull-requests: write + contents: read steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Setup shfmt - uses: mfinelli/setup-shfmt@v2 + uses: mfinelli/setup-shfmt@v3 - name: Run shfmt shell: bash run: | - # show differs first for every files that need to be formatted # fmt options: bash syntax, 4 spaces indent, indent for switch-case echo "## show the differences between formatted and original scripts..." find ${{ env.SCRIPT_DIR }} -name "*.sh" -print0 | xargs -0 shfmt -ln=bash -i 4 -ci -d || true - # perform a shell format - echo "## perform a shell format..." - # ignore the error code because `-d` will always return false when the file has difference - find ${{ env.SCRIPT_DIR }} -name "*.sh" -print0 | xargs -0 shfmt -ln=bash -i 4 -ci -w - - name: Commit any changes - uses: EndBug/add-and-commit@v9 - with: - default_author: github_actions - message: "style: auto format by CI (shfmt)" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/.github/workflows/FixPR.yml b/.github/workflows/FixPR.yml index 97b0be34afa..bbdf50b30b5 100644 --- a/.github/workflows/FixPR.yml +++ b/.github/workflows/FixPR.yml @@ -1,6 +1,6 @@ name: FixPR -# spell-checker:ignore Swatinem dtolnay +# spell-checker:ignore Swatinem dtolnay dedupe # Trigger automated fixes for PRs being merged (with associated commits) @@ -26,7 +26,9 @@ jobs: job: - { os: ubuntu-latest , features: feat_os_unix } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Initialize job variables id: vars shell: bash @@ -41,9 +43,11 @@ jobs: - name: Ensure updated 'Cargo.lock' shell: bash run: | - # Ensure updated 'Cargo.lock' - # * 'Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) - cargo fetch --locked --quiet || cargo +${{ steps.vars.outputs.RUST_MIN_SRV }} update + # Ensure updated '*/Cargo.lock' + # * '*/Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) + for dir in "." "fuzz"; do + ( cd "$dir" && (cargo fetch --locked --quiet || cargo +${{ steps.vars.outputs.RUST_MIN_SRV }} update) ) + done - name: Info shell: bash run: | @@ -63,14 +67,14 @@ jobs: echo "## dependency list" cargo fetch --locked --quiet ## * using the 'stable' toolchain is necessary to avoid "unexpected '--filter-platform'" errors - RUSTUP_TOOLCHAIN=stable cargo tree --locked --all --no-dev-dependencies --no-indent --features ${{ matrix.job.features }} | grep -vE "$PWD" | sort --unique + RUSTUP_TOOLCHAIN=stable cargo tree --locked --no-dedupe -e=no-dev --prefix=none --features ${{ matrix.job.features }} | grep -vE "$PWD" | sort --unique - name: Commit any changes (to '${{ env.BRANCH_TARGET }}') uses: EndBug/add-and-commit@v9 with: new_branch: ${{ env.BRANCH_TARGET }} default_author: github_actions - message: "maint ~ refresh 'Cargo.lock'" - add: Cargo.lock + message: "maint ~ refresh 'Cargo.lock' 'fuzz/Cargo.lock'" + add: Cargo.lock fuzz/Cargo.lock env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -85,7 +89,9 @@ jobs: job: - { os: ubuntu-latest , features: feat_os_unix } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Initialize job variables id: vars shell: bash diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml index bb64232a99a..7fe42070e82 100644 --- a/.github/workflows/GnuComment.yml +++ b/.github/workflows/GnuComment.yml @@ -1,10 +1,12 @@ name: GnuComment +# spell-checker:ignore zizmor backquote + on: workflow_run: workflows: ["GnuTests"] types: - - completed + - completed # zizmor: ignore[dangerous-triggers] permissions: {} jobs: @@ -18,7 +20,7 @@ jobs: github.event.workflow_run.event == 'pull_request' steps: - name: 'Download artifact' - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | // List all artifacts from GnuTests @@ -43,7 +45,7 @@ jobs: - run: unzip comment.zip - name: 'Comment on PR' - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index ccb5a6f7495..fc40fc91f5f 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -1,15 +1,19 @@ name: GnuTests # spell-checker:ignore (abbrev/names) CodeCov gnulib GnuTests Swatinem -# spell-checker:ignore (jargon) submodules -# spell-checker:ignore (libs/utils) autopoint chksum gperf lcov libexpect pyinotify shopt texinfo valgrind libattr libcap taiki-e +# spell-checker:ignore (jargon) submodules devel +# spell-checker:ignore (libs/utils) autopoint chksum getenforce gperf lcov libexpect limactl pyinotify setenforce shopt texinfo valgrind libattr libcap taiki-e # spell-checker:ignore (options) Ccodegen Coverflow Cpanic Zpanic # spell-checker:ignore (people) Dawid Dziurla * dawidd dtolnay # spell-checker:ignore (vars) FILESET SUBDIRS XPASS # * note: to run a single test => `REPO/util/run-gnu-test.sh PATH/TO/TEST/SCRIPT` -on: [push, pull_request] +on: + pull_request: + push: + branches: + - '*' permissions: contents: read @@ -19,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} +env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + jobs: gnu: permissions: @@ -26,7 +33,7 @@ jobs: contents: read # for actions/checkout to fetch code pull-requests: read # for dawidd6/action-download-artifact to query commit hash name: Run GNU tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Initialize workflow variables id: vars @@ -41,23 +48,31 @@ jobs: path_reference="reference" outputs path_GNU path_GNU_tests path_reference path_UUTILS # - repo_default_branch="${{ github.event.repository.default_branch }}" - repo_GNU_ref="v9.4" - repo_reference_branch="${{ github.event.repository.default_branch }}" + repo_default_branch="$DEFAULT_BRANCH" + repo_GNU_ref="v9.7" + repo_reference_branch="$DEFAULT_BRANCH" outputs repo_default_branch repo_GNU_ref repo_reference_branch # SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log" ROOT_SUITE_LOG_FILE="${path_GNU_tests}/test-suite-root.log" + SELINUX_SUITE_LOG_FILE="${path_GNU_tests}/selinux-test-suite.log" + SELINUX_ROOT_SUITE_LOG_FILE="${path_GNU_tests}/selinux-test-suite-root.log" TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support TEST_FILESET_PREFIX='test-fileset-IDs.sha1#' TEST_FILESET_SUFFIX='.txt' TEST_SUMMARY_FILE='gnu-result.json' TEST_FULL_SUMMARY_FILE='gnu-full-result.json' - outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE + TEST_ROOT_FULL_SUMMARY_FILE='gnu-root-full-result.json' + TEST_SELINUX_FULL_SUMMARY_FILE='selinux-gnu-full-result.json' + TEST_SELINUX_ROOT_FULL_SUMMARY_FILE='selinux-root-gnu-full-result.json' + AGGREGATED_SUMMARY_FILE='aggregated-result.json' + + outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE SELINUX_SUITE_LOG_FILE SELINUX_ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE TEST_ROOT_FULL_SUMMARY_FILE TEST_SELINUX_FULL_SUMMARY_FILE TEST_SELINUX_ROOT_FULL_SUMMARY_FILE AGGREGATED_SUMMARY_FILE - name: Checkout code (uutil) - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: '${{ steps.vars.outputs.path_UUTILS }}' + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: stable @@ -66,14 +81,62 @@ jobs: with: workspaces: "./${{ steps.vars.outputs.path_UUTILS }} -> target" - name: Checkout code (GNU coreutils) - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'coreutils/coreutils' path: '${{ steps.vars.outputs.path_GNU }}' ref: ${{ steps.vars.outputs.repo_GNU_ref }} - submodules: recursive + submodules: false + persist-credentials: false + + - name: Selinux - Setup Lima + uses: lima-vm/lima-actions/setup@v1 + id: lima-actions-setup + + - name: Selinux - Cache ~/.cache/lima + uses: actions/cache@v4 + with: + path: ~/.cache/lima + key: lima-${{ steps.lima-actions-setup.outputs.version }} + + - name: Selinux - Start Fedora VM with SELinux + run: limactl start --plain --name=default --cpus=4 --disk=40 --memory=8 --network=lima:user-v2 template://fedora + + - name: Selinux - Setup SSH + uses: lima-vm/lima-actions/ssh@v1 + + - name: Selinux - Verify SELinux Status and Configuration + run: | + lima getenforce + lima ls -laZ /etc/selinux + lima sudo sestatus + + # Ensure we're running in enforcing mode + lima sudo setenforce 1 + lima getenforce + + # Create test files with SELinux contexts for testing + lima sudo mkdir -p /var/test_selinux + lima sudo touch /var/test_selinux/test_file + lima sudo chcon -t etc_t /var/test_selinux/test_file + lima ls -Z /var/test_selinux/test_file # Verify context + + - name: Selinux - Install dependencies in VM + run: | + lima sudo dnf -y update + lima sudo dnf -y install git autoconf autopoint bison texinfo gperf gcc g++ gdb jq libacl-devel libattr-devel libcap-devel libselinux-devel attr rustup clang-devel texinfo-tex wget automake patch quilt + lima rustup-init -y --default-toolchain stable + + - name: Override submodule URL and initialize submodules + # Use github instead of upstream git server + run: | + git submodule sync --recursive + git config submodule.gnulib.url https://github.com/coreutils/gnulib.git + git submodule update --init --recursive --depth 1 + working-directory: ${{ steps.vars.outputs.path_GNU }} + - name: Retrieve reference artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v9 # ref: continue-on-error: true ## don't break the build for missing reference artifacts (may be expired or just not generated yet) with: @@ -87,7 +150,7 @@ jobs: run: | ## Install dependencies sudo apt-get update - sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev + sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev libselinux1-dev attr quilt - name: Add various locales shell: bash run: | @@ -99,6 +162,7 @@ jobs: sudo locale-gen sudo locale-gen --keep-existing fr_FR sudo locale-gen --keep-existing fr_FR.UTF-8 + sudo locale-gen --keep-existing es_ES.UTF-8 sudo locale-gen --keep-existing sv_SE sudo locale-gen --keep-existing sv_SE.UTF-8 sudo locale-gen --keep-existing en_US @@ -107,12 +171,68 @@ jobs: sudo update-locale echo "After:" locale -a + + - name: Selinux - Copy the sources to VM + run: | + rsync -a -e ssh . lima-default:~/work/ + - name: Build binaries shell: bash run: | ## Build binaries cd '${{ steps.vars.outputs.path_UUTILS }}' - bash util/build-gnu.sh + bash util/build-gnu.sh --release-build + + - name: Selinux - Generate selinux tests list + run: | + # Find and list all tests that require SELinux + lima bash -c "cd ~/work/gnu/ && grep -l 'require_selinux_' -r tests/ > ~/work/uutils/selinux-tests.txt" + lima bash -c "cd ~/work/uutils/ && cat selinux-tests.txt" + + # Count the tests + lima bash -c "cd ~/work/uutils/ && echo 'Found SELinux tests:'; wc -l selinux-tests.txt" + + - name: Selinux - Build for selinux tests + run: | + lima bash -c "cd ~/work/uutils/ && bash util/build-gnu.sh" + lima bash -c "mkdir -p ~/work/gnu/tests-selinux/" + + - name: Selinux - Run selinux tests + run: | + lima sudo setenforce 1 + lima getenforce + lima cat /proc/filesystems + lima bash -c "cd ~/work/uutils/ && bash util/run-gnu-test.sh \$(cat selinux-tests.txt)" + + - name: Selinux - Extract testing info from individual logs into JSON + shell: bash + run : | + lima bash -c "cd ~/work/gnu/ && python3 ../uutils/util/gnu-json-result.py tests > ~/work/gnu/tests-selinux/${{ steps.vars.outputs.TEST_SELINUX_FULL_SUMMARY_FILE }}" + + - name: Selinux/root - Run selinux tests + run: | + lima bash -c "cd ~/work/uutils/ && CI=1 bash util/run-gnu-test.sh run-root \$(cat selinux-tests.txt)" + + - name: Selinux/root - Extract testing info from individual logs into JSON + shell: bash + run : | + lima bash -c "cd ~/work/gnu/ && python3 ../uutils/util/gnu-json-result.py tests > ~/work/gnu/tests-selinux/${{ steps.vars.outputs.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }}" + + - name: Selinux - Collect test logs and test results + run: | + mkdir -p ${{ steps.vars.outputs.path_GNU_tests }}-selinux + + # Copy the test logs from the Lima VM to the host + lima bash -c "cp ~/work/gnu/tests/test-suite.log ~/work/gnu/tests-selinux/ || echo 'No test-suite.log found'" + lima bash -c "cp ~/work/gnu/tests/test-suite-root.log ~/work/gnu/tests-selinux/ || echo 'No test-suite-root.log found'" + rsync -v -a -e ssh lima-default:~/work/gnu/tests-selinux/ ./${{ steps.vars.outputs.path_GNU_tests }}-selinux/ + + # Copy SELinux logs to the main test directory for integrated processing + cp -f ${{ steps.vars.outputs.path_GNU_tests }}-selinux/test-suite.log ${{ steps.vars.outputs.path_GNU_tests }}/selinux-test-suite.log + cp -f ${{ steps.vars.outputs.path_GNU_tests }}-selinux/test-suite-root.log ${{ steps.vars.outputs.path_GNU_tests }}/selinux-test-suite-root.log + cp -f ${{ steps.vars.outputs.path_GNU_tests }}-selinux/${{ steps.vars.outputs.TEST_SELINUX_FULL_SUMMARY_FILE }} . + cp -f ${{ steps.vars.outputs.path_GNU_tests }}-selinux/${{ steps.vars.outputs.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }} . + - name: Run GNU tests shell: bash run: | @@ -120,6 +240,13 @@ jobs: path_GNU='${{ steps.vars.outputs.path_GNU }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' bash "${path_UUTILS}/util/run-gnu-test.sh" + + - name: Extract testing info from individual logs into JSON + shell: bash + run : | + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' + python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} + - name: Run GNU root tests shell: bash run: | @@ -127,35 +254,40 @@ jobs: path_GNU='${{ steps.vars.outputs.path_GNU }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' bash "${path_UUTILS}/util/run-gnu-test.sh" run-root - - name: Extract testing info into JSON + + - name: Extract testing info from individual logs (run as root) into JSON shell: bash run : | - ## Extract testing info into JSON path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' - python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} + python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_ROOT_FULL_SUMMARY_FILE }} + - name: Extract/summarize testing info id: summary shell: bash run: | ## Extract/summarize testing info outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' - # - SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}' - ROOT_SUITE_LOG_FILE='${{ steps.vars.outputs.ROOT_SUITE_LOG_FILE }}' - ls -al ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} - if test -f "${SUITE_LOG_FILE}" + # Check if the file exists + if test -f "${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }}" then - source ${path_UUTILS}/util/analyze-gnu-results.sh ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} + # Look at all individual results and summarize + eval $(python3 ${path_UUTILS}/util/analyze-gnu-results.py -o=${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }} ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} ${{ steps.vars.outputs.TEST_ROOT_FULL_SUMMARY_FILE }} ${{ steps.vars.outputs.TEST_SELINUX_FULL_SUMMARY_FILE }} ${{ steps.vars.outputs.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }}) + if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then - echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early" + echo "::error ::Failed to parse test results from '${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }}'; failing early" exit 1 fi + output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP" echo "${output}" - if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi + + if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then + echo "::warning ::${output}" + fi + jq -n \ --arg date "$(date --rfc-email)" \ --arg sha "$GITHUB_SHA" \ @@ -169,122 +301,101 @@ jobs: HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) outputs HASH else - echo "::error ::Failed to find summary of test results (missing '${SUITE_LOG_FILE}'); failing early" + echo "::error ::Failed to find summary of test results (missing '${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }}'); failing early" exit 1 fi + # Compress logs before upload (fails otherwise) gzip ${{ steps.vars.outputs.TEST_LOGS_GLOB }} - name: Reserve SHA1/ID of 'test-summary' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "${{ steps.summary.outputs.HASH }}" path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Reserve test results summary - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-summary path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - name: Reserve test logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-logs path: "${{ steps.vars.outputs.TEST_LOGS_GLOB }}" - name: Upload full json results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: gnu-full-result.json + name: gnu-full-result path: ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} + - name: Upload root json results + uses: actions/upload-artifact@v4 + with: + name: gnu-root-full-result + path: ${{ steps.vars.outputs.TEST_ROOT_FULL_SUMMARY_FILE }} + - name: Upload selinux json results + uses: actions/upload-artifact@v4 + with: + name: selinux-gnu-full-result + path: ${{ steps.vars.outputs.TEST_SELINUX_FULL_SUMMARY_FILE }} + - name: Upload selinux root json results + uses: actions/upload-artifact@v4 + with: + name: selinux-root-gnu-full-result.json + path: ${{ steps.vars.outputs.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }} + - name: Upload aggregated json results + uses: actions/upload-artifact@v4 + with: + name: aggregated-result + path: ${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }} - name: Compare test failures VS reference shell: bash run: | - ## Compare test failures VS reference - have_new_failures="" - REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' - REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' + ## Compare test failures VS reference using JSON files + REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/aggregated-result/aggregated-result.json' + CURRENT_SUMMARY_FILE='${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }}' REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' - # https://github.com/uutils/coreutils/issues/4294 - # https://github.com/uutils/coreutils/issues/4295 - IGNORE_INTERMITTENT="${path_UUTILS}/.github/workflows/ignore-intermittent.txt" - mkdir -p ${{ steps.vars.outputs.path_reference }} + # Path to ignore file for intermittent issues + IGNORE_INTERMITTENT="${path_UUTILS}/.github/workflows/ignore-intermittent.txt" + # Set up comment directory COMMENT_DIR="${{ steps.vars.outputs.path_reference }}/comment" mkdir -p ${COMMENT_DIR} echo ${{ github.event.number }} > ${COMMENT_DIR}/NR COMMENT_LOG="${COMMENT_DIR}/result.txt" - # The comment log might be downloaded from a previous run - # We only want the new changes, so remove it if it exists. - rm -f ${COMMENT_LOG} - touch ${COMMENT_LOG} + COMPARISON_RESULT=0 + if test -f "${CURRENT_SUMMARY_FILE}"; then + if test -f "${REF_SUMMARY_FILE}"; then + echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")" - if test -f "${REF_LOG_FILE}"; then - echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" - REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) - NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) - REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) - NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) - for LINE in ${REF_FAILING} - do - if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then - if ! grep ${LINE} ${IGNORE_INTERMITTENT} - then - MSG="Congrats! The gnu test ${LINE} is no longer failing!" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - else - MSG="Skipping an intermittent issue ${LINE}" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - echo "" - fi - fi - done - for LINE in ${NEW_FAILING} - do - if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" - then - if ! grep ${LINE} ${IGNORE_INTERMITTENT} - then - MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" - echo "::error ::$MSG" - echo $MSG >> ${COMMENT_LOG} - have_new_failures="true" - else - MSG="Skip an intermittent issue ${LINE}" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - echo "" - fi - fi - done - for LINE in ${REF_ERROR} - do - if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}"; then - MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - fi - done - for LINE in ${NEW_ERROR} - do - if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" - then - MSG="GNU test error: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" - echo "::error ::$MSG" - echo $MSG >> ${COMMENT_LOG} - have_new_failures="true" - fi - done + python3 ${path_UUTILS}/util/compare_test_results.py \ + --ignore-file "${IGNORE_INTERMITTENT}" \ + --output "${COMMENT_LOG}" \ + "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" + COMPARISON_RESULT=$? + else + echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'." + fi else - echo "::warning ::Skipping test failure comparison; no prior reference test logs are available." + echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early" + exit 1 + fi + + if [ ${COMPARISON_RESULT} -eq 1 ]; then + echo "ONLY_INTERMITTENT=false" >> $GITHUB_ENV + echo "::error ::Found new non-intermittent test failures" + exit 1 + else + echo "ONLY_INTERMITTENT=true" >> $GITHUB_ENV + echo "::notice ::No new test failures detected" fi - if test -n "${have_new_failures}" ; then exit -1 ; fi - name: Upload comparison log (for GnuComment workflow) if: success() || failure() # run regardless of prior step success/failure - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: comment path: ${{ steps.vars.outputs.path_reference }}/comment/ @@ -301,76 +412,3 @@ jobs: else echo "::warning ::Skipping test summary comparison; no prior reference summary is available." fi - - gnu_coverage: - name: Run GNU tests with coverage - runs-on: ubuntu-20.04 - steps: - - name: Checkout code uutil - uses: actions/checkout@v3 - with: - path: 'uutils' - - name: Checkout GNU coreutils - uses: actions/checkout@v3 - with: - repository: 'coreutils/coreutils' - path: 'gnu' - ref: 'v9.4' - submodules: recursive - - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly - components: rustfmt - - uses: taiki-e/install-action@grcov - - uses: Swatinem/rust-cache@v2 - with: - workspaces: "./uutils -> target" - - name: Install dependencies - run: | - ## Install dependencies - sudo apt-get update - sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev - - name: Add various locales - run: | - ## Add various locales - echo "Before:" - locale -a - ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' - ## Some others need a French locale - sudo locale-gen - sudo locale-gen --keep-existing fr_FR - sudo locale-gen --keep-existing fr_FR.UTF-8 - sudo update-locale - echo "After:" - locale -a - - name: Build binaries - env: - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - RUSTDOCFLAGS: "-Cpanic=abort" - run: | - ## Build binaries - cd uutils - UU_MAKE_PROFILE=debug bash util/build-gnu.sh - - name: Run GNU tests - run: bash uutils/util/run-gnu-test.sh - - name: Generate coverage data (via `grcov`) - id: coverage - run: | - ## Generate coverage data - cd uutils - COVERAGE_REPORT_DIR="target/debug" - COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" - mkdir -p "${COVERAGE_REPORT_DIR}" - sudo chown -R "$(whoami)" "${COVERAGE_REPORT_DIR}" - # display coverage files - grcov . --output-type files --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique - # generate coverage report - grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" - echo "report=${COVERAGE_REPORT_FILE}" >> $GITHUB_OUTPUT - - name: Upload coverage results (to Codecov.io) - uses: codecov/codecov-action@v3 - with: - file: ${{ steps.coverage.outputs.report }} - flags: gnutests - name: gnutests - working-directory: uutils diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index a6929b171cc..4f8edea3085 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -1,8 +1,15 @@ name: Android -# spell-checker:ignore TERMUX reactivecircus Swatinem noaudio pkill swiftshader dtolnay juliangruber +# spell-checker:ignore (people) reactivecircus Swatinem dtolnay juliangruber +# spell-checker:ignore (shell/tools) TERMUX nextest udevadm pkill +# spell-checker:ignore (misc) swiftshader playstore DATALOSS noaudio + +on: + pull_request: + push: + branches: + - '*' -on: [push, pull_request] permissions: contents: read # to fetch code (actions/checkout) @@ -12,78 +19,160 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} +env: + TERMUX: v0.118.0 + KEY_POSTFIX: nextest+rustc-hash+adb+sshd+upgrade+XGB+inc18 + COMMON_EMULATOR_OPTIONS: -no-window -noaudio -no-boot-anim -camera-back none -gpu swiftshader_indirect -metrics-collection + EMULATOR_DISK_SIZE: 12GB + EMULATOR_HEAP_SIZE: 2048M + EMULATOR_BOOT_TIMEOUT: 1200 # 20min + jobs: test_android: name: Test builds - runs-on: macos-latest timeout-minutes: 90 strategy: fail-fast: false matrix: + os: [ubuntu-latest] # , macos-latest + cores: [4] # , 6 + ram: [4096, 8192] api-level: [28] - target: [default] - arch: [x86] # , arm64-v8a + target: [google_apis_playstore] + arch: [x86, x86_64] # , arm64-v8a + exclude: + - ram: 8192 + arch: x86 + - ram: 4096 + arch: x86_64 + runs-on: ${{ matrix.os }} env: - TERMUX: v0.118.0 + EMULATOR_RAM_SIZE: ${{ matrix.ram }} + EMULATOR_CORES: ${{ matrix.cores }} + RUNNER_OS: ${{ matrix.os }} + AVD_CACHE_KEY: "set later due to limitations of github actions not able to concatenate env variables" steps: - - uses: actions/checkout@v3 + - name: Concatenate values to environment file + run: | + echo "AVD_CACHE_KEY=${{ matrix.os }}-${{ matrix.cores }}-${{ matrix.ram }}-${{ matrix.api-level }}-${{ matrix.target }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+${{ env.KEY_POSTFIX }}" >> $GITHUB_ENV + - name: Collect information about runner + if: always() + continue-on-error: true + run: | + hostname + uname -a + free -mh + df -Th + cat /proc/cpuinfo + - name: (Linux) create links from home to data partition + if: ${{ runner.os == 'Linux' }} + continue-on-error: true + run: | + ls -lah /mnt/ + cat /mnt/DATALOSS_WARNING_README.txt + sudo mkdir /mnt/data + sudo chmod a+rwx /mnt/data + mkdir /mnt/data/.android && ln -s /mnt/data/.android ~/.android + mkdir /mnt/data/work && ln -s /mnt/data/work ~/work + - name: Enable KVM group perms (linux hardware acceleration) + if: ${{ runner.os == 'Linux' }} + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Collect information about runner + if: always() + continue-on-error: true + run: | + free -mh + df -Th - name: Restore AVD cache - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 id: avd-cache + continue-on-error: true with: path: | ~/.android/avd/* ~/.android/avd/*/snapshots/* ~/.android/adb* ~/__rustc_hash__ - key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest+rustc-hash + key: avd-${{ env.AVD_CACHE_KEY }} + - name: Collect information about runner after AVD cache + if: always() + continue-on-error: true + run: | + free -mh + df -Th + ls -lah /mnt/data + du -sch /mnt/data + - name: Delete AVD Lockfile when run from cache + if: steps.avd-cache.outputs.cache-hit == 'true' + run: | + rm -f \ + ~/.android/avd/*.avd/*.lock \ + ~/.android/avd/*/*.lock - name: Create and cache emulator image if: steps.avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@v2 + uses: reactivecircus/android-emulator-runner@v2.34.0 with: api-level: ${{ matrix.api-level }} target: ${{ matrix.target }} arch: ${{ matrix.arch }} - ram-size: 2048M - disk-size: 7GB + ram-size: ${{ env.EMULATOR_RAM_SIZE }} + heap-size: ${{ env.EMULATOR_HEAP_SIZE }} + disk-size: ${{ env.EMULATOR_DISK_SIZE }} + cores: ${{ env.EMULATOR_CORES }} force-avd-creation: true - emulator-options: -no-snapshot-load -noaudio -no-boot-anim -camera-back none + emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-load + emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} script: | util/android-commands.sh init "${{ matrix.arch }}" "${{ matrix.api-level }}" "${{ env.TERMUX }}" - name: Save AVD cache if: steps.avd-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: | ~/.android/avd/* ~/.android/avd/*/snapshots/* ~/.android/adb* ~/__rustc_hash__ - key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest+rustc-hash + key: avd-${{ env.AVD_CACHE_KEY }} - uses: juliangruber/read-file-action@v1 id: read_rustc_hash with: # ~ expansion didn't work - path: /Users/runner/__rustc_hash__ + path: ${{ runner.os == 'Linux' && '/home/runner/__rustc_hash__' || '/Users/runner/__rustc_hash__' }} trim: true - name: Restore rust cache id: rust-cache - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: ~/__rust_cache__ # The version vX at the end of the key is just a development version to avoid conflicts in # the github cache during the development of this workflow key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 + - name: Collect information about runner resources + if: always() + continue-on-error: true + run: | + free -mh + df -Th - name: Build and Test - uses: reactivecircus/android-emulator-runner@v2 + uses: reactivecircus/android-emulator-runner@v2.34.0 with: api-level: ${{ matrix.api-level }} target: ${{ matrix.target }} arch: ${{ matrix.arch }} - ram-size: 2048M - disk-size: 7GB + ram-size: ${{ env.EMULATOR_RAM_SIZE }} + heap-size: ${{ env.EMULATOR_HEAP_SIZE }} + disk-size: ${{ env.EMULATOR_DISK_SIZE }} + cores: ${{ env.EMULATOR_CORES }} force-avd-creation: false - emulator-options: -no-snapshot-save -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -snapshot ${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }} + emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-save -snapshot ${{ env.AVD_CACHE_KEY }} + emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} # This is not a usual script. Every line is executed in a separate shell with `sh -c`. If # one of the lines returns with error the whole script is failed (like running a script with # set -e) and in consequences the other lines (shells) are not executed. @@ -91,10 +180,28 @@ jobs: util/android-commands.sh sync_host util/android-commands.sh build util/android-commands.sh tests - if [[ "${{ steps.rust-cache.outputs.cache-hit }}" != 'true' ]]; then util/android-commands.sh sync_image; fi; exit 0 + if [ "${{ steps.rust-cache.outputs.cache-hit }}" != 'true' ]; then util/android-commands.sh sync_image; fi; exit 0 + - name: Collect information about runner resources + if: always() + continue-on-error: true + run: | + free -mh + df -Th - name: Save rust cache if: steps.rust-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: ~/__rust_cache__ key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 + - name: archive any output (error screenshots) + if: always() + uses: actions/upload-artifact@v4 + with: + name: test_output_${{ env.AVD_CACHE_KEY }} + path: output + - name: Collect information about runner resources + if: always() + continue-on-error: true + run: | + free -mh + df -Th diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 00000000000..a5bbf42af4d --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,218 @@ +name: Code Quality + +# spell-checker:ignore (people) reactivecircus Swatinem dtolnay juliangruber pell taplo +# spell-checker:ignore (misc) TERMUX noaudio pkill swiftshader esac sccache pcoreutils shopt subshell dequote + +on: + pull_request: + push: + branches: + - '*' + +env: + # * style job configuration + STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis + +permissions: + contents: read # to fetch code (actions/checkout) + +# End the current execution if there is a new changeset in the PR. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + + style_format: + name: Style/format + runs-on: ${{ matrix.job.os }} + strategy: + fail-fast: false + matrix: + job: + - { os: ubuntu-latest , features: feat_os_unix } + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: rustfmt + - uses: Swatinem/rust-cache@v2 + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + # failure mode + unset FAIL_ON_FAULT ; case "$STYLE_FAIL_ON_FAULT" in + ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; + *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; + esac; + outputs FAIL_ON_FAULT FAULT_TYPE + - name: "`cargo fmt` testing" + shell: bash + run: | + ## `cargo fmt` testing + unset fault + fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" + fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') + # * convert any errors/warnings to GHA UI annotations; ref: + S=$(cargo fmt -- --check) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s\n" "$S" | sed -E -n -e "s/^Diff[[:space:]]+in[[:space:]]+${PWD//\//\\/}\/(.*)[[:space:]]+at[[:space:]]+[^0-9]+([0-9]+).*$/::${fault_type} file=\1,line=\2::${fault_prefix}: \`cargo fmt\`: style violation (file:'\1', line:\2; use \`cargo fmt -- \"\1\"\`)/p" ; fault=true ; } + if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi + - name: "cargo fmt on fuzz dir" + shell: bash + run: | + cd fuzz + cargo fmt --check + + style_lint: + name: Style/lint + runs-on: ${{ matrix.job.os }} + env: + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + strategy: + fail-fast: false + matrix: + job: + - { os: ubuntu-latest , features: all , workspace: true } + - { os: macos-latest , features: feat_os_macos } + - { os: windows-latest , features: feat_os_windows } + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy + - uses: Swatinem/rust-cache@v2 + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + # failure mode + unset FAIL_ON_FAULT ; case "$STYLE_FAIL_ON_FAULT" in + ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; + *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; + esac; + outputs FAIL_ON_FAULT FAULT_TYPE + - name: Install/setup prerequisites + shell: bash + run: | + ## Install/setup prerequisites + case '${{ matrix.job.os }}' in + ubuntu-*) + # selinux headers needed to enable all features + sudo apt-get -y install libselinux1-dev + ;; + esac + - name: "`cargo clippy` lint testing" + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + retry_on: error + timeout_minutes: 90 + shell: bash + command: | + ## `cargo clippy` lint testing + unset fault + fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" + fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') + # * convert any warnings to GHA UI annotations; ref: + if [[ "${{ matrix.job.features }}" == "all" ]]; then + extra="--all-features" + else + extra="--features ${{ matrix.job.features }}" + fi + case '${{ matrix.job.workspace-tests }}' in + 1|t|true|y|yes) + extra="${extra} --workspace" + ;; + esac + S=$(cargo clippy --all-targets $extra --tests -pcoreutils -- -D warnings 2>&1) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*$/::${fault_type} file=\2,line=\3,col=\4::${fault_prefix}: \`cargo clippy\`: \1 (file:'\2', line:\3)/p;" -e '}' ; fault=true ; } + if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi + + style_spellcheck: + name: Style/spelling + runs-on: ${{ matrix.job.os }} + strategy: + matrix: + job: + - { os: ubuntu-latest , features: feat_os_unix } + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + # failure mode + unset FAIL_ON_FAULT ; case "$STYLE_FAIL_ON_FAULT" in + ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; + *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; + esac; + outputs FAIL_ON_FAULT FAULT_TYPE + - name: Install/setup prerequisites + shell: bash + run: | + sudo apt-get -y update ; sudo apt-get -y install npm ; sudo npm install cspell -g ; + - name: Run `cspell` + shell: bash + run: | + ## Run `cspell` + unset fault + fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" + fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') + # * find cspell configuration ; note: avoid quotes around ${cfg_file} b/c `cspell` (v4) doesn't correctly dequote the config argument (or perhaps a subshell expansion issue?) + cfg_files=($(shopt -s nullglob ; echo {.vscode,.}/{,.}c[sS]pell{.json,.config{.js,.cjs,.json,.yaml,.yml},.yaml,.yml} ;)) + cfg_file=${cfg_files[0]} + unset CSPELL_CFG_OPTION ; if [ -n "$cfg_file" ]; then CSPELL_CFG_OPTION="--config $cfg_file" ; fi + S=$(cspell ${CSPELL_CFG_OPTION} --no-summary --no-progress .) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n "s/${PWD//\//\\/}\/(.*):(.*):(.*) - (.*)/::${fault_type} file=\1,line=\2,col=\3::${fault_type^^}: \4 (file:'\1', line:\2)/p" ; fault=true ; true ; } + if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi + + toml_format: + name: Style/toml + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check + run: npx --yes @taplo/cli fmt --check + + python: + name: Style/Python + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: ruff + uses: astral-sh/ruff-action@v3 + with: + src: "./util" + + - name: ruff - format + uses: astral-sh/ruff-action@v3 + with: + src: "./util" + args: format --check + - name: Run Python unit tests + shell: bash + run: | + python3 -m unittest util/test_compare_test_results.py diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index 095ec323098..6ba684719ed 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -1,12 +1,16 @@ name: FreeBSD -# spell-checker:ignore sshfs usesh vmactions taiki Swatinem esac fdescfs fdesc +# spell-checker:ignore sshfs usesh vmactions taiki Swatinem esac fdescfs fdesc sccache nextest copyback env: # * style job configuration STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis -on: [push, pull_request] +on: + pull_request: + push: + branches: + - '*' permissions: contents: read # to fetch code (actions/checkout) @@ -20,26 +24,30 @@ jobs: style: name: Style and Lint runs-on: ${{ matrix.job.os }} - timeout-minutes: 90 + timeout-minutes: 45 strategy: fail-fast: false matrix: job: - - { os: macos-12 , features: unix } ## GHA MacOS-11.0 VM won't have VirtualBox; refs: , + - { os: ubuntu-24.04 , features: unix } env: SCCACHE_GHA_ENABLED: "true" RUSTC_WRAPPER: "sccache" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Prepare, build and test - uses: vmactions/freebsd-vm@v0.3.1 + uses: vmactions/freebsd-vm@v1.2.0 with: usesh: true - # We need jq to run show-utils.sh and bash to use inline shell string replacement - prepare: pkg install -y curl sudo jq bash + sync: rsync + copyback: false + # We need jq and GNU coreutils to run show-utils.sh and bash to use inline shell string replacement + prepare: pkg install -y curl sudo jq coreutils bash run: | ## Prepare, build, and test # implementation modelled after ref: @@ -48,11 +56,11 @@ jobs: # TEST_USER=tester REPO_NAME=${GITHUB_WORKSPACE##*/} - WORKSPACE_PARENT="/Users/runner/work/${REPO_NAME}" + WORKSPACE_PARENT="/home/runner/work/${REPO_NAME}" WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}" # pw adduser -n ${TEST_USER} -d /root/ -g wheel -c "Coreutils user to build" -w random - chown -R ${TEST_USER}:wheel /root/ "/Users/runner/work/${REPO_NAME}"/ + chown -R ${TEST_USER}:wheel /root/ "${WORKSPACE_PARENT}"/ whoami # # Further work needs to be done in a sudo as we are changing users @@ -99,7 +107,7 @@ jobs: if [ -z "\${FAULT}" ]; then echo "## cargo clippy lint testing" # * convert any warnings to GHA UI annotations; ref: - S=\$(cargo clippy --all-targets \${CARGO_UTILITY_LIST_OPTIONS} -- -W clippy::manual_string_new -D warnings 2>&1) && printf "%s\n" "\$S" || { printf "%s\n" "\$S" ; printf "%s" "\$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*\$/::\${FAULT_TYPE} file=\2,line=\3,col=\4::\${FAULT_PREFIX}: \\\`cargo clippy\\\`: \1 (file:'\2', line:\3)/p;" -e '}' ; FAULT=true ; } + S=\$(cargo clippy --all-targets \${CARGO_UTILITY_LIST_OPTIONS} -- -D warnings 2>&1) && printf "%s\n" "\$S" || { printf "%s\n" "\$S" ; printf "%s" "\$S" | sed -E -n -e '/^error:/{' -e "N; s/^error:[[:space:]]+(.*)\\n[[:space:]]+-->[[:space:]]+(.*):([0-9]+):([0-9]+).*\$/::\${FAULT_TYPE} file=\2,line=\3,col=\4::\${FAULT_PREFIX}: \\\`cargo clippy\\\`: \1 (file:'\2', line:\3)/p;" -e '}' ; FAULT=true ; } fi # Clean to avoid to rsync back the files cargo clean @@ -109,26 +117,29 @@ jobs: test: name: Tests runs-on: ${{ matrix.job.os }} - timeout-minutes: 90 + timeout-minutes: 45 strategy: fail-fast: false matrix: job: - - { os: macos-12 , features: unix } ## GHA MacOS-11.0 VM won't have VirtualBox; refs: , + - { os: ubuntu-24.04 , features: unix } env: mem: 4096 SCCACHE_GHA_ENABLED: "true" RUSTC_WRAPPER: "sccache" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: Swatinem/rust-cache@v2 - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.9 - name: Prepare, build and test - uses: vmactions/freebsd-vm@v0.3.1 + uses: vmactions/freebsd-vm@v1.2.0 with: usesh: true - # sync: sshfs + sync: rsync + copyback: false prepare: pkg install -y curl gmake sudo run: | ## Prepare, build, and test @@ -141,12 +152,12 @@ jobs: # TEST_USER=tester REPO_NAME=${GITHUB_WORKSPACE##*/} - WORKSPACE_PARENT="/Users/runner/work/${REPO_NAME}" + WORKSPACE_PARENT="/home/runner/work/${REPO_NAME}" WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}" # pw adduser -n ${TEST_USER} -d /root/ -g wheel -c "Coreutils user to build" -w random # chown -R ${TEST_USER}:wheel /root/ "${WORKSPACE_PARENT}"/ - chown -R ${TEST_USER}:wheel /root/ "/Users/runner/work/${REPO_NAME}"/ + chown -R ${TEST_USER}:wheel /root/ "${WORKSPACE_PARENT}"/ whoami # # Further work needs to be done in a sudo as we are changing users diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml new file mode 100644 index 00000000000..e7da4b5926d --- /dev/null +++ b/.github/workflows/fuzzing.yml @@ -0,0 +1,281 @@ +name: Fuzzing + +# spell-checker:ignore fuzzer dtolnay Swatinem + +on: + pull_request: + push: + branches: + - '*' + +permissions: + contents: read # to fetch code (actions/checkout) + +# End the current execution if there is a new changeset in the PR. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + fuzz-build: + name: Build the fuzzers + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@nightly + - name: Install `cargo-fuzz` + run: cargo install cargo-fuzz + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "cargo-fuzz-cache-key" + cache-directories: "fuzz/target" + - name: Run `cargo-fuzz build` + run: cargo +nightly fuzz build + + fuzz-run: + needs: fuzz-build + name: Fuzz + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + RUN_FOR: 60 + strategy: + matrix: + test-target: + - { name: fuzz_test, should_pass: true } + # https://github.com/uutils/coreutils/issues/5311 + - { name: fuzz_date, should_pass: false } + - { name: fuzz_expr, should_pass: true } + - { name: fuzz_printf, should_pass: true } + - { name: fuzz_echo, should_pass: true } + - { name: fuzz_seq, should_pass: false } + - { name: fuzz_sort, should_pass: false } + - { name: fuzz_wc, should_pass: false } + - { name: fuzz_cut, should_pass: false } + - { name: fuzz_split, should_pass: false } + - { name: fuzz_tr, should_pass: false } + - { name: fuzz_env, should_pass: false } + - { name: fuzz_cksum, should_pass: false } + - { name: fuzz_parse_glob, should_pass: true } + - { name: fuzz_parse_size, should_pass: true } + - { name: fuzz_parse_time, should_pass: true } + - { name: fuzz_seq_parse_number, should_pass: true } + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@nightly + - name: Install `cargo-fuzz` + run: cargo install cargo-fuzz + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "cargo-fuzz-cache-key" + cache-directories: "fuzz/target" + - name: Restore Cached Corpus + uses: actions/cache/restore@v4 + with: + key: corpus-cache-${{ matrix.test-target.name }} + path: | + fuzz/corpus/${{ matrix.test-target.name }} + - name: Run ${{ matrix.test-target.name }} for XX seconds + id: run_fuzzer + shell: bash + continue-on-error: ${{ !matrix.test-target.name.should_pass }} + run: | + mkdir -p fuzz/stats + STATS_FILE="fuzz/stats/${{ matrix.test-target.name }}.txt" + cargo +nightly fuzz run ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE" + + # Extract key stats from the output + if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then + RUNS=$(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') + echo "runs=$RUNS" >> "$GITHUB_OUTPUT" + else + echo "runs=unknown" >> "$GITHUB_OUTPUT" + fi + + if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then + EXEC_RATE=$(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') + echo "exec_rate=$EXEC_RATE" >> "$GITHUB_OUTPUT" + else + echo "exec_rate=unknown" >> "$GITHUB_OUTPUT" + fi + + if grep -q "stat::new_units_added" "$STATS_FILE"; then + NEW_UNITS=$(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') + echo "new_units=$NEW_UNITS" >> "$GITHUB_OUTPUT" + else + echo "new_units=unknown" >> "$GITHUB_OUTPUT" + fi + + # Save should_pass value to file for summary job to use + echo "${{ matrix.test-target.should_pass }}" > "fuzz/stats/${{ matrix.test-target.name }}.should_pass" + + # Print stats to job output for immediate visibility + echo "----------------------------------------" + echo "FUZZING STATISTICS FOR ${{ matrix.test-target.name }}" + echo "----------------------------------------" + echo "Runs: $(grep -q "stat::number_of_executed_units" "$STATS_FILE" && grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}' || echo "unknown")" + echo "Execution Rate: $(grep -q "stat::average_exec_per_sec" "$STATS_FILE" && grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}' || echo "unknown") execs/sec" + echo "New Units: $(grep -q "stat::new_units_added" "$STATS_FILE" && grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}' || echo "unknown")" + echo "Expected: ${{ matrix.test-target.name.should_pass }}" + if grep -q "SUMMARY: " "$STATS_FILE"; then + echo "Status: $(grep "SUMMARY: " "$STATS_FILE" | head -1)" + else + echo "Status: Completed" + fi + echo "----------------------------------------" + + # Add summary to GitHub step summary + echo "### Fuzzing Results for ${{ matrix.test-target.name }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY + echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY + + if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then + echo "| Runs | $(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY + fi + + if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then + echo "| Execution Rate | $(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') execs/sec |" >> $GITHUB_STEP_SUMMARY + fi + + if grep -q "stat::new_units_added" "$STATS_FILE"; then + echo "| New Units | $(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY + fi + + echo "| Should pass | ${{ matrix.test-target.should_pass }} |" >> $GITHUB_STEP_SUMMARY + + if grep -q "SUMMARY: " "$STATS_FILE"; then + echo "| Status | $(grep "SUMMARY: " "$STATS_FILE" | head -1) |" >> $GITHUB_STEP_SUMMARY + else + echo "| Status | Completed |" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + - name: Save Corpus Cache + uses: actions/cache/save@v4 + with: + key: corpus-cache-${{ matrix.test-target.name }} + path: | + fuzz/corpus/${{ matrix.test-target.name }} + - name: Upload Stats + uses: actions/upload-artifact@v4 + with: + name: fuzz-stats-${{ matrix.test-target.name }} + path: | + fuzz/stats/${{ matrix.test-target.name }}.txt + fuzz/stats/${{ matrix.test-target.name }}.should_pass + retention-days: 5 + fuzz-summary: + needs: fuzz-run + name: Fuzzing Summary + runs-on: ubuntu-latest + if: always() + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Download all stats + uses: actions/download-artifact@v4 + with: + path: fuzz/stats-artifacts + pattern: fuzz-stats-* + merge-multiple: true + - name: Prepare stats directory + run: | + mkdir -p fuzz/stats + # Debug: List content of stats-artifacts directory + echo "Contents of stats-artifacts directory:" + find fuzz/stats-artifacts -type f | sort + + # Extract files from the artifact directories - handle nested directories + find fuzz/stats-artifacts -type f -name "*.txt" -exec cp {} fuzz/stats/ \; + find fuzz/stats-artifacts -type f -name "*.should_pass" -exec cp {} fuzz/stats/ \; + + # Debug information + echo "Contents of stats directory after extraction:" + ls -la fuzz/stats/ + echo "Contents of should_pass files (if any):" + cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found" + - name: Generate Summary + run: | + echo "# Fuzzing Summary" > fuzzing_summary.md + echo "" >> fuzzing_summary.md + echo "| Target | Runs | Exec/sec | New Units | Should pass | Status |" >> fuzzing_summary.md + echo "|--------|------|----------|-----------|-------------|--------|" >> fuzzing_summary.md + + TOTAL_RUNS=0 + TOTAL_NEW_UNITS=0 + + for stat_file in fuzz/stats/*.txt; do + TARGET=$(basename "$stat_file" .txt) + SHOULD_PASS_FILE="${stat_file%.*}.should_pass" + + # Get expected status + if [ -f "$SHOULD_PASS_FILE" ]; then + EXPECTED=$(cat "$SHOULD_PASS_FILE") + else + EXPECTED="unknown" + fi + + # Extract runs + if grep -q "stat::number_of_executed_units" "$stat_file"; then + RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}') + TOTAL_RUNS=$((TOTAL_RUNS + RUNS)) + else + RUNS="unknown" + fi + + # Extract execution rate + if grep -q "stat::average_exec_per_sec" "$stat_file"; then + EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}') + else + EXEC_RATE="unknown" + fi + + # Extract new units added + if grep -q "stat::new_units_added" "$stat_file"; then + NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}') + if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then + TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS)) + fi + else + NEW_UNITS="unknown" + fi + + # Extract status + if grep -q "SUMMARY: " "$stat_file"; then + STATUS=$(grep "SUMMARY: " "$stat_file" | head -1) + else + STATUS="Completed" + fi + + echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md + done + + echo "" >> fuzzing_summary.md + echo "## Overall Statistics" >> fuzzing_summary.md + echo "" >> fuzzing_summary.md + echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md + echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md + echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md + + # Add count by expected status + echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md + echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md + + # Write to GitHub step summary + cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY + - name: Show Summary + run: | + cat fuzzing_summary.md + - name: Upload Summary + uses: actions/upload-artifact@v4 + with: + name: fuzzing-summary + path: fuzzing_summary.md + retention-days: 5 diff --git a/.github/workflows/ignore-intermittent.txt b/.github/workflows/ignore-intermittent.txt index 759bd96eb8a..9e0e2ab0df6 100644 --- a/.github/workflows/ignore-intermittent.txt +++ b/.github/workflows/ignore-intermittent.txt @@ -1,3 +1,6 @@ -tests/tail-2/inotify-dir-recreate -tests/misc/timeout +tests/tail/inotify-dir-recreate +tests/timeout/timeout tests/rm/rm1 +tests/misc/stdbuf +tests/misc/usage_vs_getopt +tests/misc/tee diff --git a/.gitignore b/.gitignore index ed4e54ec5bc..7528e5f5380 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +# spell-checker:ignore (misc) direnv + target/ +coverage/ /src/*/gen_table /build/ /tmp/ @@ -10,9 +13,11 @@ target/ .*.swp .*.swo .idea -Cargo.lock lib*.a /docs/_build *.iml ### macOS ### .DS_Store + +### direnv ### +/.direnv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f90466bed2f..534487abc8f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,14 +4,20 @@ repos: - id: rust-linting name: Rust linting description: Run cargo fmt on files included in the commit. - entry: cargo +nightly fmt -- + entry: cargo +stable fmt -- pass_filenames: true types: [file, rust] language: system - id: rust-clippy name: Rust clippy description: Run cargo clippy on files included in the commit. - entry: cargo +nightly clippy --workspace --all-targets --all-features -- + entry: cargo +stable clippy --workspace --all-targets --all-features -- -D warnings pass_filenames: false types: [file, rust] language: system + - id: cspell + name: Code spell checker (cspell) + description: Run cspell to check for spelling errors. + entry: cspell --no-must-find-files -- + pass_filenames: true + language: system diff --git a/.vscode/cSpell.json b/.vscode/cSpell.json index 6ceb038c218..199830c2d1d 100644 --- a/.vscode/cSpell.json +++ b/.vscode/cSpell.json @@ -1,4 +1,5 @@ // `cspell` settings +// spell-checker:ignore oranda { // version of the setting file "version": "0.2", @@ -18,15 +19,19 @@ // files to ignore (globs supported) "ignorePaths": [ + ".git/**", "Cargo.lock", "oranda.json", "target/**", "tests/**/fixtures/**", "src/uu/dd/test-resources/**", "vendor/**", - "**/*.svg" + "**/*.svg", + "src/uu/*/locales/*.ftl" ], + "enableGlobDot": true, + // words to ignore (even if they are in the flagWords) "ignoreWords": [], diff --git a/.vscode/cspell.dictionaries/acronyms+names.wordlist.txt b/.vscode/cspell.dictionaries/acronyms+names.wordlist.txt index 8711913d99d..8993f5d6a31 100644 --- a/.vscode/cspell.dictionaries/acronyms+names.wordlist.txt +++ b/.vscode/cspell.dictionaries/acronyms+names.wordlist.txt @@ -37,6 +37,7 @@ aarch flac impls lzma +loongarch # * names BusyBox @@ -45,6 +46,7 @@ Codacy Cygwin Deno EditorConfig +EPEL FreeBSD Gmail GNU @@ -58,6 +60,7 @@ MinGW Minix NetBSD Novell +Nushell OpenBSD POSIX PowerPC diff --git a/.vscode/cspell.dictionaries/jargon.wordlist.txt b/.vscode/cspell.dictionaries/jargon.wordlist.txt index dca883dc804..6358f3c7682 100644 --- a/.vscode/cspell.dictionaries/jargon.wordlist.txt +++ b/.vscode/cspell.dictionaries/jargon.wordlist.txt @@ -10,8 +10,10 @@ bytewise canonicalization canonicalize canonicalizing +capget codepoint codepoints +codeready codegen colorizable colorize @@ -42,8 +44,11 @@ fileio filesystem filesystems flamegraph +fsxattr fullblock getfacl +getfattr +getopt gibi gibibytes glob @@ -63,6 +68,7 @@ kibi kibibytes libacl lcase +llistxattr lossily lstat mebi @@ -73,6 +79,7 @@ microbenchmarks microbenchmarking multibyte multicall +nmerge noatime nocache nocreat @@ -83,6 +90,7 @@ nolinks nonblock nonportable nonprinting +nonseekable notrunc noxfer ofile @@ -105,7 +113,9 @@ seedable semver semiprime semiprimes +setcap setfacl +setfattr shortcode shortcodes siginfo @@ -133,11 +143,15 @@ urand whitespace wordlist wordlists +xattrs +xpass # * abbreviations consts deps dev +fdlimit +inacc maint proc procs @@ -153,3 +167,9 @@ retval subdir val vals +inval +nofield + +# * clippy +uninlined +nonminimal diff --git a/.vscode/cspell.dictionaries/shell.wordlist.txt b/.vscode/cspell.dictionaries/shell.wordlist.txt index 95dea94a7cd..b1ddec7a4e0 100644 --- a/.vscode/cspell.dictionaries/shell.wordlist.txt +++ b/.vscode/cspell.dictionaries/shell.wordlist.txt @@ -25,6 +25,7 @@ sudoedit tcsh tzselect urandom +VARNAME wtmp zsh @@ -102,3 +103,4 @@ xargs # * directories sbin +libexec diff --git a/.vscode/cspell.dictionaries/workspace.wordlist.txt b/.vscode/cspell.dictionaries/workspace.wordlist.txt index 6d6533bcf5c..bbdb825198b 100644 --- a/.vscode/cspell.dictionaries/workspace.wordlist.txt +++ b/.vscode/cspell.dictionaries/workspace.wordlist.txt @@ -20,12 +20,12 @@ exacl filetime formatteriteminfo fsext -fundu getopts getrandom globset indicatif itertools +langid lscolors mdbook memchr @@ -47,6 +47,7 @@ termsize termwidth textwrap thiserror +unic ureq walkdir winapi @@ -59,6 +60,7 @@ clippy rustc rustfmt rustup +rustdoc # bitor # BitOr trait function bitxor # BitXor trait function @@ -135,6 +137,7 @@ vmsplice # * vars/libc COMFOLLOW +EXDEV FILENO FTSENT HOSTSIZE @@ -165,6 +168,7 @@ RTLD_NEXT RTLD SIGINT SIGKILL +SIGSTOP SIGTERM SYS_fdatasync SYS_syncfs @@ -322,15 +326,21 @@ libc libstdbuf musl tmpd +uchild ucmd ucommand utmpx uucore uucore_procs uudoc +uufuzz uumain uutil +uutests uutils # * function names getcwd + +# * other +algs diff --git a/.vscode/extensions.json b/.vscode/extensions.json index b9c161a4c28..7ee2695db0a 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,4 +1,4 @@ -// spell-checker:ignore (misc) matklad +// spell-checker:ignore (misc) matklad foxundermoon // see for the documentation about the extensions.json format // * // "foxundermoon.shell-format" ~ shell script formatting ; note: ENABLE "Use EditorConfig" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd87e2d05ba..8668c9a27eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,236 +1,273 @@ - + # Contributing to coreutils -Contributions are very welcome via Pull Requests. If you don't know where to -start, take a look at the -[`good-first-issues`](https://github.com/uutils/coreutils/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22). -If you have any questions, feel free to ask them in the issues or on -[Discord](https://discord.gg/wQVJbvJ). - -## Best practices - -1. Follow what GNU is doing in terms of options and behavior. It is recommended - to look at the GNU Coreutils manual ([on the - web](https://www.gnu.org/software/coreutils/manual/html_node/index.html), or - locally using `info `). It is more in depth than the man pages and - provides a good description of available features and their implementation - details. -1. If possible, look at the GNU test suite execution in the CI and make the test - work if failing. -1. Use clap for argument management. -1. Make sure that the code coverage is covering all of the cases, including - errors. -1. The code must be clippy-warning-free and rustfmt-compliant. -1. Don't hesitate to move common functions into uucore if they can be reused by - other binaries. -1. Unsafe code should be documented with Safety comments. -1. uutils is original code. It cannot contain code from existing GNU or Unix-like - utilities, nor should it link to or reference GNU libraries. +Hi! Welcome to uutils/coreutils! -## Platforms +Thanks for wanting to contribute to this project! This document explains +everything you need to know to contribute. Before you start make sure to also +check out these documents: -We take pride in supporting many operating systems and architectures. Any code -you contribute must at least compile without warnings for all platforms in the -CI. However, you can use `#[cfg(...)]` attributes to create platform dependent features. +- Our community's [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md). +- [DEVELOPMENT.md](./DEVELOPMENT.md) for setting up your development + environment. -**Tip:** For Windows, Microsoft provides some images (VMWare, Hyper-V, -VirtualBox and Parallels) for development: - +Now follows a very important warning: -## Tools +> [!WARNING] +> uutils is original code and cannot contain any code from GNU or +> other implementations. This means that **we cannot accept any changes based on +> the GNU source code**. To make sure that cannot happen, **you cannot link to +> the GNU source code** either. It is however possible to look at other implementations +> under a BSD or MIT license like [Apple's implementation](https://github.com/apple-oss-distributions/file_cmds/) +> or [OpenBSD](https://github.com/openbsd/src/tree/master/bin). -We have an extensive CI that will check your code before it can be merged. This -section explains how to run those checks locally to avoid waiting for the CI. +Finally, feel free to join our [Discord](https://discord.gg/wQVJbvJ)! -### pre-commit hooks +## Getting Oriented -A configuration for `pre-commit` is provided in the repository. It allows -automatically checking every git commit you make to ensure it compiles, and -passes `clippy` and `rustfmt` without warnings. +uutils is a big project consisting of many parts. Here are the most important +parts for getting started: -To use the provided hook: +- [`src/uu`](https://github.com/uutils/coreutils/tree/main/src/uu/): The code for all utilities +- [`src/uucore`](https://github.com/uutils/coreutils/tree/main/src/uucore/): Crate containing all the shared code between + the utilities. +- [`tests/by-util`](https://github.com/uutils/coreutils/tree/main/tests/by-util/): The tests for all utilities. +- [`src/bin/coreutils.rs`](https://github.com/uutils/coreutils/tree/main/src/bin/coreutils.rs): Code for the multicall + binary. +- [`docs`](https://github.com/uutils/coreutils/tree/main/docs/src): the documentation for the website +- [`tests/uutests/`](https://github.com/uutils/coreutils/tree/main/tests/uutests/): + Crate implementing the various functions to test uutils commands. -1. [Install `pre-commit`](https://pre-commit.com/#install) -1. Run `pre-commit install` while in the repository directory +Each utility is defined as a separate crate. The structure of each of these +crates is as follows: -Your git commits will then automatically be checked. If a check fails, an error -message will explain why, and your commit will be canceled. You can then make -the suggested changes, and run `git commit ...` again. +- `Cargo.toml` +- `src/main.rs`: contains only a single macro call +- `src/.rs`: the actual code for the utility +- `.md`: the documentation for the utility -### clippy +We have separated repositories for crates that we maintain but also publish for +use by others: -```shell -cargo clippy --all-targets --all-features -``` +- [uutils-term-grid](https://github.com/uutils/uutils-term-grid) +- [parse_datetime](https://github.com/uutils/parse_datetime) -The `msrv` key in the clippy configuration file `clippy.toml` is used to disable -lints pertaining to newer features by specifying the minimum supported Rust -version (MSRV). +## Design Goals -### rustfmt +We have the following goals with our development: -```shell -cargo fmt --all -``` +- **Compatible**: The utilities should be a drop-in replacement for the GNU + coreutils. +- **Cross-platform**: All utilities should run on as many of the supported + platforms as possible. +- **Reliable**: The utilities should never unexpectedly fail. +- **Performant**: Our utilities should be written in fast idiomatic Rust. We aim + to match or exceed the performance of the GNU utilities. +- **Well-tested**: We should have a lot of tests to be able to guarantee + reliability and compatibility. -### cargo-deny +## How to Help -This project uses [cargo-deny](https://github.com/EmbarkStudios/cargo-deny/) to -detect duplicate dependencies, checks licenses, etc. To run it locally, first -install it and then run with: +There are several ways to help and writing code is just one of them. Reporting +issues and writing documentation are just as important as writing code. -``` -cargo deny --all-features check all -``` +### Reporting Issues -### Markdown linter +We can't fix bugs we don't know about, so good issues are super helpful! Here +are some tips for writing good issues: -We use [markdownlint](https://github.com/DavidAnson/markdownlint) to lint the -Markdown files in the repository. +- If you find a bug, make sure it's still a problem on the `main` branch. +- Search through the existing issues to see whether it has already been + reported. +- Make sure to include all relevant information, such as: + - Which version of uutils did you check? + - Which version of GNU coreutils are you comparing with? + - What platform are you on? +- Provide a way to reliably reproduce the issue. +- Be as specific as possible! -### Spell checker +### Writing Documentation -We use `cspell` as spell checker for all files in the project. If you are using -VS Code, you can install the -[code spell checker](https://marketplace.visualstudio.com/items?itemName=streetsidesoftware.code-spell-checker) -extension to enable spell checking within your editor. Otherwise, you can -install [cspell](https://cspell.org/) separately. +There's never enough documentation. If you come across any documentation that +could be improved, feel free to submit a PR for it! -If you want to make the spell checker ignore a word, you can add +### Writing Code -```rust -// spell-checker:ignore word_to_ignore -``` +If you want to submit a PR, make sure that you've discussed the solution with +the maintainers beforehand. We want to avoid situations where you put a lot of +work into a fix that we can't merge! If there's no issue for what you're trying +to fix yet, make one _before_ you start working on the PR. -at the top of the file. +Generally, we try to follow what GNU is doing in terms of options and behavior. +It is recommended to look at the GNU coreutils manual +([on the web](https://www.gnu.org/software/coreutils/manual/html_node/index.html), +or locally using `info `). It is more in depth than the man pages and +provides a good description of available features and their implementation +details. But remember, you cannot look at the GNU source code! -## Testing +Also remember that we can only merge PRs which pass our test suite, follow +rustfmt, and do not have any warnings from clippy. See +[DEVELOPMENT.md](./DEVELOPMENT.md) for more information. Be sure to also read +about our [Rust style](#our-rust-style). -Testing can be done using either Cargo or `make`. +## Our Rust Style -### Testing with Cargo +We want uutils to be written in idiomatic Rust, so here are some guidelines to +follow. Some of these are aspirational, meaning that we don't do them correctly +everywhere in the code. If you find violations of the advice below, feel free to +submit a patch! -Just like with building, we follow the standard procedure for testing using -Cargo: +### Don't `panic!` -```shell -cargo test -``` +The coreutils should be very reliable. This means that we should never `panic!`. +Therefore, you should avoid using `.unwrap()` and `panic!`. Sometimes the use of +`unreachable!` can be justified with a comment explaining why that code is +unreachable. -By default, `cargo test` only runs the common programs. To run also platform -specific tests, run: +### Don't `exit` -```shell -cargo test --features unix -``` +We want uutils to be embeddable in other programs. This means that no function +in uutils should exit the program. Doing so would also lead to code with more +confusing control flow. Avoid therefore `std::process::exit` and similar +functions which exit the program early. -If you would prefer to test a select few utilities: +### `unsafe` -```shell -cargo test --features "chmod mv tail" --no-default-features -``` +uutils cannot be entirely safe, because we have to call out to `libc` and do +syscalls. However, we still want to limit our use of `unsafe`. We generally only +accept `unsafe` for FFI, with very few exceptions. Note that performance is very +rarely a valid argument for using `unsafe`. -If you also want to test the core utilities: +If you still need to write code with `unsafe`, make sure to read the +[Rustonomicon](https://doc.rust-lang.org/nomicon/intro.html) and annotate the +calls with `// SAFETY:` comments explaining why the use of `unsafe` is sound. -```shell -cargo test -p uucore -p coreutils -``` +### Macros -Running the complete test suite might take a while. We use [nextest](https://nexte.st/index.html) in -the CI and you might want to try it out locally. It can speed up the execution time of the whole -test run significantly if the cpu has multiple cores. +Macros can be a great tool, but they are also usually hard to understand. They +should be used sparingly. Make sure to explore simpler options before you reach +for a solution involving macros. -```shell -cargo nextest run --features unix --no-fail-fast -``` +### `str`, `OsStr` & `Path` -To debug: +Rust has many string-like types, and sometimes it's hard to choose the right +one. It's tempting to use `str` (and `String`) for everything, but that is not +always the right choice for uutils, because we need to support invalid UTF-8, +just like the GNU coreutils. For example, paths on Linux might not be valid +UTF-8! Whenever we are dealing with paths, we should therefore stick with +`OsStr` and `Path`. Make sure that you only convert to `str`/`String` if you +know that something is always valid UTF-8. If you need more operations on +`OsStr`, you can use the [`bstr`](https://docs.rs/bstr/latest/bstr/) crate. + +### Doc-comments + +We use rustdoc for our documentation, so it's best to follow +[rustdoc's guidelines](https://doc.rust-lang.org/rustdoc/how-to-write-documentation.html#documenting-components). +Make sure that your documentation is not just repeating the name of the +function, but actually giving more useful information. Rustdoc recommends the +following structure: -```shell -gdb --args target/debug/coreutils ls -(gdb) b ls.rs:79 -(gdb) run ``` +[short sentence explaining what it is] -### Testing with GNU Make +[more detailed explanation] -To simply test all available utilities: +[at least one code example that users can copy/paste to try it] -```shell -make test +[even more advanced explanations if necessary] ``` -To test all but a few of the available utilities: +### Other comments -```shell -make SKIP_UTILS='UTILITY_1 UTILITY_2' test -``` +Comments should be written to _explain_ the code, not to _describe_ the code. +Try to focus on explaining _why_ the code is the way it is. If you feel like you +have to describe the code, that's usually a sign that you could improve the +naming of variables and functions. -To test only a few of the available utilities: +If you edit a piece of code, make sure to update any comments that need to +change as a result. The only thing worse than having no comments is having +outdated comments! -```shell -make UTILS='UTILITY_1 UTILITY_2' test -``` +## Git Etiquette -To include tests for unimplemented behavior: +To ensure easy collaboration, we have guidelines for using Git and GitHub. -```shell -make UTILS='UTILITY_1 UTILITY_2' SPEC=y test -``` +### Commits -To run tests with `nextest` just use the nextest target. Note you'll need to -[install](https://nexte.st/book/installation.html) `nextest` first. The `nextest` target accepts the -same arguments like the default `test` target, so it's possible to pass arguments to `nextest run` -via `CARGOFLAGS`: +- Make small and atomic commits. +- Keep a clean history of commits. +- Write informative commit messages. +- Annotate your commit message with the component you're editing. For example: + `cp: do not overwrite on with -i` or `uucore: add support for FreeBSD`. +- Do not unnecessarily move items around in the code. This makes the changes + much harder to review. If you do need to move things around, do that in a + separate commit. -```shell -make CARGOFLAGS='--no-fail-fast' UTILS='UTILITY_1 UTILITY_2' nextest -``` +### Commit messages -### Run Busybox Tests +You can read [this section in the Git book](https://git-scm.com/book/ms/v2/Distributed-Git-Contributing-to-a-Project) to learn how to write good commit +messages. -This testing functionality is only available on *nix operating systems and -requires `make`. +In addition, here are a few examples for a summary line when committing to +uutils: -To run busybox tests for all utilities for which busybox has tests +- commit for a single utility -```shell -make busytest +``` +nohup: cleanup and refactor ``` -To run busybox tests for a few of the available utilities +- commit for a utility's tests -```shell -make UTILS='UTILITY_1 UTILITY_2' busytest +``` +tests/rm: test new feature ``` -To pass an argument like "-v" to the busybox test runtime +Beyond changes to an individual utility or its tests, other summary lines for +non-utility modules include: -```shell -make UTILS='UTILITY_1 UTILITY_2' RUNTEST_ARGS='-v' busytest ``` - -### Comparing with GNU - -To run uutils against the GNU test suite locally, run the following commands: - -```shell -bash util/build-gnu.sh -# Build uutils without release optimizations -UU_MAKE_PROFILE=debug bash util/build-gnu.sh -bash util/run-gnu-test.sh -# To run a single test: -bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example -# To run several tests: -bash util/run-gnu-test.sh tests/touch/not-owner.sh tests/rm/no-give-up.sh # for example -# If this is a perl (.pl) test, to run in debug: -DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl +README: add help +uucore: add new modules +uutils: add new utility +gitignore: add temporary files ``` -Note that it relies on individual utilities (not the multicall binary). +### PRs + +- Make the titles of PRs descriptive. + - This means describing the problem you solve. For example, do not write + `Fix #1234`, but `ls: fix version sort order`. + - You can prefix the title with the utility the PR concerns. +- Keep PRs small and self-contained. A set of small PRs is much more likely to + get merged quickly than one large PR. +- Make sure the CI passes (up to intermittently failing tests). +- You know your code best, that's why it's best if you can solve merge conflicts + on your branch yourself. + - It's up to you whether you want to use `git merge main` or + `git rebase main`. + - Feel free to ask for help with merge conflicts. +- You do not need to ping maintainers to request a review, but it's fine to do + so if you don't get a response within a few days. -### Improving the GNU compatibility +## Platforms + +We take pride in supporting many operating systems and architectures. Any code +you contribute must at least compile without warnings for all platforms in the +CI. However, you can use `#[cfg(...)]` attributes to create platform dependent +features. + +**Tip:** For Windows, Microsoft provides some images (VMWare, Hyper-V, +VirtualBox and Parallels) for development [on their official download page](https://developer.microsoft.com/windows/downloads/virtual-machines/). + +## Improving the GNU compatibility + +Please make sure you have installed +[GNU utils and prerequisites](DEVELOPMENT.md#gnu-utils-and-prerequisites) and +can execute commands described in +[Comparing with GNU](DEVELOPMENT.md#comparing-with-gnu) section of +[DEVELOPMENT.md](DEVELOPMENT.md) The Python script `./util/remaining-gnu-error.py` shows the list of failing tests in the CI. @@ -254,112 +291,29 @@ To improve the GNU compatibility, the following process is recommended: 1. Start to modify the Rust implementation to match the expected behavior 1. Add a test to make sure that we don't regress (our test suite is super quick) -## Commit messages - -To help the project maintainers review pull requests from contributors across -numerous utilities, the team has settled on conventions for commit messages. - -From : - -``` -Capitalized, short (50 chars or less) summary - -More detailed explanatory text, if necessary. Wrap it to about 72 -characters or so. In some contexts, the first line is treated as the -subject of an email and the rest of the text as the body. The blank -line separating the summary from the body is critical (unless you omit -the body entirely); tools like rebase will confuse you if you run the -two together. - -Write your commit message in the imperative: "Fix bug" and not "Fixed bug" -or "Fixes bug." This convention matches up with commit messages generated -by commands like git merge and git revert. - -Further paragraphs come after blank lines. - - - Bullet points are okay, too - - - Typically a hyphen or asterisk is used for the bullet, followed by a - single space, with blank lines in between, but conventions vary here - - - Use a hanging indent -``` - -Furthermore, here are a few examples for a summary line: - -* commit for a single utility - -``` -nohup: cleanup and refactor -``` - -* commit for a utility's tests - -``` -tests/rm: test new feature -``` - -Beyond changes to an individual utility or its tests, other summary -lines for non-utility modules include: - -``` -README: add help -``` - -``` -uucore: add new modules -``` - -``` -uutils: add new utility -``` - -``` -gitignore: add temporary files -``` - ## Code coverage - - -Code coverage report can be generated using [grcov](https://github.com/mozilla/grcov). - -### Using Nightly Rust - -To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-a-rust-project) coverage report - -```shell -export CARGO_INCREMENTAL=0 -export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" -export RUSTDOCFLAGS="-Cpanic=abort" -cargo build # e.g., --features feat_os_unix -cargo test # e.g., --features feat_os_unix test_pathchk -grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing --ignore build.rs --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?\#\[derive\()" -o ./target/debug/coverage/ -# open target/debug/coverage/index.html in browser -``` - -if changes are not reflected in the report then run `cargo clean` and run the above commands. - -### Using Stable Rust - -If you are using stable version of Rust that doesn't enable code coverage instrumentation by default -then add `-Z-Zinstrument-coverage` flag to `RUSTFLAGS` env variable specified above. +To generate code coverage report locally please follow +[Code coverage report](DEVELOPMENT.md#code-coverage-report) section of +[DEVELOPMENT.md](DEVELOPMENT.md) ## Other implementations -The Coreutils have different implementations, with different levels of completions: +The Coreutils have different implementations, with different levels of +completions: -* [GNU's](https://git.savannah.gnu.org/gitweb/?p=coreutils.git) -* [OpenBSD](https://github.com/openbsd/src/tree/master/bin) -* [Busybox](https://github.com/mirror/busybox/tree/master/coreutils) -* [Toybox (Android)](https://github.com/landley/toybox/tree/master/toys/posix) -* [V lang](https://github.com/vlang/coreutils) -* [SerenityOS](https://github.com/SerenityOS/serenity/tree/master/Userland/Utilities) -* [Initial Unix](https://github.com/dspinellis/unix-history-repo) -* [Perl Power Tools](https://metacpan.org/pod/PerlPowerTools) +- [GNU's](https://git.savannah.gnu.org/gitweb/?p=coreutils.git) +- [OpenBSD](https://github.com/openbsd/src/tree/master/bin) +- [Busybox](https://github.com/mirror/busybox/tree/master/coreutils) +- [Toybox (Android)](https://github.com/landley/toybox/tree/master/toys/posix) +- [Mac OS](https://github.com/apple-oss-distributions/file_cmds/) +- [V lang](https://github.com/vlang/coreutils) +- [SerenityOS](https://github.com/SerenityOS/serenity/tree/master/Userland/Utilities) +- [Initial Unix](https://github.com/dspinellis/unix-history-repo) +- [Perl Power Tools](https://metacpan.org/pod/PerlPowerTools) -However, when reimplementing the tools/options in Rust, don't read their source codes -when they are using reciprocal licenses (ex: GNU GPL, GNU LGPL, etc). +However, when reimplementing the tools/options in Rust, don't read their source +codes when they are using reciprocal licenses (ex: GNU GPL, GNU LGPL, etc). ## Licensing @@ -374,17 +328,17 @@ If you wish to add or change dependencies as part of a contribution to the project, a tool like `cargo-license` can be used to show their license details. The following types of license are acceptable: -* MIT License -* Dual- or tri-license with an MIT License option ("Apache-2.0 or MIT" is a +- MIT License +- Dual- or tri-license with an MIT License option ("Apache-2.0 or MIT" is a popular combination) -* "MIT equivalent" license (2-clause BSD, 3-clause BSD, ISC) -* License less restrictive than the MIT License (CC0 1.0 Universal) -* Apache License version 2.0 +- "MIT equivalent" license (2-clause BSD, 3-clause BSD, ISC) +- License less restrictive than the MIT License (CC0 1.0 Universal) +- Apache License version 2.0 Licenses we will not use: -* An ambiguous license, or no license -* Strongly reciprocal licenses (GNU GPL, GNU LGPL) +- An ambiguous license, or no license +- Strongly reciprocal licenses (GNU GPL, GNU LGPL) If you wish to add a reference but it doesn't meet these requirements, please raise an issue to describe the dependency. diff --git a/Cargo.lock b/Cargo.lock index f4885faa05c..d447ee4e408 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,22 +1,28 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -32,79 +38,99 @@ dependencies = [ "libc", ] +[[package]] +name = "ansi-width" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219e3ce6f2611d83b51ec2098a12702112c29e57203a6b0a0929b2cddb486608" +dependencies = [ + "unicode-width 0.1.14", +] + [[package]] name = "anstream" -version = "0.3.2" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "once_cell", + "windows-sys 0.59.0", +] + +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", ] [[package]] name = "arrayref" -version = "0.3.6" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bigdecimal" -version = "0.4.0" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5274a6b6e0ee020148397245b973e30163b7bffbc6d473613f850cb99888581e" +checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" dependencies = [ + "autocfg", "libm", "num-bigint", "num-integer", @@ -120,26 +146,44 @@ dependencies = [ "compare", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" -version = "0.63.0" +version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d860121800b2a9a94f9b5604b332d5cffb234ce17609ea479d723dbc9d3885" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools 0.13.0", "log", - "peeking_take_while", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 1.0.109", - "which", + "syn", ] [[package]] @@ -150,15 +194,27 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" [[package]] -name = "blake2b_simd" +name = "bitvec" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2b_simd" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06e903a20b159e944f91ec8499fe1e55651480c541ea0a584f5d967c49ad9d99" dependencies = [ "arrayref", "arrayvec", @@ -167,32 +223,31 @@ dependencies = [ [[package]] name = "blake3" -version = "1.4.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729b71f35bd3fa1a4c86b85d32c8b9069ea7fe14f7a53cfabb65f62d4265b888" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "digest", ] [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] name = "bstr" -version = "1.6.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", "regex-automata", @@ -201,27 +256,30 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytecount" -version = "0.6.3" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cc" -version = "1.0.79" +version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +dependencies = [ + "shlex", +] [[package]] name = "cexpr" @@ -229,7 +287,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -238,23 +296,50 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" -version = "0.4.28" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-targets 0.48.0", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efdce149c370f133a071ca8ef6ea340b7b88748ab0810097a9e2976eaa34b4f3" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" +dependencies = [ + "parse-zoneinfo", + "phf_codegen", ] [[package]] name = "clang-sys" -version = "1.4.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -263,47 +348,46 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.21" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" +checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.3.21" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" +checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" dependencies = [ "anstream", "anstyle", "clap_lex", - "once_cell", "strsim", "terminal_size", ] [[package]] name = "clap_complete" -version = "4.3.0" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a04ddfaacc3bc9e6ea67d024575fafc2a813027cf374b8f24f7bc233c6b6be12" +checksum = "8d2267df7f3c8e74e38268887ea5235d4dfadd39bfff2d56ab82d61776be355e" dependencies = [ "clap", ] [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "clap_mangen" -version = "0.2.9" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0f09a0ca8f0dd8ac92c546b426f466ef19828185c6d504c80c48c9c2768ed9" +checksum = "724842fa9b144f9b89b3f3d371a89f3455eea660361d13a554f68f8ae5d6c13a" dependencies = [ "clap", "roff", @@ -311,9 +395,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "compare" @@ -323,86 +407,85 @@ checksum = "120133d4db2ec47efe2e26502ee984747630c67f51974fca0b6c1340cf2368d3" [[package]] name = "console" -version = "0.15.7" +version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ "encode_unicode", - "lazy_static", "libc", - "unicode-width", - "windows-sys 0.45.0", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", ] [[package]] name = "const-random" -version = "0.1.15" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", - "proc-macro-hack", ] [[package]] name = "const-random-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", - "proc-macro-hack", "tiny-keccak", ] [[package]] name = "constant_time_eq" -version = "0.2.4" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] -name = "conv" -version = "0.3.3" +name = "convert_case" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ff10625fd0ac447827aa30ea8b861fead473bb60aeb73af6c1c58caf0d1299" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" dependencies = [ - "custom_derive", + "unicode-segmentation", ] [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "coreutils" -version = "0.0.21" +version = "0.1.0" dependencies = [ + "bincode", "chrono", "clap", "clap_complete", "clap_mangen", - "conv", + "ctor", "filetime", "glob", "hex-literal", - "is-terminal", "libc", "nix", - "once_cell", + "num-prime", "phf", "phf_codegen", "pretty_assertions", "procfs", - "rand", - "rand_pcg", + "rand 0.9.1", "regex", "rlimit", "rstest", "selinux", + "serde", + "serde-big-array", "sha1", "tempfile", "textwrap", @@ -474,7 +557,6 @@ dependencies = [ "uu_pwd", "uu_readlink", "uu_realpath", - "uu_relpath", "uu_rm", "uu_rmdir", "uu_runcon", @@ -513,60 +595,52 @@ dependencies = [ "uu_yes", "uucore", "uuhelp_parser", + "uutests", "walkdir", + "xattr", "zip", ] -[[package]] -name = "coz" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef55b3fe2f5477d59e12bc792e8b3c95a25bd099eadcfae006ecea136de76e2" -dependencies = [ - "libc", - "once_cell", -] - [[package]] name = "cpp" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa65869ef853e45c60e9828aa08cdd1398cb6e13f3911d9cb2a079b144fcd64" +checksum = "f36bcac3d8234c1fb813358e83d1bb6b0290a3d2b3b5efc6b88bfeaf9d8eec17" dependencies = [ "cpp_macros", ] [[package]] name = "cpp_build" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e361fae2caf9758164b24da3eedd7f7d7451be30d90d8e7b5d2be29a2f0cf5b" +checksum = "27f8638c97fbd79cc6fc80b616e0e74b49bac21014faed590bbc89b7e2676c90" dependencies = [ "cc", "cpp_common", "lazy_static", "proc-macro2", "regex", - "syn 2.0.23", + "syn", "unicode-xid", ] [[package]] name = "cpp_common" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1a2532e4ed4ea13031c13bc7bc0dbca4aae32df48e9d77f0d1e743179f2ea1" +checksum = "25fcfea2ee05889597d35e986c2ad0169694320ae5cc8f6d2640a4bb8a884560" dependencies = [ "lazy_static", "proc-macro2", - "syn 2.0.23", + "syn", ] [[package]] name = "cpp_macros" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47ec9cc90633446f779ef481a9ce5a0077107dd5b87016440448d908625a83fd" +checksum = "d156158fe86e274820f5a53bc9edb0885a6e7113909497aa8d883b69dd171871" dependencies = [ "aho-corasick", "byteorder", @@ -574,81 +648,66 @@ dependencies = [ "lazy_static", "proc-macro2", "quote", - "syn 2.0.23", + "syn", ] [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.8" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", - "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" -dependencies = [ - "cfg-if", -] +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" -version = "0.27.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.9.0", "crossterm_winapi", - "libc", + "derive_more", + "document-features", + "filedescriptor", "mio", "parking_lot", + "rustix 1.0.1", "signal-hook", "signal-hook-mio", "winapi", @@ -665,9 +724,9 @@ dependencies = [ [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-common" @@ -680,32 +739,42 @@ dependencies = [ ] [[package]] -name = "ctrlc" -version = "3.4.0" +name = "ctor" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a011bbe2c35ce9c1f143b7af6f94f29a167beb4cd1d29e6740ce836f723120e" +checksum = "a4735f265ba6a1188052ca32d461028a7d1125868be18e287e756019da7607b5" dependencies = [ - "nix", - "windows-sys 0.48.0", + "ctor-proc-macro", + "dtor", ] [[package]] -name = "custom_derive" -version = "0.1.7" +name = "ctor-proc-macro" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" +checksum = "4f211af61d8efdd104f96e57adf5e426ba1bc3ed7a4ead616e15e5881fd79c4d" + +[[package]] +name = "ctrlc" +version = "3.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73" +dependencies = [ + "nix", + "windows-sys 0.59.0", +] [[package]] name = "data-encoding" -version = "2.4.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "data-encoding-macro" -version = "0.1.13" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c904b33cc60130e1aeea4956ab803d08a3f4a0ca82d64ed757afac3891f2bb99" +checksum = "47ce6c96ea0102f01122a185683611bd5ac8d99e62bc59dd12e6bda344ee673d" dependencies = [ "data-encoding", "data-encoding-macro-internal", @@ -713,12 +782,53 @@ dependencies = [ [[package]] name = "data-encoding-macro-internal" -version = "0.1.11" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fdf3fce3ce863539ec1d7fd1b6dcc3c645663376b43ed376bbf887733e4f772" +checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn", +] + +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -735,23 +845,33 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] name = "dlv-list" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d529fd73d344663edfd598ccb3f344e46034db51ebd103518eae34338248ad73" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" dependencies = [ "const-random", ] [[package]] name = "dns-lookup" -version = "2.0.2" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f332aa79f9e9de741ac013237294ef42ce2e9c6394dc7d766725812f1238812" +checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc" dependencies = [ "cfg-if", "libc", @@ -759,62 +879,71 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "document-features" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +dependencies = [ + "litrs", +] + +[[package]] +name = "dtor" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cbdf2ad6846025e8e25df05171abfb30e3ababa12ee0a0e44b9bbe570633a8" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7454e41ff9012c00d53cf7f475c5e3afa3b91b7c90568495495e8d9bf47a1055" + [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" -version = "1.8.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] -name = "env_logger" -version = "0.8.4" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" -dependencies = [ - "log", - "regex", -] +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] name = "exacl" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfeb22a59deb24c3262c43ffcafd1eb807180f371f9fcc99098d181b5d639be" +checksum = "22be12de19decddab85d09f251ec8363f060ccb22ec9c81bc157c0c8433946d8" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "log", "scopeguard", "uuid", @@ -822,9 +951,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file_diff" @@ -832,164 +961,166 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31a7a908b8f32538a2143e59a6e4e2508988832d5d4d6f7c156b3cbc762643a5" +[[package]] +name = "filedescriptor" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" +dependencies = [ + "libc", + "thiserror 1.0.69", + "winapi", +] + [[package]] name = "filetime" -version = "0.2.22" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", - "redox_syscall", - "windows-sys 0.48.0", + "libredox", + "windows-sys 0.59.0", ] [[package]] name = "flate2" -version = "1.0.24" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] [[package]] -name = "fnv" -version = "1.0.7" +name = "fluent" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "8137a6d5a2c50d6b0ebfcb9aaa91a28154e0a70605f112d30cb0cd4a78670477" +dependencies = [ + "fluent-bundle", + "unic-langid", +] [[package]] -name = "fs_extra" -version = "1.3.0" +name = "fluent-bundle" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +checksum = "01203cb8918f5711e73891b347816d932046f95f54207710bda99beaeb423bf4" +dependencies = [ + "fluent-langneg", + "fluent-syntax", + "intl-memoizer", + "intl_pluralrules", + "rustc-hash", + "self_cell", + "smallvec", + "unic-langid", +] [[package]] -name = "fsevent-sys" -version = "4.1.0" +name = "fluent-langneg" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +checksum = "2c4ad0989667548f06ccd0e306ed56b61bd4d35458d54df5ec7587c0e8ed5e94" dependencies = [ - "libc", + "unic-langid", ] [[package]] -name = "fts-sys" -version = "0.2.4" +name = "fluent-syntax" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a66c0a21e344f20c87b4ca12643cf4f40a7018f132c98d344e989b959f49dd1" +checksum = "54f0d287c53ffd184d04d8677f590f4ac5379785529e5e08b1c8083acdd5c198" dependencies = [ - "bindgen", - "libc", + "memchr", + "thiserror 2.0.12", ] [[package]] -name = "fundu" -version = "2.0.0" +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c04cb831a8dccadfe3774b07cba4574a1ec24974d761510e65d8a543c2d7cb4" -dependencies = [ - "fundu-core", -] +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "fundu-core" -version = "0.3.0" +name = "foldhash" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a889e633afd839fb5b04fe53adfd588cefe518e71ec8d3c929698c6daf2acd" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" [[package]] -name = "futures" -version = "0.3.28" +name = "fs_extra" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] -name = "futures-channel" -version = "0.3.28" +name = "fsevent-sys" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" dependencies = [ - "futures-core", - "futures-sink", + "libc", ] [[package]] -name = "futures-core" -version = "0.3.28" +name = "fts-sys" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "43119ec0f2227f8505c8bb6c60606b5eefc328607bfe1a421e561c4decfa02ab" +dependencies = [ + "bindgen", + "libc", +] [[package]] -name = "futures-executor" -version = "0.3.28" +name = "funty" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] -name = "futures-io" -version = "0.3.28" +name = "futures-core" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn", ] -[[package]] -name = "futures-sink" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" - [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "futures-channel", "futures-core", - "futures-io", "futures-macro", - "futures-sink", "futures-task", - "memchr", "pin-project-lite", "pin-utils", "slab", @@ -1003,9 +1134,9 @@ checksum = "1d758ba1b47b00caf47f24925c0074ecb20d6dfcffe7f6d53395c0465674841a" [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -1013,41 +1144,59 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] -name = "glob" +name = "getrandom" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "half" -version = "2.2.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ + "cfg-if", "crunchy", ] [[package]] name = "hashbrown" -version = "0.13.2" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] -name = "hermit-abi" -version = "0.3.2" +name = "hashbrown" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "hex" @@ -1057,33 +1206,34 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hex-literal" -version = "0.4.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" +checksum = "bcaaec4551594c969335c98c903c1397853d4198408ea609190f420500f6be71" [[package]] name = "hostname" -version = "0.3.1" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" dependencies = [ + "cfg-if", "libc", - "match_cfg", - "winapi", + "windows-link", ] [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "winapi", + "windows-core", ] [[package]] @@ -1095,25 +1245,36 @@ dependencies = [ "cc", ] +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown 0.15.2", +] + [[package]] name = "indicatif" -version = "0.17.3" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width", + "unicode-width 0.2.0", + "web-time", ] [[package]] name = "inotify" -version = "0.9.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "inotify-sys", "libc", ] @@ -1128,65 +1289,78 @@ dependencies = [ ] [[package]] -name = "io-lifetimes" -version = "1.0.11" +name = "intl-memoizer" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +checksum = "310da2e345f5eb861e7a07ee182262e94975051db9e4223e909ba90f392f163f" dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", + "type-map", + "unic-langid", ] [[package]] -name = "is-terminal" -version = "0.4.9" +name = "intl_pluralrules" +version = "7.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" dependencies = [ - "hermit-abi", - "rustix 0.38.8", - "windows-sys 0.48.0", + "unic-langid", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] name = "keccak" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ "cpufeatures", ] [[package]] name = "kqueue" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c8fc60ba15bf51257aa9807a48a61013db043fcf3a78cb0d916e8e396dcad98" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" dependencies = [ "kqueue-sys", "libc", @@ -1194,9 +1368,9 @@ dependencies = [ [[package]] name = "kqueue-sys" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8367585489f01bc55dd27404dcf56b95e6da061a256a666ab23be9ba96a2e587" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" dependencies = [ "bitflags 1.3.2", "libc", @@ -1204,123 +1378,136 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "winapi", + "windows-targets 0.48.5", ] [[package]] name = "libm" -version = "0.2.7" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] -name = "linux-raw-sys" -version = "0.1.4" +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.9.0", + "libc", + "redox_syscall", +] + +[[package]] +name = "libz-rs-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" -version = "0.4.17" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "cfg-if", + "hashbrown 0.15.2", ] [[package]] name = "lscolors" -version = "0.15.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7015a04103ad78abb77e4b79ed151e767922d1cfde5f62640471c629a2320d" +checksum = "61183da5de8ba09a58e330d55e5ea796539d8443bd00fdeb863eac39724aa4ab" dependencies = [ + "aho-corasick", "nu-ansi-term", ] -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.6.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5486aed0026218e61b8a01d5fbd5a0a134649abb71a0e53b7bc088529dced86e" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.7.0" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180d4b35be83d33392d1d1bfbd2ae1eca7ff5de1a94d3fc87faaa99a069e7cbd" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" -dependencies = [ - "autocfg", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1329,35 +1516,35 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "0.8.6" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", "log", - "wasi", - "windows-sys 0.45.0", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", ] [[package]] name = "nix" -version = "0.26.2" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "cfg-if", + "cfg_aliases", "libc", - "static_assertions", ] [[package]] @@ -1370,78 +1557,116 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "notify" -version = "6.0.1" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5738a2795d57ea20abec2d6d76c6081186709c0024187cd5977265eda6598b51" +checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" dependencies = [ - "bitflags 1.3.2", - "crossbeam-channel", + "bitflags 2.9.0", "filetime", "fsevent-sys", "inotify", "kqueue", "libc", + "log", "mio", + "notify-types", "walkdir", - "windows-sys 0.45.0", + "windows-sys 0.59.0", ] +[[package]] +name = "notify-types" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" + [[package]] name = "nu-ansi-term" -version = "0.49.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c073d3c1930d0751774acf49e66653acecb416c3a54c6ec095a9b11caddb5a68" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", + "rand 0.8.5", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] -name = "num-traits" -version = "0.2.16" +name = "num-modular" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "64a5fe11d4135c3bcdf3a95b18b194afa9608a5f6ff034f5d857bc9a27fb0119" dependencies = [ - "autocfg", + "num-bigint", + "num-integer", + "num-traits", ] [[package]] -name = "num_cpus" -version = "1.16.0" +name = "num-prime" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "e238432a7881ec7164503ccc516c014bf009be7984cde1ba56837862543bdec3" dependencies = [ - "hermit-abi", - "libc", + "bitvec", + "either", + "lru", + "num-bigint", + "num-integer", + "num-modular", + "num-traits", + "rand 0.8.5", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", ] [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] @@ -1454,17 +1679,17 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.18.0" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "onig" -version = "6.4.0" +version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "libc", "once_cell", "onig_sys", @@ -1472,9 +1697,9 @@ dependencies = [ [[package]] name = "onig_sys" -version = "69.8.1" +version = "69.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" dependencies = [ "cc", "pkg-config", @@ -1482,28 +1707,28 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.6.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" dependencies = [ "dlv-list", - "hashbrown", + "hashbrown 0.14.5", ] [[package]] name = "os_display" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6229bad892b46b0dcfaaeb18ad0d2e56400f5aaea05b768bde96e73676cf75" +checksum = "ad5fd71b79026fb918650dde6d125000a233764f1c2f1659a1c71118e33ea08f" dependencies = [ - "unicode-width", + "unicode-width 0.2.0", ] [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1511,47 +1736,51 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.0", + "windows-targets 0.52.6", ] [[package]] -name = "parse_datetime" -version = "0.4.0" +name = "parse-zoneinfo" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecceaede7767a9a98058687a321bc91742eff7670167a34104afb30fc8757df" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" dependencies = [ - "chrono", "regex", ] [[package]] -name = "peeking_take_while" -version = "0.1.2" +name = "parse_datetime" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +checksum = "2fd3830b49ee3a0dcc8fdfadc68c6354c97d00101ac1cac5b2eee25d35c42066" +dependencies = [ + "chrono", + "nom 8.0.0", + "regex", +] [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", "phf_shared", @@ -1559,28 +1788,28 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -1590,15 +1819,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "platform-info" -version = "2.0.2" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6259c4860e53bf665016f1b2f46a8859cadfa717581dc9d597ae4069de6300f" +checksum = "7539aeb3fdd8cb4f6a331307cf71a1039cee75e94e8a71725b9484f4a0d9451a" dependencies = [ "libc", "winapi", @@ -1606,80 +1835,106 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "0.3.15" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15eb2c6e362923af47e13c23ca5afb859e83d54452c55b0b9ac763b8f7c1ac16" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", ] [[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" +name = "prettyplease" +version = "0.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" +checksum = "f1ccf34da56fc294e7d4ccf69a85992b7dfb826b7cf57bac6a70bba3494cc08a" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] [[package]] name = "procfs" -version = "0.15.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943ca7f9f29bab5844ecd8fdb3992c5969b6622bb9609b9502fef9b4310e3f1f" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 1.3.2", - "byteorder", + "bitflags 2.9.0", "hex", - "lazy_static", - "rustix 0.36.15", + "procfs-core", + "rustix 0.38.44", ] [[package]] -name = "quick-error" -version = "2.0.1" +name = "procfs-core" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" - -[[package]] -name = "quickcheck" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "env_logger", - "log", - "rand", + "bitflags 2.9.0", + "hex", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" -version = "1.0.29" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.5" @@ -1687,8 +1942,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -1698,7 +1963,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -1707,23 +1982,23 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] -name = "rand_pcg" -version = "0.3.1" +name = "rand_core" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "rand_core", + "getrandom 0.3.1", ] [[package]] name = "rayon" -version = "1.7.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -1731,36 +2006,28 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", ] -[[package]] -name = "reference-counted-singleton" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bfbf25d7eb88ddcbb1ec3d755d0634da8f7657b2cb8b74089121409ab8228f" - [[package]] name = "regex" -version = "1.9.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -1770,9 +2037,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1781,125 +2048,118 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "relative-path" -version = "1.8.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf2521270932c3c7bed1a59151222bd7643c79310f2916f01925e1e16255698" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "rlimit" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3560f70f30a0f16d11d01ed078a07740fe6b489667abc7c7b029155d9f21c3d8" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" dependencies = [ "libc", ] [[package]] name = "roff" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" +checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3" [[package]] name = "rstest" -version = "0.18.2" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" +checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" dependencies = [ - "futures", "futures-timer", + "futures-util", "rstest_macros", "rustc_version", ] [[package]] name = "rstest_macros" -version = "0.18.2" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" +checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" dependencies = [ "cfg-if", "glob", + "proc-macro-crate", "proc-macro2", "quote", "regex", "relative-path", "rustc_version", - "syn 2.0.23", + "syn", "unicode-ident", ] [[package]] name = "rust-ini" -version = "0.19.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" +checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" dependencies = [ "cfg-if", "ordered-multimap", + "trim-in-place", ] [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.36.15" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c37f1bd5ef1b5422177b7646cba67430579cfe2ace80f284fee876bca52ad941" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "errno", - "io-lifetimes", "libc", - "linux-raw-sys 0.1.4", - "windows-sys 0.45.0", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", ] [[package]] name = "rustix" -version = "0.37.23" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "dade4812df5c384711475be5fcd8c162555352945401aed22a35bffeab61f657" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "errno", - "io-lifetimes", "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", + "linux-raw-sys 0.9.4", + "windows-sys 0.52.0", ] [[package]] -name = "rustix" -version = "0.38.8" +name = "rustversion" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" -dependencies = [ - "bitflags 2.3.3", - "errno", - "libc", - "linux-raw-sys 0.4.5", - "windows-sys 0.48.0", -] +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] name = "same-file" @@ -1912,35 +2172,35 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "self_cell" -version = "1.0.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c309e515543e67811222dbc9e3dd7e1056279b782e1dacffe4242b718734fb6" +checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749" [[package]] name = "selinux" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00576725d21b588213fbd4af84cd7e4cc4304e8e9bd6c0f5a1498a3e2ca6a51" +checksum = "e37f432dfe840521abd9a72fefdf88ed7ad0f43bbea7d9d1d3d80383e9f4ad13" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "libc", "once_cell", - "reference-counted-singleton", + "parking_lot", "selinux-sys", - "thiserror", + "thiserror 2.0.12", ] [[package]] name = "selinux-sys" -version = "0.6.2" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806d381649bb85347189d2350728817418138d11d738e2482cb644ec7f3c755d" +checksum = "280da3df1236da180be5ac50a893b26a1d3c49e3a44acb2d10d1f082523ff916" dependencies = [ "bindgen", "cc", @@ -1950,21 +2210,44 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.14" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "serde" -version = "1.0.147" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -1973,9 +2256,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -1994,15 +2277,15 @@ dependencies = [ [[package]] name = "shlex" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" dependencies = [ "libc", "signal-hook-registry", @@ -2010,9 +2293,9 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" dependencies = [ "libc", "mio", @@ -2021,24 +2304,30 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "siphasher" -version = "0.3.10" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -2054,49 +2343,37 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "smawk" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "subtle" -version = "2.4.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "e02e925281e18ffd9d640e234264753c43edc62d64b2d4cf898f1bc5e75f3fc2" dependencies = [ "proc-macro2", "quote", @@ -2104,89 +2381,98 @@ dependencies = [ ] [[package]] -name = "syn" -version = "2.0.23" +name = "tap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ - "cfg-if", "fastrand", - "redox_syscall", - "rustix 0.38.8", - "windows-sys 0.48.0", -] - -[[package]] -name = "term_grid" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230d3e804faaed5a39b08319efb797783df2fd9671b39b7596490cb486d702cf" -dependencies = [ - "unicode-width", + "getrandom 0.3.1", + "once_cell", + "rustix 1.0.1", + "windows-sys 0.52.0", ] [[package]] name = "terminal_size" -version = "0.2.6" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237" +checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" dependencies = [ - "rustix 0.37.23", - "windows-sys 0.48.0", + "rustix 1.0.1", + "windows-sys 0.59.0", ] [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" dependencies = [ "smawk", "terminal_size", "unicode-linebreak", - "unicode-width", + "unicode-width 0.2.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", ] [[package]] name = "thiserror" -version = "1.0.37" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ - "thiserror-impl", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "time" -version = "0.3.20" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ + "deranged", "itoa", "libc", + "num-conv", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -2194,16 +2480,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ + "num-conv", "time-core", ] @@ -2216,17 +2503,77 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "toml_datetime" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" + +[[package]] +name = "toml_edit" +version = "0.22.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "trim-in-place" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" + +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash", +] + [[package]] name = "typenum" -version = "1.15.0" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "tinystr", +] [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-linebreak" @@ -2236,37 +2583,73 @@ checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unindent" -version = "0.2.1" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "unty" +version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa30f5ea51ff7edfc797c6d3f9ec8cbd8cfedef5371766b7181d33977f4814f" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "utmp-classic" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e24c654e19afaa6b8f3877ece5d3bed849c2719c56f6752b18ca7da4fcc6e85a" +dependencies = [ + "cfg-if", + "libc", + "thiserror 1.0.69", + "time", + "utmp-classic-raw", + "zerocopy", +] + +[[package]] +name = "utmp-classic-raw" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "22c226537a3d6e01c440c1926ca0256dbee2d19b2229ede6fc4863a6493dd831" +dependencies = [ + "cfg-if", + "zerocopy", +] [[package]] name = "uu_arch" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "platform-info", @@ -2275,7 +2658,7 @@ dependencies = [ [[package]] name = "uu_base32" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2283,15 +2666,16 @@ dependencies = [ [[package]] name = "uu_base64" -version = "0.0.21" +version = "0.1.0" dependencies = [ + "clap", "uu_base32", "uucore", ] [[package]] name = "uu_basename" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2299,7 +2683,7 @@ dependencies = [ [[package]] name = "uu_basenc" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uu_base32", @@ -2308,30 +2692,30 @@ dependencies = [ [[package]] name = "uu_cat" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "is-terminal", + "memchr", "nix", - "thiserror", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_chcon" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "fts-sys", "libc", "selinux", - "thiserror", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_chgrp" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2339,7 +2723,7 @@ dependencies = [ [[package]] name = "uu_chmod" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -2348,7 +2732,7 @@ dependencies = [ [[package]] name = "uu_chown" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2356,24 +2740,26 @@ dependencies = [ [[package]] name = "uu_chroot" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_cksum" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "hex", + "regex", "uucore", ] [[package]] name = "uu_comm" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2381,13 +2767,14 @@ dependencies = [ [[package]] name = "uu_cp" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "exacl", "filetime", "indicatif", "libc", + "linux-raw-sys 0.9.4", "quick-error", "selinux", "uucore", @@ -2397,62 +2784,63 @@ dependencies = [ [[package]] name = "uu_csplit" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "regex", - "thiserror", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_cut" -version = "0.0.21" +version = "0.1.0" dependencies = [ "bstr", "clap", - "is-terminal", "memchr", "uucore", ] [[package]] name = "uu_date" -version = "0.0.21" +version = "0.1.0" dependencies = [ "chrono", "clap", "libc", "parse_datetime", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_dd" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "gcd", "libc", "nix", "signal-hook", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_df" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "tempfile", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_dir" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uu_ls", @@ -2461,7 +2849,7 @@ dependencies = [ [[package]] name = "uu_dircolors" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2469,7 +2857,7 @@ dependencies = [ [[package]] name = "uu_dirname" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2477,18 +2865,19 @@ dependencies = [ [[package]] name = "uu_du" -version = "0.0.21" +version = "0.1.0" dependencies = [ "chrono", "clap", "glob", + "thiserror 2.0.12", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_echo" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2496,50 +2885,53 @@ dependencies = [ [[package]] name = "uu_env" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "nix", "rust-ini", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_expand" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_expr" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "num-bigint", "num-traits", "onig", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_factor" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "coz", + "num-bigint", + "num-prime", "num-traits", - "quickcheck", - "rand", + "rand 0.9.1", "smallvec", "uucore", ] [[package]] name = "uu_false" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2547,16 +2939,16 @@ dependencies = [ [[package]] name = "uu_fmt" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "unicode-width", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_fold" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2564,15 +2956,16 @@ dependencies = [ [[package]] name = "uu_groups" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_hashsum" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "hex", @@ -2583,16 +2976,17 @@ dependencies = [ [[package]] name = "uu_head" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "memchr", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_hostid" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -2601,17 +2995,18 @@ dependencies = [ [[package]] name = "uu_hostname" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", + "dns-lookup", "hostname", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_id" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "selinux", @@ -2620,27 +3015,29 @@ dependencies = [ [[package]] name = "uu_install" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "file_diff", "filetime", "libc", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_join" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "memchr", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_kill" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "nix", @@ -2649,7 +3046,7 @@ dependencies = [ [[package]] name = "uu_link" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2657,15 +3054,16 @@ dependencies = [ [[package]] name = "uu_ln" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_logname" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -2674,25 +3072,25 @@ dependencies = [ [[package]] name = "uu_ls" -version = "0.0.21" +version = "0.1.0" dependencies = [ + "ansi-width", "chrono", "clap", "glob", - "is-terminal", + "hostname", "lscolors", "number_prefix", - "once_cell", "selinux", - "term_grid", "terminal_size", - "unicode-width", + "thiserror 2.0.12", "uucore", + "uutils_term_grid", ] [[package]] name = "uu_mkdir" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2700,7 +3098,7 @@ dependencies = [ [[package]] name = "uu_mkfifo" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -2709,7 +3107,7 @@ dependencies = [ [[package]] name = "uu_mknod" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -2718,40 +3116,44 @@ dependencies = [ [[package]] name = "uu_mktemp" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "rand", + "rand 0.9.1", "tempfile", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_more" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "crossterm", - "is-terminal", "nix", + "tempfile", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_mv" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "fs_extra", "indicatif", + "libc", + "thiserror 2.0.12", "uucore", + "windows-sys 0.59.0", ] [[package]] name = "uu_nice" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -2761,7 +3163,7 @@ dependencies = [ [[package]] name = "uu_nl" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "regex", @@ -2770,17 +3172,17 @@ dependencies = [ [[package]] name = "uu_nohup" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "is-terminal", "libc", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_nproc" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -2789,15 +3191,16 @@ dependencies = [ [[package]] name = "uu_numfmt" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_od" -version = "0.0.21" +version = "0.1.0" dependencies = [ "byteorder", "clap", @@ -2807,7 +3210,7 @@ dependencies = [ [[package]] name = "uu_paste" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2815,7 +3218,7 @@ dependencies = [ [[package]] name = "uu_pathchk" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -2824,7 +3227,7 @@ dependencies = [ [[package]] name = "uu_pinky" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2832,19 +3235,19 @@ dependencies = [ [[package]] name = "uu_pr" -version = "0.0.21" +version = "0.1.0" dependencies = [ "chrono", "clap", - "itertools", - "quick-error", + "itertools 0.14.0", "regex", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_printenv" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2852,7 +3255,7 @@ dependencies = [ [[package]] name = "uu_printf" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2860,16 +3263,17 @@ dependencies = [ [[package]] name = "uu_ptx" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "regex", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_pwd" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2877,7 +3281,7 @@ dependencies = [ [[package]] name = "uu_readlink" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2885,15 +3289,7 @@ dependencies = [ [[package]] name = "uu_realpath" -version = "0.0.21" -dependencies = [ - "clap", - "uucore", -] - -[[package]] -name = "uu_relpath" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -2901,18 +3297,17 @@ dependencies = [ [[package]] name = "uu_rm" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", "uucore", - "walkdir", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_rmdir" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -2921,95 +3316,98 @@ dependencies = [ [[package]] name = "uu_runcon" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", "selinux", - "thiserror", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_seq" -version = "0.0.21" +version = "0.1.0" dependencies = [ "bigdecimal", "clap", "num-bigint", "num-traits", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_shred" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", - "rand", + "rand 0.9.1", "uucore", ] [[package]] name = "uu_shuf" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "memchr", - "rand", - "rand_core", + "rand 0.9.1", + "rand_core 0.9.3", "uucore", ] [[package]] name = "uu_sleep" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "fundu", "uucore", ] [[package]] name = "uu_sort" -version = "0.0.21" +version = "0.1.0" dependencies = [ "binary-heap-plus", "clap", "compare", "ctrlc", "fnv", - "itertools", + "itertools 0.14.0", "memchr", - "rand", + "nix", + "rand 0.9.1", "rayon", "self_cell", "tempfile", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_split" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "memchr", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_stat" -version = "0.0.21" +version = "0.1.0" dependencies = [ + "chrono", "clap", "uucore", ] [[package]] name = "uu_stdbuf" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "tempfile", @@ -3019,17 +3417,16 @@ dependencies = [ [[package]] name = "uu_stdbuf_libstdbuf" -version = "0.0.21" +version = "0.1.0" dependencies = [ "cpp", "cpp_build", "libc", - "uucore", ] [[package]] name = "uu_stty" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "nix", @@ -3038,7 +3435,7 @@ dependencies = [ [[package]] name = "uu_sum" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -3046,33 +3443,32 @@ dependencies = [ [[package]] name = "uu_sync" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", "nix", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_tac" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "memchr", "memmap2", "regex", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_tail" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "fundu", - "is-terminal", "libc", "memchr", "notify", @@ -3080,31 +3476,30 @@ dependencies = [ "same-file", "uucore", "winapi-util", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_tee" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "libc", + "nix", "uucore", ] [[package]] name = "uu_test" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", - "redox_syscall", "uucore", ] [[package]] name = "uu_timeout" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "libc", @@ -3114,28 +3509,29 @@ dependencies = [ [[package]] name = "uu_touch" -version = "0.0.21" +version = "0.1.0" dependencies = [ "chrono", "clap", "filetime", "parse_datetime", + "thiserror 2.0.12", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_tr" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "nom", + "nom 8.0.0", "uucore", ] [[package]] name = "uu_true" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -3143,7 +3539,7 @@ dependencies = [ [[package]] name = "uu_truncate" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -3151,25 +3547,25 @@ dependencies = [ [[package]] name = "uu_tsort" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", + "thiserror 2.0.12", "uucore", ] [[package]] name = "uu_tty" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "is-terminal", "nix", "uucore", ] [[package]] name = "uu_uname" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "platform-info", @@ -3178,16 +3574,17 @@ dependencies = [ [[package]] name = "uu_unexpand" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_uniq" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -3195,7 +3592,7 @@ dependencies = [ [[package]] name = "uu_unlink" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -3203,24 +3600,27 @@ dependencies = [ [[package]] name = "uu_uptime" -version = "0.0.21" +version = "0.1.0" dependencies = [ "chrono", "clap", + "thiserror 2.0.12", + "utmp-classic", "uucore", ] [[package]] name = "uu_users" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", + "utmp-classic", "uucore", ] [[package]] name = "uu_vdir" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uu_ls", @@ -3229,20 +3629,20 @@ dependencies = [ [[package]] name = "uu_wc" -version = "0.0.21" +version = "0.1.0" dependencies = [ "bytecount", "clap", "libc", "nix", - "thiserror", - "unicode-width", + "thiserror 2.0.12", + "unicode-width 0.2.0", "uucore", ] [[package]] name = "uu_who" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", "uucore", @@ -3250,63 +3650,75 @@ dependencies = [ [[package]] name = "uu_whoami" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "libc", "uucore", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "uu_yes" -version = "0.0.21" +version = "0.1.0" dependencies = [ "clap", - "itertools", + "itertools 0.14.0", "nix", "uucore", ] [[package]] name = "uucore" -version = "0.0.21" +version = "0.1.0" dependencies = [ + "bigdecimal", "blake2b_simd", "blake3", + "chrono", + "chrono-tz", "clap", + "crc32fast", "data-encoding", "data-encoding-macro", "digest", "dns-lookup", "dunce", + "fluent", + "fluent-bundle", "glob", "hex", - "itertools", + "iana-time-zone", + "itertools 0.14.0", "libc", "md-5", "memchr", "nix", - "once_cell", + "num-traits", + "number_prefix", "os_display", + "regex", + "selinux", "sha1", "sha2", "sha3", "sm3", "tempfile", - "thiserror", + "thiserror 2.0.12", "time", + "unic-langid", + "utmp-classic", "uucore_procs", "walkdir", "wild", "winapi-util", - "windows-sys 0.48.0", + "windows-sys 0.59.0", + "xattr", "z85", ] [[package]] name = "uucore_procs" -version = "0.0.21" +version = "0.1.0" dependencies = [ "proc-macro2", "quote", @@ -3315,28 +3727,60 @@ dependencies = [ [[package]] name = "uuhelp_parser" -version = "0.0.21" +version = "0.1.0" [[package]] name = "uuid" -version = "1.2.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" + +[[package]] +name = "uutests" +version = "0.1.0" +dependencies = [ + "ctor", + "glob", + "libc", + "nix", + "pretty_assertions", + "rand 0.9.1", + "regex", + "rlimit", + "tempfile", + "time", + "uucore", + "xattr", +] + +[[package]] +name = "uutils_term_grid" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcba141ce511bad08e80b43f02976571072e1ff4286f7d628943efbd277c6361" +dependencies = [ + "ansi-width", +] [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "virtue" +version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -3346,36 +3790,46 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.23", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3383,39 +3837,41 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] -name = "which" -version = "4.3.0" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ - "either", - "libc", - "once_cell", + "js-sys", + "wasm-bindgen", ] [[package]] name = "wild" -version = "2.1.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b116685a6be0c52f5a103334cbff26db643826c7b3735fc0a3ba9871310a74" +checksum = "a3131afc8c575281e1e80f36ed6a092aa502c08b18ed7524e86fbbb12bb410e1" dependencies = [ "glob", ] @@ -3438,11 +3894,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -3452,12 +3908,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.45.0" +name = "windows-core" +version = "0.60.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-result" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" dependencies = [ - "windows-targets 0.42.2", + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", ] [[package]] @@ -3466,152 +3972,263 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.9.0", +] + +[[package]] +name = "wyz" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] [[package]] name = "xattr" -version = "1.0.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" +checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" dependencies = [ "libc", + "rustix 1.0.1", ] [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "z85" -version = "3.0.5" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a599daf1b507819c1121f0bf87fa37eb19daac6aff3aefefd4e6e2e0f2020fc" +checksum = "9b3a41ce106832b4da1c065baa4c31cf640cf965fa1483816402b7f6b96f0a64" [[package]] -name = "zip" -version = "0.6.6" +name = "zerocopy" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "zerofrom", +] + +[[package]] +name = "zip" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "153a6fff49d264c4babdcfa6b4d534747f520e56e8f0f384f3b808c4b64cc1fd" +dependencies = [ + "arbitrary", "crc32fast", - "crossbeam-utils", "flate2", + "indexmap", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", ] diff --git a/Cargo.toml b/Cargo.toml index 320aa0f4307..4a61b5f0321 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,25 +1,26 @@ # coreutils (uutils) # * see the repository LICENSE, README, and CONTRIBUTING files for more information -# spell-checker:ignore (libs) bigdecimal datetime fundu gethostid kqueue libselinux mangen memmap procfs uuhelp +# spell-checker:ignore (libs) bigdecimal datetime serde bincode gethostid kqueue libselinux mangen memmap procfs uuhelp startswith constness expl [package] name = "coreutils" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "coreutils ~ GNU coreutils (updated); implemented as universal (cross-platform) utils, written in Rust" default-run = "coreutils" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils" readme = "README.md" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -rust-version = "1.64.0" -edition = "2021" - +rust-version = "1.85.0" build = "build.rs" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true + +[package.metadata.docs.rs] +all-features = true [features] default = ["feat_common_core"] @@ -30,6 +31,9 @@ windows = ["feat_os_windows"] ## project-specific feature shortcodes nightly = [] test_unimplemented = [] +expensive_tests = [] +# "test_risky_names" == enable tests that create problematic file names (would make a network share inaccessible to Windows, breaks SVN on Mac OS, etc.) +test_risky_names = [] # * only build `uudoc` when `--feature uudoc` is activated uudoc = ["zip", "dep:uuhelp_parser"] ## features @@ -45,7 +49,12 @@ feat_acl = ["cp/feat_acl"] feat_selinux = [ "cp/selinux", "id/selinux", + "install/selinux", "ls/selinux", + "mkdir/selinux", + "mkfifo/selinux", + "mknod/selinux", + "stat/selinux", "selinux", "feat_require_selinux", ] @@ -100,7 +109,6 @@ feat_common_core = [ "pwd", "readlink", "realpath", - "relpath", "rm", "rmdir", "seq", @@ -145,7 +153,8 @@ feat_os_macos = [ # "feat_require_unix_hostid", ] -# "feat_os_unix" == set of utilities which can be built/run on modern/usual *nix platforms +# "feat_os_unix" == set of utilities which can be built/run on modern/usual *nix platforms. +# Also used for targets binding to the "musl" library (ref: ) feat_os_unix = [ "feat_Tier1", # @@ -167,13 +176,6 @@ feat_os_unix_gnueabihf = [ "feat_require_unix_hostid", "feat_require_unix_utmpx", ] -# "feat_os_unix_musl" == set of utilities which can be built/run on targets binding to the "musl" library (ref: ) -feat_os_unix_musl = [ - "feat_Tier1", - # - "feat_require_unix", - "feat_require_unix_hostid", -] feat_os_unix_android = [ "feat_Tier1", # @@ -241,6 +243,7 @@ feat_os_unix_redox = [ "feat_common_core", # "chmod", + "stat", "uname", ] # "feat_os_windows_legacy" == slightly restricted set of utilities which can be built/run on early windows platforms (eg, "WinXP") @@ -257,101 +260,120 @@ feat_os_windows_legacy = [ # * bypass/override ~ translate 'test' feature name to avoid dependency collision with rust core 'test' crate (o/w surfaces as compiler errors during testing) test = ["uu_test"] +[workspace.package] +authors = ["uutils developers"] +categories = ["command-line-utilities"] +edition = "2024" +homepage = "https://github.com/uutils/coreutils" +keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] +license = "MIT" +readme = "README.package.md" +version = "0.1.0" + [workspace.dependencies] +ansi-width = "0.1.0" bigdecimal = "0.4" binary-heap-plus = "0.5.0" -bstr = "1.6" -bytecount = "0.6.3" -byteorder = "1.4.3" -chrono = { version = "^0.4.28", default-features = false, features = [ +bstr = "1.9.1" +bytecount = "0.6.8" +byteorder = "1.5.0" +chrono = { version = "0.4.41", default-features = false, features = [ "std", "alloc", "clock", ] } -clap = { version = "4.3", features = ["wrap_help", "cargo"] } -clap_complete = "4.3" +chrono-tz = "0.10.0" +clap = { version = "4.5", features = ["wrap_help", "cargo"] } +clap_complete = "4.4" clap_mangen = "0.2" compare = "0.1.0" -coz = { version = "0.1.3" } -crossterm = ">=0.27.0" -ctrlc = { version = "3.4", features = ["termination"] } -exacl = "0.10.0" +crossterm = "0.29.0" +ctrlc = { version = "3.4.7", features = ["termination"] } +dns-lookup = { version = "2.0.4" } +exacl = "0.12.0" file_diff = "1.0.0" -filetime = "0.2" +filetime = "0.2.23" fnv = "1.0.7" fs_extra = "1.3.0" -fts-sys = "0.2" -fundu = "2.0.0" +fts-sys = "0.2.16" gcd = "2.3" glob = "0.3.1" -half = "2.2" -indicatif = "0.17" -is-terminal = "0.4.9" -itertools = "0.11.0" -libc = "0.2.147" -lscolors = { version = "0.15.0", default-features = false, features = [ - "nu-ansi-term", +half = "2.4.1" +hostname = "0.4" +iana-time-zone = "0.1.57" +indicatif = "0.17.8" +itertools = "0.14.0" +libc = "0.2.172" +linux-raw-sys = "0.9" +lscolors = { version = "0.20.0", default-features = false, features = [ + "gnu_legacy", ] } -memchr = "2" -memmap2 = "0.7" -nix = { version = "0.26", default-features = false } -nom = "7.1.3" -notify = { version = "=6.0.1", features = ["macos_kqueue"] } +memchr = "2.7.2" +memmap2 = "0.9.4" +nix = { version = "0.30", default-features = false } +nom = "8.0.0" +notify = { version = "=8.0.0", features = ["macos_kqueue"] } num-bigint = "0.4.4" -num-traits = "0.2.16" +num-prime = "0.4.4" +num-traits = "0.2.19" number_prefix = "0.4" -once_cell = "1.18.0" -onig = { version = "~6.4", default-features = false } -parse_datetime = "0.4.0" +onig = { version = "~6.5.1", default-features = false } +parse_datetime = "0.9.0" phf = "0.11.2" phf_codegen = "0.11.2" -platform-info = "2.0.2" +platform-info = "2.0.3" quick-error = "2.0.1" -rand = { version = "0.8", features = ["small_rng"] } -rand_core = "0.6" -rayon = "1.7" -redox_syscall = "0.3" -regex = "1.9.5" -rstest = "0.18.2" -rust-ini = "0.19.0" +rand = { version = "0.9.0", features = ["small_rng"] } +rand_core = "0.9.0" +rayon = "1.10" +regex = "1.10.4" +rstest = "0.25.0" +rust-ini = "0.21.0" same-file = "1.0.6" -self_cell = "1.0.1" -selinux = "0.4" +self_cell = "1.0.4" +selinux = "0.5.1" +selinux-sys = "0.6.14" signal-hook = "0.3.17" -smallvec = { version = "1.11", features = ["union"] } -tempfile = "3.8.0" -term_grid = "0.1.5" -terminal_size = "0.2.6" -textwrap = { version = "0.16.0", features = ["terminal_size"] } -thiserror = "1.0" -time = { version = "0.3" } -unicode-segmentation = "1.10.1" -unicode-width = "0.1.10" -utf-8 = "0.7.6" -walkdir = "2.3" -winapi-util = "0.1.5" -windows-sys = { version = "0.48.0", default-features = false } -xattr = "1.0.1" -zip = { version = "0.6.6", default_features = false, features = ["deflate"] } +smallvec = { version = "1.13.2", features = ["union"] } +tempfile = "3.15.0" +terminal_size = "0.4.0" +textwrap = { version = "0.16.1", features = ["terminal_size"] } +thiserror = "2.0.3" +time = { version = "0.3.36" } +unicode-segmentation = "1.11.0" +unicode-width = "0.2.0" +utmp-classic = "0.1.6" +uutils_term_grid = "0.7" +walkdir = "2.5" +winapi-util = "0.1.8" +windows-sys = { version = "0.59.0", default-features = false } +xattr = "1.3.1" +zip = { version = "4.0.0", default-features = false, features = ["deflate"] } hex = "0.4.3" -md-5 = "0.10.5" -sha1 = "0.10.5" -sha2 = "0.10.7" +md-5 = "0.10.6" +sha1 = "0.10.6" +sha2 = "0.10.8" sha3 = "0.10.8" -blake2b_simd = "1.0.1" -blake3 = "1.4.0" +blake2b_simd = "1.0.2" +blake3 = "1.5.1" sm3 = "0.4.2" +crc32fast = "1.4.2" digest = "0.10.7" -uucore = { version = ">=0.0.19", package = "uucore", path = "src/uucore" } -uucore_procs = { version = ">=0.0.19", package = "uucore_procs", path = "src/uucore_procs" } -uu_ls = { version = ">=0.0.18", path = "src/uu/ls" } -uu_base32 = { version = ">=0.0.18", path = "src/uu/base32" } +# Fluent dependencies +fluent-bundle = "0.16.0" +fluent = "0.17.0" +unic-langid = "0.9.6" + +uucore = { version = "0.1.0", package = "uucore", path = "src/uucore" } +uucore_procs = { version = "0.1.0", package = "uucore_procs", path = "src/uucore_procs" } +uu_ls = { version = "0.1.0", path = "src/uu/ls" } +uu_base32 = { version = "0.1.0", path = "src/uu/base32" } +uutests = { version = "0.1.0", package = "uutests", path = "tests/uutests/" } [dependencies] clap = { workspace = true } -once_cell = { workspace = true } uucore = { workspace = true } clap_complete = { workspace = true } clap_mangen = { workspace = true } @@ -363,110 +385,109 @@ zip = { workspace = true, optional = true } uuhelp_parser = { optional = true, version = ">=0.0.19", path = "src/uuhelp_parser" } # * uutils -uu_test = { optional = true, version = "0.0.21", package = "uu_test", path = "src/uu/test" } +uu_test = { optional = true, version = "0.1.0", package = "uu_test", path = "src/uu/test" } # -arch = { optional = true, version = "0.0.21", package = "uu_arch", path = "src/uu/arch" } -base32 = { optional = true, version = "0.0.21", package = "uu_base32", path = "src/uu/base32" } -base64 = { optional = true, version = "0.0.21", package = "uu_base64", path = "src/uu/base64" } -basename = { optional = true, version = "0.0.21", package = "uu_basename", path = "src/uu/basename" } -basenc = { optional = true, version = "0.0.21", package = "uu_basenc", path = "src/uu/basenc" } -cat = { optional = true, version = "0.0.21", package = "uu_cat", path = "src/uu/cat" } -chcon = { optional = true, version = "0.0.21", package = "uu_chcon", path = "src/uu/chcon" } -chgrp = { optional = true, version = "0.0.21", package = "uu_chgrp", path = "src/uu/chgrp" } -chmod = { optional = true, version = "0.0.21", package = "uu_chmod", path = "src/uu/chmod" } -chown = { optional = true, version = "0.0.21", package = "uu_chown", path = "src/uu/chown" } -chroot = { optional = true, version = "0.0.21", package = "uu_chroot", path = "src/uu/chroot" } -cksum = { optional = true, version = "0.0.21", package = "uu_cksum", path = "src/uu/cksum" } -comm = { optional = true, version = "0.0.21", package = "uu_comm", path = "src/uu/comm" } -cp = { optional = true, version = "0.0.21", package = "uu_cp", path = "src/uu/cp" } -csplit = { optional = true, version = "0.0.21", package = "uu_csplit", path = "src/uu/csplit" } -cut = { optional = true, version = "0.0.21", package = "uu_cut", path = "src/uu/cut" } -date = { optional = true, version = "0.0.21", package = "uu_date", path = "src/uu/date" } -dd = { optional = true, version = "0.0.21", package = "uu_dd", path = "src/uu/dd" } -df = { optional = true, version = "0.0.21", package = "uu_df", path = "src/uu/df" } -dir = { optional = true, version = "0.0.21", package = "uu_dir", path = "src/uu/dir" } -dircolors = { optional = true, version = "0.0.21", package = "uu_dircolors", path = "src/uu/dircolors" } -dirname = { optional = true, version = "0.0.21", package = "uu_dirname", path = "src/uu/dirname" } -du = { optional = true, version = "0.0.21", package = "uu_du", path = "src/uu/du" } -echo = { optional = true, version = "0.0.21", package = "uu_echo", path = "src/uu/echo" } -env = { optional = true, version = "0.0.21", package = "uu_env", path = "src/uu/env" } -expand = { optional = true, version = "0.0.21", package = "uu_expand", path = "src/uu/expand" } -expr = { optional = true, version = "0.0.21", package = "uu_expr", path = "src/uu/expr" } -factor = { optional = true, version = "0.0.21", package = "uu_factor", path = "src/uu/factor" } -false = { optional = true, version = "0.0.21", package = "uu_false", path = "src/uu/false" } -fmt = { optional = true, version = "0.0.21", package = "uu_fmt", path = "src/uu/fmt" } -fold = { optional = true, version = "0.0.21", package = "uu_fold", path = "src/uu/fold" } -groups = { optional = true, version = "0.0.21", package = "uu_groups", path = "src/uu/groups" } -hashsum = { optional = true, version = "0.0.21", package = "uu_hashsum", path = "src/uu/hashsum" } -head = { optional = true, version = "0.0.21", package = "uu_head", path = "src/uu/head" } -hostid = { optional = true, version = "0.0.21", package = "uu_hostid", path = "src/uu/hostid" } -hostname = { optional = true, version = "0.0.21", package = "uu_hostname", path = "src/uu/hostname" } -id = { optional = true, version = "0.0.21", package = "uu_id", path = "src/uu/id" } -install = { optional = true, version = "0.0.21", package = "uu_install", path = "src/uu/install" } -join = { optional = true, version = "0.0.21", package = "uu_join", path = "src/uu/join" } -kill = { optional = true, version = "0.0.21", package = "uu_kill", path = "src/uu/kill" } -link = { optional = true, version = "0.0.21", package = "uu_link", path = "src/uu/link" } -ln = { optional = true, version = "0.0.21", package = "uu_ln", path = "src/uu/ln" } -ls = { optional = true, version = "0.0.21", package = "uu_ls", path = "src/uu/ls" } -logname = { optional = true, version = "0.0.21", package = "uu_logname", path = "src/uu/logname" } -mkdir = { optional = true, version = "0.0.21", package = "uu_mkdir", path = "src/uu/mkdir" } -mkfifo = { optional = true, version = "0.0.21", package = "uu_mkfifo", path = "src/uu/mkfifo" } -mknod = { optional = true, version = "0.0.21", package = "uu_mknod", path = "src/uu/mknod" } -mktemp = { optional = true, version = "0.0.21", package = "uu_mktemp", path = "src/uu/mktemp" } -more = { optional = true, version = "0.0.21", package = "uu_more", path = "src/uu/more" } -mv = { optional = true, version = "0.0.21", package = "uu_mv", path = "src/uu/mv" } -nice = { optional = true, version = "0.0.21", package = "uu_nice", path = "src/uu/nice" } -nl = { optional = true, version = "0.0.21", package = "uu_nl", path = "src/uu/nl" } -nohup = { optional = true, version = "0.0.21", package = "uu_nohup", path = "src/uu/nohup" } -nproc = { optional = true, version = "0.0.21", package = "uu_nproc", path = "src/uu/nproc" } -numfmt = { optional = true, version = "0.0.21", package = "uu_numfmt", path = "src/uu/numfmt" } -od = { optional = true, version = "0.0.21", package = "uu_od", path = "src/uu/od" } -paste = { optional = true, version = "0.0.21", package = "uu_paste", path = "src/uu/paste" } -pathchk = { optional = true, version = "0.0.21", package = "uu_pathchk", path = "src/uu/pathchk" } -pinky = { optional = true, version = "0.0.21", package = "uu_pinky", path = "src/uu/pinky" } -pr = { optional = true, version = "0.0.21", package = "uu_pr", path = "src/uu/pr" } -printenv = { optional = true, version = "0.0.21", package = "uu_printenv", path = "src/uu/printenv" } -printf = { optional = true, version = "0.0.21", package = "uu_printf", path = "src/uu/printf" } -ptx = { optional = true, version = "0.0.21", package = "uu_ptx", path = "src/uu/ptx" } -pwd = { optional = true, version = "0.0.21", package = "uu_pwd", path = "src/uu/pwd" } -readlink = { optional = true, version = "0.0.21", package = "uu_readlink", path = "src/uu/readlink" } -realpath = { optional = true, version = "0.0.21", package = "uu_realpath", path = "src/uu/realpath" } -relpath = { optional = true, version = "0.0.21", package = "uu_relpath", path = "src/uu/relpath" } -rm = { optional = true, version = "0.0.21", package = "uu_rm", path = "src/uu/rm" } -rmdir = { optional = true, version = "0.0.21", package = "uu_rmdir", path = "src/uu/rmdir" } -runcon = { optional = true, version = "0.0.21", package = "uu_runcon", path = "src/uu/runcon" } -seq = { optional = true, version = "0.0.21", package = "uu_seq", path = "src/uu/seq" } -shred = { optional = true, version = "0.0.21", package = "uu_shred", path = "src/uu/shred" } -shuf = { optional = true, version = "0.0.21", package = "uu_shuf", path = "src/uu/shuf" } -sleep = { optional = true, version = "0.0.21", package = "uu_sleep", path = "src/uu/sleep" } -sort = { optional = true, version = "0.0.21", package = "uu_sort", path = "src/uu/sort" } -split = { optional = true, version = "0.0.21", package = "uu_split", path = "src/uu/split" } -stat = { optional = true, version = "0.0.21", package = "uu_stat", path = "src/uu/stat" } -stdbuf = { optional = true, version = "0.0.21", package = "uu_stdbuf", path = "src/uu/stdbuf" } -stty = { optional = true, version = "0.0.21", package = "uu_stty", path = "src/uu/stty" } -sum = { optional = true, version = "0.0.21", package = "uu_sum", path = "src/uu/sum" } -sync = { optional = true, version = "0.0.21", package = "uu_sync", path = "src/uu/sync" } -tac = { optional = true, version = "0.0.21", package = "uu_tac", path = "src/uu/tac" } -tail = { optional = true, version = "0.0.21", package = "uu_tail", path = "src/uu/tail" } -tee = { optional = true, version = "0.0.21", package = "uu_tee", path = "src/uu/tee" } -timeout = { optional = true, version = "0.0.21", package = "uu_timeout", path = "src/uu/timeout" } -touch = { optional = true, version = "0.0.21", package = "uu_touch", path = "src/uu/touch" } -tr = { optional = true, version = "0.0.21", package = "uu_tr", path = "src/uu/tr" } -true = { optional = true, version = "0.0.21", package = "uu_true", path = "src/uu/true" } -truncate = { optional = true, version = "0.0.21", package = "uu_truncate", path = "src/uu/truncate" } -tsort = { optional = true, version = "0.0.21", package = "uu_tsort", path = "src/uu/tsort" } -tty = { optional = true, version = "0.0.21", package = "uu_tty", path = "src/uu/tty" } -uname = { optional = true, version = "0.0.21", package = "uu_uname", path = "src/uu/uname" } -unexpand = { optional = true, version = "0.0.21", package = "uu_unexpand", path = "src/uu/unexpand" } -uniq = { optional = true, version = "0.0.21", package = "uu_uniq", path = "src/uu/uniq" } -unlink = { optional = true, version = "0.0.21", package = "uu_unlink", path = "src/uu/unlink" } -uptime = { optional = true, version = "0.0.21", package = "uu_uptime", path = "src/uu/uptime" } -users = { optional = true, version = "0.0.21", package = "uu_users", path = "src/uu/users" } -vdir = { optional = true, version = "0.0.21", package = "uu_vdir", path = "src/uu/vdir" } -wc = { optional = true, version = "0.0.21", package = "uu_wc", path = "src/uu/wc" } -who = { optional = true, version = "0.0.21", package = "uu_who", path = "src/uu/who" } -whoami = { optional = true, version = "0.0.21", package = "uu_whoami", path = "src/uu/whoami" } -yes = { optional = true, version = "0.0.21", package = "uu_yes", path = "src/uu/yes" } +arch = { optional = true, version = "0.1.0", package = "uu_arch", path = "src/uu/arch" } +base32 = { optional = true, version = "0.1.0", package = "uu_base32", path = "src/uu/base32" } +base64 = { optional = true, version = "0.1.0", package = "uu_base64", path = "src/uu/base64" } +basename = { optional = true, version = "0.1.0", package = "uu_basename", path = "src/uu/basename" } +basenc = { optional = true, version = "0.1.0", package = "uu_basenc", path = "src/uu/basenc" } +cat = { optional = true, version = "0.1.0", package = "uu_cat", path = "src/uu/cat" } +chcon = { optional = true, version = "0.1.0", package = "uu_chcon", path = "src/uu/chcon" } +chgrp = { optional = true, version = "0.1.0", package = "uu_chgrp", path = "src/uu/chgrp" } +chmod = { optional = true, version = "0.1.0", package = "uu_chmod", path = "src/uu/chmod" } +chown = { optional = true, version = "0.1.0", package = "uu_chown", path = "src/uu/chown" } +chroot = { optional = true, version = "0.1.0", package = "uu_chroot", path = "src/uu/chroot" } +cksum = { optional = true, version = "0.1.0", package = "uu_cksum", path = "src/uu/cksum" } +comm = { optional = true, version = "0.1.0", package = "uu_comm", path = "src/uu/comm" } +cp = { optional = true, version = "0.1.0", package = "uu_cp", path = "src/uu/cp" } +csplit = { optional = true, version = "0.1.0", package = "uu_csplit", path = "src/uu/csplit" } +cut = { optional = true, version = "0.1.0", package = "uu_cut", path = "src/uu/cut" } +date = { optional = true, version = "0.1.0", package = "uu_date", path = "src/uu/date" } +dd = { optional = true, version = "0.1.0", package = "uu_dd", path = "src/uu/dd" } +df = { optional = true, version = "0.1.0", package = "uu_df", path = "src/uu/df" } +dir = { optional = true, version = "0.1.0", package = "uu_dir", path = "src/uu/dir" } +dircolors = { optional = true, version = "0.1.0", package = "uu_dircolors", path = "src/uu/dircolors" } +dirname = { optional = true, version = "0.1.0", package = "uu_dirname", path = "src/uu/dirname" } +du = { optional = true, version = "0.1.0", package = "uu_du", path = "src/uu/du" } +echo = { optional = true, version = "0.1.0", package = "uu_echo", path = "src/uu/echo" } +env = { optional = true, version = "0.1.0", package = "uu_env", path = "src/uu/env" } +expand = { optional = true, version = "0.1.0", package = "uu_expand", path = "src/uu/expand" } +expr = { optional = true, version = "0.1.0", package = "uu_expr", path = "src/uu/expr" } +factor = { optional = true, version = "0.1.0", package = "uu_factor", path = "src/uu/factor" } +false = { optional = true, version = "0.1.0", package = "uu_false", path = "src/uu/false" } +fmt = { optional = true, version = "0.1.0", package = "uu_fmt", path = "src/uu/fmt" } +fold = { optional = true, version = "0.1.0", package = "uu_fold", path = "src/uu/fold" } +groups = { optional = true, version = "0.1.0", package = "uu_groups", path = "src/uu/groups" } +hashsum = { optional = true, version = "0.1.0", package = "uu_hashsum", path = "src/uu/hashsum" } +head = { optional = true, version = "0.1.0", package = "uu_head", path = "src/uu/head" } +hostid = { optional = true, version = "0.1.0", package = "uu_hostid", path = "src/uu/hostid" } +hostname = { optional = true, version = "0.1.0", package = "uu_hostname", path = "src/uu/hostname" } +id = { optional = true, version = "0.1.0", package = "uu_id", path = "src/uu/id" } +install = { optional = true, version = "0.1.0", package = "uu_install", path = "src/uu/install" } +join = { optional = true, version = "0.1.0", package = "uu_join", path = "src/uu/join" } +kill = { optional = true, version = "0.1.0", package = "uu_kill", path = "src/uu/kill" } +link = { optional = true, version = "0.1.0", package = "uu_link", path = "src/uu/link" } +ln = { optional = true, version = "0.1.0", package = "uu_ln", path = "src/uu/ln" } +ls = { optional = true, version = "0.1.0", package = "uu_ls", path = "src/uu/ls" } +logname = { optional = true, version = "0.1.0", package = "uu_logname", path = "src/uu/logname" } +mkdir = { optional = true, version = "0.1.0", package = "uu_mkdir", path = "src/uu/mkdir" } +mkfifo = { optional = true, version = "0.1.0", package = "uu_mkfifo", path = "src/uu/mkfifo" } +mknod = { optional = true, version = "0.1.0", package = "uu_mknod", path = "src/uu/mknod" } +mktemp = { optional = true, version = "0.1.0", package = "uu_mktemp", path = "src/uu/mktemp" } +more = { optional = true, version = "0.1.0", package = "uu_more", path = "src/uu/more" } +mv = { optional = true, version = "0.1.0", package = "uu_mv", path = "src/uu/mv" } +nice = { optional = true, version = "0.1.0", package = "uu_nice", path = "src/uu/nice" } +nl = { optional = true, version = "0.1.0", package = "uu_nl", path = "src/uu/nl" } +nohup = { optional = true, version = "0.1.0", package = "uu_nohup", path = "src/uu/nohup" } +nproc = { optional = true, version = "0.1.0", package = "uu_nproc", path = "src/uu/nproc" } +numfmt = { optional = true, version = "0.1.0", package = "uu_numfmt", path = "src/uu/numfmt" } +od = { optional = true, version = "0.1.0", package = "uu_od", path = "src/uu/od" } +paste = { optional = true, version = "0.1.0", package = "uu_paste", path = "src/uu/paste" } +pathchk = { optional = true, version = "0.1.0", package = "uu_pathchk", path = "src/uu/pathchk" } +pinky = { optional = true, version = "0.1.0", package = "uu_pinky", path = "src/uu/pinky" } +pr = { optional = true, version = "0.1.0", package = "uu_pr", path = "src/uu/pr" } +printenv = { optional = true, version = "0.1.0", package = "uu_printenv", path = "src/uu/printenv" } +printf = { optional = true, version = "0.1.0", package = "uu_printf", path = "src/uu/printf" } +ptx = { optional = true, version = "0.1.0", package = "uu_ptx", path = "src/uu/ptx" } +pwd = { optional = true, version = "0.1.0", package = "uu_pwd", path = "src/uu/pwd" } +readlink = { optional = true, version = "0.1.0", package = "uu_readlink", path = "src/uu/readlink" } +realpath = { optional = true, version = "0.1.0", package = "uu_realpath", path = "src/uu/realpath" } +rm = { optional = true, version = "0.1.0", package = "uu_rm", path = "src/uu/rm" } +rmdir = { optional = true, version = "0.1.0", package = "uu_rmdir", path = "src/uu/rmdir" } +runcon = { optional = true, version = "0.1.0", package = "uu_runcon", path = "src/uu/runcon" } +seq = { optional = true, version = "0.1.0", package = "uu_seq", path = "src/uu/seq" } +shred = { optional = true, version = "0.1.0", package = "uu_shred", path = "src/uu/shred" } +shuf = { optional = true, version = "0.1.0", package = "uu_shuf", path = "src/uu/shuf" } +sleep = { optional = true, version = "0.1.0", package = "uu_sleep", path = "src/uu/sleep" } +sort = { optional = true, version = "0.1.0", package = "uu_sort", path = "src/uu/sort" } +split = { optional = true, version = "0.1.0", package = "uu_split", path = "src/uu/split" } +stat = { optional = true, version = "0.1.0", package = "uu_stat", path = "src/uu/stat" } +stdbuf = { optional = true, version = "0.1.0", package = "uu_stdbuf", path = "src/uu/stdbuf" } +stty = { optional = true, version = "0.1.0", package = "uu_stty", path = "src/uu/stty" } +sum = { optional = true, version = "0.1.0", package = "uu_sum", path = "src/uu/sum" } +sync = { optional = true, version = "0.1.0", package = "uu_sync", path = "src/uu/sync" } +tac = { optional = true, version = "0.1.0", package = "uu_tac", path = "src/uu/tac" } +tail = { optional = true, version = "0.1.0", package = "uu_tail", path = "src/uu/tail" } +tee = { optional = true, version = "0.1.0", package = "uu_tee", path = "src/uu/tee" } +timeout = { optional = true, version = "0.1.0", package = "uu_timeout", path = "src/uu/timeout" } +touch = { optional = true, version = "0.1.0", package = "uu_touch", path = "src/uu/touch" } +tr = { optional = true, version = "0.1.0", package = "uu_tr", path = "src/uu/tr" } +true = { optional = true, version = "0.1.0", package = "uu_true", path = "src/uu/true" } +truncate = { optional = true, version = "0.1.0", package = "uu_truncate", path = "src/uu/truncate" } +tsort = { optional = true, version = "0.1.0", package = "uu_tsort", path = "src/uu/tsort" } +tty = { optional = true, version = "0.1.0", package = "uu_tty", path = "src/uu/tty" } +uname = { optional = true, version = "0.1.0", package = "uu_uname", path = "src/uu/uname" } +unexpand = { optional = true, version = "0.1.0", package = "uu_unexpand", path = "src/uu/unexpand" } +uniq = { optional = true, version = "0.1.0", package = "uu_uniq", path = "src/uu/uniq" } +unlink = { optional = true, version = "0.1.0", package = "uu_unlink", path = "src/uu/unlink" } +uptime = { optional = true, version = "0.1.0", package = "uu_uptime", path = "src/uu/uptime" } +users = { optional = true, version = "0.1.0", package = "uu_users", path = "src/uu/users" } +vdir = { optional = true, version = "0.1.0", package = "uu_vdir", path = "src/uu/vdir" } +wc = { optional = true, version = "0.1.0", package = "uu_wc", path = "src/uu/wc" } +who = { optional = true, version = "0.1.0", package = "uu_who", path = "src/uu/who" } +whoami = { optional = true, version = "0.1.0", package = "uu_whoami", path = "src/uu/whoami" } +yes = { optional = true, version = "0.1.0", package = "uu_yes", path = "src/uu/yes" } # this breaks clippy linting with: "tests/by-util/test_factor_benches.rs: No such file or directory (os error 2)" # factor_benches = { optional = true, version = "0.0.0", package = "uu_factor_benches", path = "tests/benches/factor" } @@ -478,30 +499,45 @@ yes = { optional = true, version = "0.0.21", package = "uu_yes", path = "src/uu/ [dev-dependencies] chrono = { workspace = true } -conv = "0.3" filetime = { workspace = true } glob = { workspace = true } libc = { workspace = true } -pretty_assertions = "1" +num-prime = { workspace = true } +pretty_assertions = "1.4.0" rand = { workspace = true } regex = { workspace = true } -sha1 = { version = "0.10", features = ["std"] } +sha1 = { workspace = true, features = ["std"] } tempfile = { workspace = true } time = { workspace = true, features = ["local-offset"] } -unindent = "0.2" -uucore = { workspace = true, features = ["entries", "process", "signals"] } +unindent = "0.2.3" +uutests = { workspace = true } +uucore = { workspace = true, features = [ + "mode", + "entries", + "process", + "signals", + "utmpx", +] } walkdir = { workspace = true } -is-terminal = { workspace = true } -hex-literal = "0.4.1" +hex-literal = "1.0.0" rstest = { workspace = true } +ctor = "0.4.1" [target.'cfg(any(target_os = "linux", target_os = "android"))'.dev-dependencies] -procfs = { version = "0.15", default-features = false } -rlimit = "0.10.1" +procfs = { version = "0.17", default-features = false } [target.'cfg(unix)'.dev-dependencies] -nix = { workspace = true, features = ["process", "signal", "user"] } -rand_pcg = "0.3" +nix = { workspace = true, features = ["process", "signal", "user", "term"] } +rlimit = "0.10.1" +xattr = { workspace = true } + +# Specifically used in test_uptime::test_uptime_with_file_containing_valid_boot_time_utmpx_record +# to deserialize a utmpx struct into a binary file +[target.'cfg(all(target_family= "unix",not(target_os = "macos")))'.dev-dependencies] +serde = { version = "1.0.202", features = ["derive"] } +bincode = { version = "2.0.1", features = ["serde"] } +serde-big-array = "0.5.1" + [build-dependencies] phf_codegen = { workspace = true } @@ -515,9 +551,9 @@ name = "uudoc" path = "src/bin/uudoc.rs" required-features = ["uudoc"] -# The default release profile. It contains all optimizations, without -# sacrificing debug info. With this profile (like in the standard -# release profile), the debug info and the stack traces will still be available. +# The default release profile. It contains all optimizations. +# With this profile (like in the standard release profile), +# the stack traces will still be available. [profile.release] lto = true @@ -533,3 +569,108 @@ inherits = "release" opt-level = "z" panic = "abort" strip = true + +# A release-like profile with debug info, useful for profiling. +# See https://github.com/mstange/samply . +[profile.profiling] +inherits = "release" +debug = true + +[lints.clippy] +multiple_crate_versions = "allow" +cargo_common_metadata = "allow" +uninlined_format_args = "allow" +missing_panics_doc = "allow" +# TODO remove when https://github.com/rust-lang/rust-clippy/issues/13774 is fixed +large_stack_arrays = "allow" + +use_self = "warn" +needless_pass_by_value = "warn" +semicolon_if_nothing_returned = "warn" +single_char_pattern = "warn" +explicit_iter_loop = "warn" +if_not_else = "warn" +manual_let_else = "warn" + +all = { level = "deny", priority = -1 } +cargo = { level = "warn", priority = -1 } +pedantic = { level = "deny", priority = -1 } + +# This is the linting configuration for all crates. +# Eventually the clippy settings from the `[lints]` section should be moved here. +# In order to use these, all crates have `[lints] workspace = true` section. +[workspace.lints.rust] +unused_qualifications = "warn" + +[workspace.lints.clippy] +# The counts were generated with this command: +# cargo clippy --all-targets --workspace --message-format=json --quiet \ +# | jq -r '.message.code.code | select(. != null and startswith("clippy::"))' \ +# | sort | uniq -c | sort -h -r +# +# TODO: +# remove large_stack_arrays when https://github.com/rust-lang/rust-clippy/issues/13774 is fixed +# +all = { level = "warn", priority = -1 } +cargo = { level = "warn", priority = -1 } +pedantic = { level = "warn", priority = -1 } +cargo_common_metadata = "allow" # 3240 +multiple_crate_versions = "allow" # 2314 +missing_errors_doc = "allow" # 1504 +missing_panics_doc = "allow" # 946 +must_use_candidate = "allow" # 322 +doc_markdown = "allow" # 267 +match_same_arms = "allow" # 212 +unnecessary_semicolon = "allow" # 156 +redundant_closure_for_method_calls = "allow" # 133 +cast_possible_truncation = "allow" # 118 +too_many_lines = "allow" # 81 +cast_possible_wrap = "allow" # 76 +trivially_copy_pass_by_ref = "allow" # 74 +cast_sign_loss = "allow" # 70 +struct_excessive_bools = "allow" # 68 +single_match_else = "allow" # 66 +redundant_else = "allow" # 58 +map_unwrap_or = "allow" # 54 +cast_precision_loss = "allow" # 52 +unnested_or_patterns = "allow" # 40 +inefficient_to_string = "allow" # 38 +unnecessary_wraps = "allow" # 37 +cast_lossless = "allow" # 33 +ignored_unit_patterns = "allow" # 29 +needless_continue = "allow" # 28 +items_after_statements = "allow" # 22 +similar_names = "allow" # 20 +wildcard_imports = "allow" # 18 +used_underscore_binding = "allow" # 16 +large_stack_arrays = "allow" # 14 +float_cmp = "allow" # 12 +# semicolon_if_nothing_returned = "allow" # 9 +used_underscore_items = "allow" # 8 +return_self_not_must_use = "allow" # 8 +needless_pass_by_value = "allow" # 8 +# manual_let_else = "allow" # 8 +# needless_raw_string_hashes = "allow" # 7 +match_on_vec_items = "allow" # 6 +inline_always = "allow" # 6 +# format_push_string = "allow" # 6 +fn_params_excessive_bools = "allow" # 6 +# single_char_pattern = "allow" # 4 +# ptr_cast_constness = "allow" # 4 +# match_wildcard_for_single_variants = "allow" # 4 +# manual_is_variant_and = "allow" # 4 +# explicit_deref_methods = "allow" # 4 +# enum_glob_use = "allow" # 3 +# unnecessary_literal_bound = "allow" # 2 +# stable_sort_primitive = "allow" # 2 +should_panic_without_expect = "allow" # 2 +# ptr_as_ptr = "allow" # 2 +# needless_for_each = "allow" # 2 +if_not_else = "allow" # 2 +expl_impl_clone_on_copy = "allow" # 2 +# cloned_instead_of_copied = "allow" # 2 +# borrow_as_ptr = "allow" # 2 +bool_to_int_with_if = "allow" # 2 +# ref_as_ptr = "allow" # 2 +# unreadable_literal = "allow" # 1 +uninlined_format_args = "allow" # ? diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000000..6091c394fc4 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,342 @@ + + +# Setting up your local development environment + +For contributing rules and best practices please refer to [CONTRIBUTING.md](CONTRIBUTING.md) + +## Before you start + +For this guide we assume that you already have a GitHub account and have `git` and your favorite code editor or IDE installed and configured. +Before you start working on coreutils, please follow these steps: + +1. Fork the [coreutils repository](https://github.com/uutils/coreutils) to your GitHub account. +***Tip:*** See [this GitHub guide](https://docs.github.com/en/get-started/quickstart/fork-a-repo) for more information on this step. +2. Clone that fork to your local development environment: + +```shell +git clone https://github.com/YOUR-GITHUB-ACCOUNT/coreutils +cd coreutils +``` + +## Tools + +You will need the tools mentioned in this section to build and test your code changes locally. +This section will explain how to install and configure these tools. +We also have an extensive CI that uses these tools and will check your code before it can be merged. +The next section [Testing](#testing) will explain how to run those checks locally to avoid waiting for the CI. + +### Rust toolchain + +[Install Rust](https://www.rust-lang.org/tools/install) + +If you're using rustup to install and manage your Rust toolchains, `clippy` and `rustfmt` are usually already installed. If you are using one of the alternative methods, please make sure to install them manually. See following sub-sections for their usage: [clippy](#clippy) [rustfmt](#rustfmt). + +***Tip*** You might also need to add 'llvm-tools' component if you are going to [generate code coverage reports locally](#code-coverage-report): + +```shell +rustup component add llvm-tools-preview +``` + +### GNU utils and prerequisites + +If you are developing on Linux, most likely you already have all/most GNU utilities and prerequisites installed. + +To make sure, please check GNU coreutils [README-prereq](https://github.com/coreutils/coreutils/blob/master/README-prereq). + +You will need these to [run uutils against the GNU test suite locally](#comparing-with-gnu). + +For MacOS and Windows platform specific setup please check [MacOS GNU utils](#macos-gnu-utils) and [Windows GNU utils](#windows-gnu-utils) sections respectfully. + +### pre-commit hooks + +A configuration for `pre-commit` is provided in the repository. It allows +automatically checking every git commit you make to ensure it compiles, and +passes `clippy` and `rustfmt` without warnings. + +To use the provided hook: + +1. [Install `pre-commit`](https://pre-commit.com/#install) +1. Run `pre-commit install` while in the repository directory + +Your git commits will then automatically be checked. If a check fails, an error +message will explain why, and your commit will be canceled. You can then make +the suggested changes, and run `git commit ...` again. + +**NOTE: On MacOS** the pre-commit hooks are currently broken. There are workarounds involving switching to unstable nightly Rust and components. + +### clippy + +```shell +cargo clippy --all-targets --all-features +``` + +The `msrv` key in the clippy configuration file `clippy.toml` is used to disable +lints pertaining to newer features by specifying the minimum supported Rust +version (MSRV). + +### rustfmt + +```shell +cargo fmt --all +``` + +### cargo-deny + +This project uses [cargo-deny](https://github.com/EmbarkStudios/cargo-deny/) to +detect duplicate dependencies, checks licenses, etc. To run it locally, first +install it and then run with: + +```shell +cargo deny --all-features check all +``` + +### Markdown linter + +We use [markdownlint](https://github.com/DavidAnson/markdownlint) to lint the +Markdown files in the repository. + +### Spell checker + +We use `cspell` as spell checker for all files in the project. If you are using +VS Code, you can install the +[code spell checker](https://marketplace.visualstudio.com/items?itemName=streetsidesoftware.code-spell-checker) +extension to enable spell checking within your editor. Otherwise, you can +install [cspell](https://cspell.org/) separately. + +If you want to make the spell checker ignore a word, you can add + +```rust +// spell-checker:ignore word_to_ignore +``` + +at the top of the file. + +## Testing + +This section explains how to run our CI checks locally. +Testing can be done using either Cargo or `make`. + +### Testing with Cargo + +Just like with building, we follow the standard procedure for testing using +Cargo: + +```shell +cargo test +``` + +By default, `cargo test` only runs the common programs. To run also platform +specific tests, run: + +```shell +cargo test --features unix +``` + +If you would prefer to test a select few utilities: + +```shell +cargo test --features "chmod mv tail" --no-default-features +``` + +If you also want to test the core utilities: + +```shell +cargo test -p uucore -p coreutils +# or +cargo test --all-features -p uucore +``` + +Running the complete test suite might take a while. We use [nextest](https://nexte.st/index.html) in +the CI and you might want to try it out locally. It can speed up the execution time of the whole +test run significantly if the cpu has multiple cores. + +```shell +cargo nextest run --features unix --no-fail-fast +``` + +To debug: + +```shell +rust-gdb --args target/debug/coreutils ls +(gdb) b ls.rs:79 +(gdb) run +``` + +### Testing with GNU Make + +To simply test all available utilities: + +```shell +make test +``` + +To test all but a few of the available utilities: + +```shell +make SKIP_UTILS='UTILITY_1 UTILITY_2' test +``` + +To test only a few of the available utilities: + +```shell +make UTILS='UTILITY_1 UTILITY_2' test +``` + +To include tests for unimplemented behavior: + +```shell +make UTILS='UTILITY_1 UTILITY_2' SPEC=y test +``` + +To run tests with `nextest` just use the nextest target. Note you'll need to +[install](https://nexte.st/book/installation.html) `nextest` first. The `nextest` target accepts the +same arguments like the default `test` target, so it's possible to pass arguments to `nextest run` +via `CARGOFLAGS`: + +```shell +make CARGOFLAGS='--no-fail-fast' UTILS='UTILITY_1 UTILITY_2' nextest +``` + +### Run Busybox Tests + +This testing functionality is only available on *nix operating systems and +requires `make`. + +To run busybox tests for all utilities for which busybox has tests + +```shell +make busytest +``` + +To run busybox tests for a few of the available utilities + +```shell +make UTILS='UTILITY_1 UTILITY_2' busytest +``` + +To pass an argument like "-v" to the busybox test runtime + +```shell +make UTILS='UTILITY_1 UTILITY_2' RUNTEST_ARGS='-v' busytest +``` + +### Comparing with GNU + +To run uutils against the GNU test suite locally, run the following commands: + +```shell +bash util/build-gnu.sh +# Build uutils with release optimizations +bash util/build-gnu.sh --release-build +bash util/run-gnu-test.sh +# To run a single test: +bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example +# To run several tests: +bash util/run-gnu-test.sh tests/touch/not-owner.sh tests/rm/no-give-up.sh # for example +# If this is a perl (.pl) test, to run in debug: +DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl +``` + +***Tip:*** First time you run `bash util/build-gnu.sh` command, it will provide instructions on how to checkout GNU coreutils repository at the correct release tag. Please follow those instructions and when done, run `bash util/build-gnu.sh` command again. + +Note that GNU test suite relies on individual utilities (not the multicall binary). + +You also need to install [quilt](https://savannah.nongnu.org/projects/quilt), a tool used to manage a stack of patches for modifying GNU tests. + +On FreeBSD, you need to install packages for GNU coreutils and sed (used in shell scripts instead of system commands): + +```shell +pkg install coreutils gsed +``` + +## Code coverage report + +Code coverage report can be generated using [grcov](https://github.com/mozilla/grcov). + +To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-a-rust-project) coverage report + +```shell +export CARGO_INCREMENTAL=0 +export RUSTFLAGS="-Cinstrument-coverage -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" +export RUSTDOCFLAGS="-Cpanic=abort" +cargo build # e.g., --features feat_os_unix +cargo test # e.g., --features feat_os_unix test_pathchk +grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing --ignore build.rs --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?\#\[derive\()" -o ./target/debug/coverage/ +# open target/debug/coverage/index.html in browser +``` + +if changes are not reflected in the report then run `cargo clean` and run the above commands. + +## Tips for setting up on Mac + +### C Compiler and linker + +On MacOS you'll need to install C compiler & linker: + +```shell +xcode-select --install +``` + +### MacOS GNU utils + +On MacOS you will need to install [Homebrew](https://docs.brew.sh/Installation) and use it to install the following Homebrew formulas: + +```shell +brew install \ + coreutils \ + autoconf \ + gettext \ + wget \ + texinfo \ + xz \ + automake \ + gnu-sed \ + m4 \ + bison \ + pre-commit \ + findutils +``` + +After installing these Homebrew formulas, please make sure to add the following lines to your `zsh` or `bash` rc file, i.e. `~/.profile` or `~/.zshrc` or `~/.bashrc` ... +(assuming Homebrew is installed at default location `/opt/homebrew`): + +```shell +eval "$(/opt/homebrew/bin/brew shellenv)" +export PATH="/opt/homebrew/opt/coreutils/libexec/gnubin:$PATH" +export PATH="/opt/homebrew/opt/bison/bin:$PATH" +export PATH="/opt/homebrew/opt/findutils/libexec/gnubin:$PATH" +``` + +Last step is to link Homebrew coreutils version of `timeout` to `/usr/local/bin` (as admin user): + +```shell +sudo ln -s /opt/homebrew/bin/timeout /usr/local/bin/timeout +``` + +Do not forget to either source updated rc file or restart you terminal session to update environment variables. + +## Tips for setting up on Windows + +### MSVC build tools + +On Windows you'll need the MSVC build tools for Visual Studio 2013 or later. + +If you are using `rustup-init.exe` to install Rust toolchain, it will guide you through the process of downloading and installing these prerequisites. + +Otherwise please follow [this guide](https://learn.microsoft.com/en-us/windows/dev-environment/rust/setup). + +### Windows GNU utils + +If you have used [Git for Windows](https://gitforwindows.org) to install `git` on you Windows system you might already have some GNU core utilities installed as part of "GNU Bash" included in Git for Windows package, but it is not a complete package. [This article](https://gist.github.com/evanwill/0207876c3243bbb6863e65ec5dc3f058) provides instruction on how to add more to it. + +Alternatively you can install [Cygwin](https://www.cygwin.com) and/or use [WSL2](https://learn.microsoft.com/en-us/windows/wsl/compare-versions#whats-new-in-wsl-2) to get access to all GNU core utilities on Windows. + +## Preparing a new release + +1. Modify `util/update-version.sh` (FROM & TO) and run it +1. Submit a new PR with these changes and wait for it to be merged +1. Tag the new release `git tag -a X.Y.Z` and `git push --tags` +1. Once the CI is green, a new release will be automatically created in draft mode. + Reuse this release and make sure that assets have been added. +1. Write the release notes (it takes time) following previous examples +1. Run `util/publish.sh --do-it` to publish the new release to crates.io diff --git a/GNUmakefile b/GNUmakefile index c672458a1cd..dbcdcd0ce08 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -3,11 +3,20 @@ # Config options PROFILE ?= debug MULTICALL ?= n +COMPLETIONS ?= y +MANPAGES ?= y +LOCALES ?= y INSTALL ?= install ifneq (,$(filter install, $(MAKECMDGOALS))) override PROFILE:=release endif +# Needed for the foreach loops to split each loop into a separate command +define newline + + +endef + PROFILE_CMD := ifeq ($(PROFILE),release) PROFILE_CMD = --release @@ -33,23 +42,32 @@ PROG_PREFIX ?= # This won't support any directory with spaces in its name, but you can just # make a symlink without spaces that points to the directory. BASEDIR ?= $(shell pwd) +ifdef CARGO_TARGET_DIR +BUILDDIR := $(CARGO_TARGET_DIR)/${PROFILE} +else BUILDDIR := $(BASEDIR)/target/${PROFILE} +endif PKG_BUILDDIR := $(BUILDDIR)/deps DOCSDIR := $(BASEDIR)/docs BUSYBOX_ROOT := $(BASEDIR)/tmp -BUSYBOX_VER := 1.35.0 +BUSYBOX_VER := 1.36.1 BUSYBOX_SRC := $(BUSYBOX_ROOT)/busybox-$(BUSYBOX_VER) TOYBOX_ROOT := $(BASEDIR)/tmp -TOYBOX_VER := 0.8.8 +TOYBOX_VER := 0.8.12 TOYBOX_SRC := $(TOYBOX_ROOT)/toybox-$(TOYBOX_VER) -ifeq ($(SELINUX_ENABLED),) - SELINUX_ENABLED := 0 + +ifdef SELINUX_ENABLED + override SELINUX_ENABLED := 0 +# Now check if we should enable it (only on non-Windows) ifneq ($(OS),Windows_NT) - ifeq ($(shell /sbin/selinuxenabled 2>/dev/null ; echo $$?),0) - SELINUX_ENABLED := 1 + ifeq ($(shell if [ -x /sbin/selinuxenabled ] && /sbin/selinuxenabled 2>/dev/null; then echo 0; else echo 1; fi),0) + override SELINUX_ENABLED := 1 +$(info /sbin/selinuxenabled successful) + else +$(info SELINUX_ENABLED=1 but /sbin/selinuxenabled failed) endif endif endif @@ -102,7 +120,6 @@ PROGS := \ pwd \ readlink \ realpath \ - relpath \ rm \ rmdir \ seq \ @@ -111,6 +128,7 @@ PROGS := \ sleep \ sort \ split \ + stty \ sum \ sync \ tac \ @@ -165,9 +183,7 @@ SELINUX_PROGS := \ ifneq ($(OS),Windows_NT) PROGS := $(PROGS) $(UNIX_PROGS) -endif - -ifeq ($(SELINUX_ENABLED),1) +# Build the selinux command even if not on the system PROGS := $(PROGS) $(SELINUX_PROGS) endif @@ -222,6 +238,7 @@ TEST_PROGS := \ rmdir \ runcon \ seq \ + sleep \ sort \ split \ stat \ @@ -252,7 +269,8 @@ TEST_NO_FAIL_FAST :=--no-fail-fast TEST_SPEC_FEATURE := test_unimplemented else ifeq ($(SELINUX_ENABLED),1) TEST_NO_FAIL_FAST := -TEST_SPEC_FEATURE := feat_selinux +TEST_SPEC_FEATURE := selinux +BUILD_SPEC_FEATURE := selinux endif define TEST_BUSYBOX @@ -276,11 +294,15 @@ use_default := 1 build-pkgs: ifneq (${MULTICALL}, y) +ifdef BUILD_SPEC_FEATURE + ${CARGO} build ${CARGOFLAGS} --features "$(BUILD_SPEC_FEATURE)" ${PROFILE_CMD} $(foreach pkg,$(EXES),-p uu_$(pkg)) +else ${CARGO} build ${CARGOFLAGS} ${PROFILE_CMD} $(foreach pkg,$(EXES),-p uu_$(pkg)) endif +endif -build-coreutils: - ${CARGO} build ${CARGOFLAGS} --features "${EXES}" ${PROFILE_CMD} --no-default-features +build-coreutils: locales + ${CARGO} build ${CARGOFLAGS} --features "${EXES} $(BUILD_SPEC_FEATURE)" ${PROFILE_CMD} --no-default-features build: build-coreutils build-pkgs @@ -337,45 +359,83 @@ clean: distclean: clean $(CARGO) clean $(CARGOFLAGS) && $(CARGO) update $(CARGOFLAGS) +ifeq ($(MANPAGES),y) manpages: build-coreutils mkdir -p $(BUILDDIR)/man/ $(foreach prog, $(INSTALLEES), \ - $(BUILDDIR)/coreutils manpage $(prog) > $(BUILDDIR)/man/$(PROG_PREFIX)$(prog).1; \ + $(BUILDDIR)/coreutils manpage $(prog) > $(BUILDDIR)/man/$(PROG_PREFIX)$(prog).1 $(newline) \ ) +install-manpages: manpages + mkdir -p $(DESTDIR)$(DATAROOTDIR)/man/man1 + $(foreach prog, $(INSTALLEES), \ + $(INSTALL) $(BUILDDIR)/man/$(PROG_PREFIX)$(prog).1 $(DESTDIR)$(DATAROOTDIR)/man/man1/ $(newline) \ + ) +else +install-manpages: +endif + +ifeq ($(COMPLETIONS),y) completions: build-coreutils mkdir -p $(BUILDDIR)/completions/zsh $(BUILDDIR)/completions/bash $(BUILDDIR)/completions/fish $(foreach prog, $(INSTALLEES), \ - $(BUILDDIR)/coreutils completion $(prog) zsh > $(BUILDDIR)/completions/zsh/_$(PROG_PREFIX)$(prog); \ - $(BUILDDIR)/coreutils completion $(prog) bash > $(BUILDDIR)/completions/bash/$(PROG_PREFIX)$(prog); \ - $(BUILDDIR)/coreutils completion $(prog) fish > $(BUILDDIR)/completions/fish/$(PROG_PREFIX)$(prog).fish; \ + $(BUILDDIR)/coreutils completion $(prog) zsh > $(BUILDDIR)/completions/zsh/_$(PROG_PREFIX)$(prog) $(newline) \ + $(BUILDDIR)/coreutils completion $(prog) bash > $(BUILDDIR)/completions/bash/$(PROG_PREFIX)$(prog) $(newline) \ + $(BUILDDIR)/coreutils completion $(prog) fish > $(BUILDDIR)/completions/fish/$(PROG_PREFIX)$(prog).fish $(newline) \ + ) + +install-completions: completions + mkdir -p $(DESTDIR)$(DATAROOTDIR)/zsh/site-functions + mkdir -p $(DESTDIR)$(DATAROOTDIR)/bash-completion/completions + mkdir -p $(DESTDIR)$(DATAROOTDIR)/fish/vendor_completions.d + $(foreach prog, $(INSTALLEES), \ + $(INSTALL) $(BUILDDIR)/completions/zsh/_$(PROG_PREFIX)$(prog) $(DESTDIR)$(DATAROOTDIR)/zsh/site-functions/ $(newline) \ + $(INSTALL) $(BUILDDIR)/completions/bash/$(PROG_PREFIX)$(prog) $(DESTDIR)$(DATAROOTDIR)/bash-completion/completions/ $(newline) \ + $(INSTALL) $(BUILDDIR)/completions/fish/$(PROG_PREFIX)$(prog).fish $(DESTDIR)$(DATAROOTDIR)/fish/vendor_completions.d/ $(newline) \ + ) +else +install-completions: +endif + +ifeq ($(LOCALES),y) +locales: + $(foreach prog, $(INSTALLEES), \ + if [ -d "$(BASEDIR)/src/uu/$(prog)/locales" ]; then \ + mkdir -p "$(BUILDDIR)/locales/$(prog)"; \ + for locale_file in "$(BASEDIR)"/src/uu/$(prog)/locales/*.ftl; do \ + $(INSTALL) -v "$$locale_file" "$(BUILDDIR)/locales/$(prog)/"; \ + done; \ + fi $(newline) \ ) -install: build manpages completions + +install-locales: + $(foreach prog, $(INSTALLEES), \ + if [ -d "$(BASEDIR)/src/uu/$(prog)/locales" ]; then \ + mkdir -p "$(DESTDIR)$(DATAROOTDIR)/locales/$(prog)"; \ + for locale_file in "$(BASEDIR)"/src/uu/$(prog)/locales/*.ftl; do \ + $(INSTALL) -v "$$locale_file" "$(DESTDIR)$(DATAROOTDIR)/locales/$(prog)/"; \ + done; \ + fi $(newline) \ + ) +else +install-locales: +endif + +install: build install-manpages install-completions install-locales mkdir -p $(INSTALLDIR_BIN) ifeq (${MULTICALL}, y) $(INSTALL) $(BUILDDIR)/coreutils $(INSTALLDIR_BIN)/$(PROG_PREFIX)coreutils - cd $(INSTALLDIR_BIN) && $(foreach prog, $(filter-out coreutils, $(INSTALLEES)), \ - ln -fs $(PROG_PREFIX)coreutils $(PROG_PREFIX)$(prog) &&) : + $(foreach prog, $(filter-out coreutils, $(INSTALLEES)), \ + cd $(INSTALLDIR_BIN) && ln -fs $(PROG_PREFIX)coreutils $(PROG_PREFIX)$(prog) $(newline) \ + ) $(if $(findstring test,$(INSTALLEES)), cd $(INSTALLDIR_BIN) && ln -fs $(PROG_PREFIX)coreutils $(PROG_PREFIX)[) else $(foreach prog, $(INSTALLEES), \ - $(INSTALL) $(BUILDDIR)/$(prog) $(INSTALLDIR_BIN)/$(PROG_PREFIX)$(prog);) + $(INSTALL) $(BUILDDIR)/$(prog) $(INSTALLDIR_BIN)/$(PROG_PREFIX)$(prog) $(newline) \ + ) $(if $(findstring test,$(INSTALLEES)), $(INSTALL) $(BUILDDIR)/test $(INSTALLDIR_BIN)/$(PROG_PREFIX)[) endif - mkdir -p $(DESTDIR)$(DATAROOTDIR)/man/man1 - $(foreach prog, $(INSTALLEES), \ - $(INSTALL) $(BUILDDIR)/man/$(PROG_PREFIX)$(prog).1 $(DESTDIR)$(DATAROOTDIR)/man/man1/; \ - ) - - mkdir -p $(DESTDIR)$(DATAROOTDIR)/zsh/site-functions - mkdir -p $(DESTDIR)$(DATAROOTDIR)/bash-completion/completions - mkdir -p $(DESTDIR)$(DATAROOTDIR)/fish/vendor_completions.d - $(foreach prog, $(INSTALLEES), \ - $(INSTALL) $(BUILDDIR)/completions/zsh/_$(PROG_PREFIX)$(prog) $(DESTDIR)$(DATAROOTDIR)/zsh/site-functions/; \ - $(INSTALL) $(BUILDDIR)/completions/bash/$(PROG_PREFIX)$(prog) $(DESTDIR)$(DATAROOTDIR)/bash-completion/completions/; \ - $(INSTALL) $(BUILDDIR)/completions/fish/$(PROG_PREFIX)$(prog).fish $(DESTDIR)$(DATAROOTDIR)/fish/vendor_completions.d/; \ - ) uninstall: ifeq (${MULTICALL}, y) diff --git a/LICENSE b/LICENSE index 49fdbd4cf5f..21bd44404e3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) Jordi Boggiano and many others +Copyright (c) uutils developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.md b/README.md index c5b4609c924..1d9a7ddd190 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ [![dependency status](https://deps.rs/repo/github/uutils/coreutils/status.svg)](https://deps.rs/repo/github/uutils/coreutils) [![CodeCov](https://codecov.io/gh/uutils/coreutils/branch/master/graph/badge.svg)](https://codecov.io/gh/uutils/coreutils) -![MSRV](https://img.shields.io/badge/MSRV-1.64.0-brightgreen) +![MSRV](https://img.shields.io/badge/MSRV-1.85.0-brightgreen) @@ -45,17 +45,16 @@ uutils aims to be a drop-in replacement for the GNU utils. Differences with GNU are treated as bugs. uutils aims to work on as many platforms as possible, to be able to use the same -utils on Linux, Mac, Windows and other platforms. This ensures, for example, +utils on Linux, macOS, Windows and other platforms. This ensures, for example, that scripts can be easily transferred between platforms.
## Documentation - uutils has both user and developer documentation available: -- [User Manual](https://uutils.github.io/coreutils/book/) -- [Developer Documentation](https://uutils.github.io/dev/coreutils/) (currently offline, you can use docs.rs in the meantime) +- [User Manual](https://uutils.github.io/coreutils/docs/) +- [Developer Documentation](https://docs.rs/crate/coreutils/) Both can also be generated locally, the instructions for that can be found in the [coreutils docs](https://github.com/uutils/uutils.github.io) repository. @@ -71,7 +70,7 @@ the [coreutils docs](https://github.com/uutils/uutils.github.io) repository. ### Rust Version uutils follows Rust's release channels and is tested against stable, beta and -nightly. The current Minimum Supported Rust Version (MSRV) is `1.64.0`. +nightly. The current Minimum Supported Rust Version (MSRV) is `1.85.0`. ## Building @@ -79,7 +78,7 @@ There are currently two methods to build the uutils binaries: either Cargo or GNU Make. > Building the full package, including all documentation, requires both Cargo -> and Gnu Make on a Unix platform. +> and GNU Make on a Unix platform. For either method, we first need to fetch the repository: @@ -224,6 +223,12 @@ Installing with `make` installs shell completions for all installed utilities for `bash`, `fish` and `zsh`. Completions for `elvish` and `powershell` can also be generated; See `Manually install shell completions`. +To skip installation of completions and manpages: + +```shell +make COMPLETIONS=n MANPAGES=n install +``` + ### Manually install shell completions The `coreutils` binary can generate completions for the `bash`, `elvish`, @@ -267,7 +272,7 @@ Make to uninstall. To uninstall uutils: ```shell -cargo uninstall uutils +cargo uninstall coreutils ``` ### Uninstall with GNU Make @@ -303,12 +308,12 @@ make PREFIX=/my/path uninstall Below is the evolution of how many GNU tests uutils passes. A more detailed breakdown of the GNU test results of the main branch can be found -[in the user manual](https://uutils.github.io/coreutils/book/test_coverage.html). +[in the user manual](https://uutils.github.io/coreutils/docs/test_coverage.html). -See for the main meta bugs +See for the main meta bugs (many are missing). -![Evolution over time](https://github.com/uutils/coreutils-tracking/blob/main/gnu-results.png?raw=true) +![Evolution over time](https://github.com/uutils/coreutils-tracking/blob/main/gnu-results.svg?raw=true)
diff --git a/README.package.md b/README.package.md new file mode 100644 index 00000000000..355b153db28 --- /dev/null +++ b/README.package.md @@ -0,0 +1,31 @@ + + + +
+
+ +![uutils logo](docs/src/logo.svg) + +# uutils coreutils + +[![Crates.io](https://img.shields.io/crates/v/coreutils.svg)](https://crates.io/crates/coreutils) +[![Discord](https://img.shields.io/badge/discord-join-7289DA.svg?logo=discord&longCache=true&style=flat)](https://discord.gg/wQVJbvJ) +[![License](http://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/uutils/coreutils/blob/main/LICENSE) +[![dependency status](https://deps.rs/repo/github/uutils/coreutils/status.svg)](https://deps.rs/repo/github/uutils/coreutils) + +[![CodeCov](https://codecov.io/gh/uutils/coreutils/branch/master/graph/badge.svg)](https://codecov.io/gh/uutils/coreutils) +![MSRV](https://img.shields.io/badge/MSRV-1.70.0-brightgreen) + +
+ +--- + +
+ +This package is part of uutils coreutils. + +uutils coreutils is a cross-platform reimplementation of the GNU coreutils in +[Rust](http://www.rust-lang.org). + +This package does not have its specific `README.md`. + diff --git a/build.rs b/build.rs index bb4e2b53609..3b6aa3878d1 100644 --- a/build.rs +++ b/build.rs @@ -11,14 +11,18 @@ use std::io::Write; use std::path::Path; pub fn main() { - if let Ok(profile) = env::var("PROFILE") { - println!("cargo:rustc-cfg=build={profile:?}"); - } - const ENV_FEATURE_PREFIX: &str = "CARGO_FEATURE_"; const FEATURE_PREFIX: &str = "feat_"; const OVERRIDE_PREFIX: &str = "uu_"; + // Do not rebuild build script unless the script itself or the enabled features are modified + // See + println!("cargo:rerun-if-changed=build.rs"); + + if let Ok(profile) = env::var("PROFILE") { + println!("cargo:rustc-cfg=build={profile:?}"); + } + let out_dir = env::var("OUT_DIR").unwrap(); let mut crates = Vec::new(); @@ -29,8 +33,10 @@ pub fn main() { #[allow(clippy::match_same_arms)] match krate.as_ref() { "default" | "macos" | "unix" | "windows" | "selinux" | "zip" => continue, // common/standard feature names - "nightly" | "test_unimplemented" => continue, // crate-local custom features - "uudoc" => continue, // is not a utility + "nightly" | "test_unimplemented" | "expensive_tests" | "test_risky_names" => { + continue; + } // crate-local custom features + "uudoc" => continue, // is not a utility "test" => continue, // over-ridden with 'uu_test' to avoid collision with rust core crate 'test' s if s.starts_with(FEATURE_PREFIX) => continue, // crate feature sets _ => {} // util feature name @@ -46,6 +52,7 @@ pub fn main() { "type UtilityMap = phf::OrderedMap<&'static str, (fn(T) -> i32, fn() -> Command)>;\n\ \n\ #[allow(clippy::too_many_lines)] + #[allow(clippy::unreadable_literal)] fn util_map() -> UtilityMap {\n" .as_bytes(), ) diff --git a/deny.toml b/deny.toml index c2e580a4de0..1b1700dcdf4 100644 --- a/deny.toml +++ b/deny.toml @@ -6,10 +6,8 @@ [advisories] db-path = "~/.cargo/advisory-db" db-urls = ["https://github.com/rustsec/advisory-db"] -vulnerability = "warn" -unmaintained = "warn" +version = 2 yanked = "warn" -notice = "warn" ignore = [ #"RUSTSEC-0000-0000", ] @@ -18,20 +16,18 @@ ignore = [ # More documentation for the licenses section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html [licenses] -unlicensed = "deny" +version = 2 allow = [ "MIT", "Apache-2.0", "ISC", "BSD-2-Clause", - "BSD-2-Clause-FreeBSD", "BSD-3-Clause", + "BSL-1.0", "CC0-1.0", - "Unicode-DFS-2016", + "Unicode-3.0", + "Zlib", ] -copyleft = "deny" -allow-osi-fsf-free = "neither" -default = "deny" confidence-threshold = 0.8 [[licenses.clarify]] @@ -58,35 +54,50 @@ highlight = "all" # introduces it. # spell-checker: disable skip = [ - # procfs - { name = "rustix", version = "0.36.15" }, - # rustix - { name = "linux-raw-sys", version = "0.1.4" }, - { name = "linux-raw-sys", version = "0.3.8" }, - # terminal_size - { name = "rustix", version = "0.37.23" }, - # various crates - { name = "windows-sys", version = "0.45.0" }, + # dns-lookup + { name = "windows-sys", version = "0.48.0" }, + # mio, nu-ansi-term, socket2 + { name = "windows-sys", version = "0.52.0" }, # windows-sys - { name = "windows-targets", version = "0.42.2" }, + { name = "windows-targets", version = "0.48.0" }, # windows-targets - { name = "windows_aarch64_gnullvm", version = "0.42.2" }, + { name = "windows_aarch64_gnullvm", version = "0.48.0" }, # windows-targets - { name = "windows_aarch64_msvc", version = "0.42.2" }, + { name = "windows_aarch64_msvc", version = "0.48.0" }, # windows-targets - { name = "windows_i686_gnu", version = "0.42.2" }, + { name = "windows_i686_gnu", version = "0.48.0" }, # windows-targets - { name = "windows_i686_msvc", version = "0.42.2" }, + { name = "windows_i686_msvc", version = "0.48.0" }, # windows-targets - { name = "windows_x86_64_gnu", version = "0.42.2" }, + { name = "windows_x86_64_gnu", version = "0.48.0" }, # windows-targets - { name = "windows_x86_64_gnullvm", version = "0.42.2" }, + { name = "windows_x86_64_gnullvm", version = "0.48.0" }, # windows-targets - { name = "windows_x86_64_msvc", version = "0.42.2" }, - # various crates - { name = "syn", version = "1.0.109" }, - # various crates + { name = "windows_x86_64_msvc", version = "0.48.0" }, + # kqueue-sys, onig { name = "bitflags", version = "1.3.2" }, + # ansi-width + { name = "unicode-width", version = "0.1.13" }, + # filedescriptor, utmp-classic + { name = "thiserror", version = "1.0.69" }, + # thiserror + { name = "thiserror-impl", version = "1.0.69" }, + # bindgen + { name = "itertools", version = "0.13.0" }, + # ordered-multimap + { name = "hashbrown", version = "0.14.5" }, + # cexpr (via bindgen) + { name = "nom", version = "7.1.3" }, + # const-random-macro, rand_core + { name = "getrandom", version = "0.2.15" }, + # getrandom, mio + { name = "wasi", version = "0.11.0+wasi-snapshot-preview1" }, + # num-bigint, num-prime, phf_generator + { name = "rand", version = "0.8.5" }, + # rand + { name = "rand_chacha", version = "0.3.1" }, + # rand + { name = "rand_core", version = "0.6.4" }, ] # spell-checker: enable diff --git a/docs/compiles_table.csv b/docs/compiles_table.csv index d18854e0e91..e263067b78d 100644 --- a/docs/compiles_table.csv +++ b/docs/compiles_table.csv @@ -1,21 +1,21 @@ -target,arch,base32,base64,basename,cat,chgrp,chmod,chown,chroot,cksum,comm,cp,csplit,cut,date,df,dircolors,dirname,du,echo,env,expand,expr,factor,false,fmt,fold,groups,hashsum,head,hostid,hostname,id,install,join,kill,link,ln,logname,ls,mkdir,mkfifo,mknod,mktemp,more,mv,nice,nl,nohup,nproc,numfmt,od,paste,pathchk,pinky,printenv,printf,ptx,pwd,readlink,realpath,relpath,rm,rmdir,seq,shred,shuf,sleep,sort,split,stat,stdbuf,sum,sync,tac,tail,tee,test,timeout,touch,tr,true,truncate,tsort,tty,uname,unexpand,uniq,unlink,uptime,users,wc,who,whoami,yes,chcon,pr,dir,vdir,dd,basenc,runcon -aarch64-unknown-linux-gnu,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -i686-unknown-linux-gnu,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -powerpc64-unknown-linux-gnu,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -riscv64gc-unknown-linux-gnu,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 -x86_64-unknown-linux-gnu,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -aarch64-pc-windows-msvc,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,101,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,101,0,101,0,101,101,0,0,0,0,0,0,0,0 -i686-pc-windows-gnu,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,0,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,101,0,101,0,0,0,0,0,0,0,0,0 -i686-pc-windows-msvc,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,0,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,101,0,101,0,101,0,0,0,0,0,0,0,0,0 -x86_64-pc-windows-gnu,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,0,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,101,0,101,0,0,0,0,0,0,0,0,0 -x86_64-pc-windows-msvc,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,0,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,101,0,101,0,101,0,0,0,0,0,0,0,0,0 -x86_64-apple-darwin,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -x86_64-unknown-freebsd,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -x86_64-unknown-netbsd,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,101,101,0,101,0,0,0,0,0,0,0,0,0 -aarch64-linux-android,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,101,101,0,101,0,0,0,0,0,0,0,0,0 -x86_64-linux-android,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,101,101,0,101,0,0,0,0,0,0,0,0,0 -x86_64-sun-solaris,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 -wasm32-wasi,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 -x86_64-unknown-redox,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 -aarch64-fuchsia,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 -x86_64-fuchsia,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 +target,arch,base32,base64,basename,cat,chgrp,chmod,chown,chroot,cksum,comm,cp,csplit,cut,date,df,dircolors,dirname,du,echo,env,expand,expr,factor,false,fmt,fold,groups,hashsum,head,hostid,hostname,id,install,join,kill,link,ln,logname,ls,mkdir,mkfifo,mknod,mktemp,more,mv,nice,nl,nohup,nproc,numfmt,od,paste,pathchk,pinky,printenv,printf,ptx,pwd,readlink,realpath,rm,rmdir,seq,shred,shuf,sleep,sort,split,stat,stdbuf,sum,sync,tac,tail,tee,test,timeout,touch,tr,true,truncate,tsort,tty,uname,unexpand,uniq,unlink,uptime,users,wc,who,whoami,yes,chcon,pr,dir,vdir,dd,basenc,runcon +aarch64-unknown-linux-gnu,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +i686-unknown-linux-gnu,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +powerpc64-unknown-linux-gnu,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +riscv64gc-unknown-linux-gnu,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 +x86_64-unknown-linux-gnu,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +aarch64-pc-windows-msvc,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,101,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,101,0,101,0,101,101,0,0,0,0,0,0,0,0 +i686-pc-windows-gnu,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,0,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,101,0,101,0,0,0,0,0,0,0,0,0 +i686-pc-windows-msvc,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,0,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,101,0,101,0,101,0,0,0,0,0,0,0,0,0 +x86_64-pc-windows-gnu,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,0,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,101,0,101,0,0,0,0,0,0,0,0,0 +x86_64-pc-windows-msvc,0,0,0,0,0,101,101,101,101,0,0,0,0,0,0,0,0,0,101,0,0,0,101,0,0,0,0,101,0,0,0,0,101,101,0,101,0,0,0,0,0,101,101,0,0,0,101,0,101,0,0,0,0,101,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,101,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,101,0,101,0,101,0,0,0,0,0,0,0,0,0 +x86_64-apple-darwin,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +x86_64-unknown-freebsd,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +x86_64-unknown-netbsd,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,101,101,0,101,0,0,0,0,0,0,0,0,0 +aarch64-linux-android,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,101,101,0,101,0,0,0,0,0,0,0,0,0 +x86_64-linux-android,0,0,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,0,0,0,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,0,0,101,0,0,0,0,101,0,0,0,0,0,0,101,0,0,0,101,101,0,101,0,0,0,0,0,0,0,0,0 +x86_64-sun-solaris,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 +wasm32-wasi,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 +x86_64-unknown-redox,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 +aarch64-fuchsia,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 +x86_64-fuchsia,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101 diff --git a/docs/compiles_table.py b/docs/compiles_table.py index 83c98bed1de..884b0bc39e1 100644 --- a/docs/compiles_table.py +++ b/docs/compiles_table.py @@ -10,7 +10,7 @@ # third party dependencies from tqdm import tqdm -# spell-checker:ignore (libs) tqdm imap ; (shell/mac) xcrun ; (vars) nargs +# spell-checker:ignore (libs) tqdm imap ; (shell/mac) xcrun ; (vars) nargs retcode csvfile BINS_PATH = Path("../src/uu") CACHE_PATH = Path("compiles_table.csv") diff --git a/docs/src/CODE_OF_CONDUCT.md b/docs/src/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..ce326a1ee03 --- /dev/null +++ b/docs/src/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ + + +{{ #include ../../CODE_OF_CONDUCT.md }} diff --git a/docs/src/contributing.md b/docs/src/CONTRIBUTING.md similarity index 100% rename from docs/src/contributing.md rename to docs/src/CONTRIBUTING.md diff --git a/docs/src/DEVELOPMENT.md b/docs/src/DEVELOPMENT.md new file mode 100644 index 00000000000..580cecf0855 --- /dev/null +++ b/docs/src/DEVELOPMENT.md @@ -0,0 +1,3 @@ + + +{{ #include ../../DEVELOPMENT.md }} diff --git a/docs/src/extensions.md b/docs/src/extensions.md index eeb00ff350e..af6119da51a 100644 --- a/docs/src/extensions.md +++ b/docs/src/extensions.md @@ -1,3 +1,5 @@ + + # Extensions over GNU Though the main goal of the project is compatibility, uutils supports a few @@ -22,8 +24,13 @@ $ ls -w=80 ## `env` -`env` has an additional `-f`/`--file` flag that can parse `.env` files and set -variables accordingly. This feature is adopted from `dotenv` style packages. +GNU `env` allows the empty string to be used as an environment variable name. +This is unsupported by uutils, and it will show a warning on any such +assignment. + + `env` has an additional `-f`/`--file` flag that can +parse `.env` files and set variables accordingly. This feature is adopted from `dotenv` style +packages. ## `cp` @@ -66,8 +73,84 @@ feature is adopted from [FreeBSD](https://www.freebsd.org/cgi/man.cgi?cut). mail headers in the input. `-q`/`--quick` breaks lines more quickly. And `-T`/`--tab-width` defines the number of spaces representing a tab when determining the line length. +## `printf` + +`printf` uses arbitrary precision decimal numbers to parse and format floating point +numbers. GNU coreutils uses `long double`, whose actual size may be [double precision +64-bit float](https://en.wikipedia.org/wiki/Double-precision_floating-point_format) +(e.g 32-bit arm), [extended precision 80-bit float](https://en.wikipedia.org/wiki/Extended_precision) +(x86(-64)), or +[quadruple precision 128-bit float](https://en.wikipedia.org/wiki/Quadruple-precision_floating-point_format) (e.g. arm64). + +Practically, this means that printing a number with a large precision will stay exact: +``` +printf "%.48f\n" 0.1 +0.100000000000000000000000000000000000000000000000 << uutils on all platforms +0.100000000000000000001355252715606880542509316001 << GNU coreutils on x86(-64) +0.100000000000000000000000000000000004814824860968 << GNU coreutils on arm64 +0.100000000000000005551115123125782702118158340454 << GNU coreutils on armv7 (32-bit) +``` + +### Hexadecimal floats + +For hexadecimal float format (`%a`), POSIX only states that one hexadecimal number +should be present left of the decimal point (`0xh.hhhhp±d` [1]), but does not say how +many _bits_ should be included (between 1 and 4). On x86(-64), the first digit always +includes 4 bits, so its value is always between `0x8` and `0xf`, while on other +architectures, only 1 bit is included, so the value is always `0x1`. + +However, the first digit will of course be `0x0` if the number is zero. Also, +rounding numbers may cause the first digit to be `0x1` on x86(-64) (e.g. +`0xf.fffffffp-5` rounds to `0x1.00p-1`), or `0x2` on other architectures. + +We chose to replicate x86-64 behavior on all platforms. + +Additionally, the default precision of the hexadecimal float format (`%a` without +any specifier) is expected to be "sufficient for exact representation of the value" [1]. +This is not possible in uutils as we store arbitrary precision numbers that may be +periodic in hexadecimal form (`0.1 = 0xc.ccc...p-7`), so we revert +to the number of digits that would be required to exactly print an +[extended precision 80-bit float](https://en.wikipedia.org/wiki/Extended_precision), +emulating GNU coreutils behavior on x86(-64). An 80-bit float has 64 bits in its +integer and fractional part, so 16 hexadecimal digits are printed in total (1 digit +before the decimal point, 15 after). + +Practically, this means that the default hexadecimal floating point output is +identical to x86(-64) GNU coreutils: +``` +printf "%a\n" 0.1 +0xc.ccccccccccccccdp-7 << uutils on all platforms +0xc.ccccccccccccccdp-7 << GNU coreutils on x86-64 +0x1.999999999999999999999999999ap-4 << GNU coreutils on arm64 +0x1.999999999999ap-4 << GNU coreutils on armv7 (32-bit) +``` + +We _can_ print an arbitrary number of digits if a larger precision is requested, +and the leading digit will still be in the `0x8`-`0xf` range: +``` +printf "%.32a\n" 0.1 +0xc.cccccccccccccccccccccccccccccccdp-7 << uutils on all platforms +0xc.ccccccccccccccd00000000000000000p-7 << GNU coreutils on x86-64 +0x1.999999999999999999999999999a0000p-4 << GNU coreutils on arm64 +0x1.999999999999a0000000000000000000p-4 << GNU coreutils on armv7 (32-bit) +``` + +***Note: The architecture-specific behavior on non-x86(-64) platforms may change in +the future.*** + ## `seq` +Unlike GNU coreutils, `seq` always uses arbitrary precision decimal numbers, no +matter the parameters (integers, decimal numbers, positive or negative increments, +format specified, etc.), so its output will be more correct than GNU coreutils for +some inputs (e.g. small fractional increments where GNU coreutils uses `long double`). + +The only limitation is that the position of the decimal point is stored in a `i64`, +so values smaller than 10**(-2**63) will underflow to 0, and some values larger +than 10**(2**63) may overflow to infinity. + +See also comments under `printf` for formatting precision and differences. + `seq` provides `-t`/`--terminator` to set the terminator character. ## `ls` @@ -75,6 +158,29 @@ number of spaces representing a tab when determining the line length. GNU `ls` provides two ways to use a long listing format: `-l` and `--format=long`. We support a third way: `--long`. +GNU `ls --sort=VALUE` only supports special non-default sort orders. +We support `--sort=name`, which makes it possible to override an earlier value. + ## `du` -`du` allows `birth` and `creation` as values for the `--time` argument to show the creation time. +`du` allows `birth` and `creation` as values for the `--time` argument to show the creation time. It +also provides a `-v`/`--verbose` flag. + +## `id` + +`id` has three additional flags: +* `-P` displays the id as a password file entry +* `-p` makes the output human-readable +* `-A` displays the process audit user ID + +## `uptime` + +Similar to the proc-ps implementation and unlike GNU/Coreutils, `uptime` provides `-s`/`--since` to show since when the system is up. + +## `base32/base64/basenc` + +Just like on macOS, `base32/base64/basenc` provides `-D` to decode data. + +## `shred` + +The number of random passes is deterministic in both GNU and uutils. However, uutils `shred` computes the number of random passes in a simplified way, specifically `max(3, x / 10)`, which is very close but not identical to the number of random passes that GNU would do. This also satisfies an expectation that reasonable users might have, namely that the number of random passes increases monotonically with the number of passes overall; GNU `shred` violates this assumption. diff --git a/docs/src/installation.md b/docs/src/installation.md index da124ead977..856ca9d22f5 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -1,4 +1,4 @@ - + # Installation @@ -6,21 +6,21 @@ This is a list of uutils packages in various distributions and package managers. Note that these are packaged by third-parties and the packages might contain patches. -You can also [build uutils from source](/build.md). +You can also [build uutils from source](build.md). ## Cargo -[![crates.io package](https://repology.org/badge/version-for-repo/crates_io/uutils-coreutils.svg)](https://repology.org/project/uutils-coreutils/versions) +[![crates.io package](https://repology.org/badge/version-for-repo/crates_io/uutils-coreutils.svg)](https://crates.io/crates/coreutils) ```shell # Linux -cargo install coreutils --features unix +cargo install coreutils --features unix --locked # MacOs -cargo install coreutils --features macos +cargo install coreutils --features macos --locked # Windows -cargo install coreutils --features windows +cargo install coreutils --features windows --locked ``` ## Linux @@ -53,7 +53,15 @@ apt install rust-coreutils export PATH=/usr/lib/cargo/bin/coreutils:$PATH ``` -> **Note**: Only available from Bookworm (Debian 12) +### Fedora + +[![Fedora package](https://repology.org/badge/version-for-repo/fedora_rawhide/uutils-coreutils.svg)](https://packages.fedoraproject.org/pkgs/rust-coreutils/uutils-coreutils) + +```shell +dnf install uutils-coreutils +# To use it: +export PATH=/usr/libexec/uutils-coreutils:$PATH +``` ### Gentoo @@ -65,9 +73,9 @@ emerge -pv sys-apps/uutils-coreutils ### Manjaro -![Manjaro Stable package](https://repology.org/badge/version-for-repo/manjaro_stable/uutils-coreutils.svg) -[![Manjaro Testing package](https://repology.org/badge/version-for-repo/manjaro_testing/uutils-coreutils.svg)](https://repology.org/project/uutils-coreutils/versions) -[![Manjaro Unstable package](https://repology.org/badge/version-for-repo/manjaro_unstable/uutils-coreutils.svg)](https://repology.org/project/uutils-coreutils/versions) +[![Manjaro Stable package](https://repology.org/badge/version-for-repo/manjaro_stable/uutils-coreutils.svg)](https://packages.manjaro.org/?query=uutils-coreutils) +[![Manjaro Testing package](https://repology.org/badge/version-for-repo/manjaro_testing/uutils-coreutils.svg)](https://packages.manjaro.org/?query=uutils-coreutils) +[![Manjaro Unstable package](https://repology.org/badge/version-for-repo/manjaro_unstable/uutils-coreutils.svg)](https://packages.manjaro.org/?query=uutils-coreutils) ```shell pacman -S uutils-coreutils @@ -77,7 +85,7 @@ pamac install uutils-coreutils ### NixOS -[![nixpkgs unstable package](https://repology.org/badge/version-for-repo/nix_unstable/uutils-coreutils.svg)](https://repology.org/project/uutils-coreutils/versions) +[![nixpkgs unstable package](https://repology.org/badge/version-for-repo/nix_unstable/uutils-coreutils.svg)](https://search.nixos.org/packages?query=uutils-coreutils) ```shell nix-env -iA nixos.uutils-coreutils @@ -91,9 +99,22 @@ nix-env -iA nixos.uutils-coreutils dnf install uutils-coreutils ``` +### RHEL/AlmaLinux/CENTOS Stream/Rocky Linux/EPEL 9 + +[![epel 9 package](https://repology.org/badge/version-for-repo/epel_9/uutils-coreutils.svg)](https://packages.fedoraproject.org/pkgs/rust-coreutils/uutils-coreutils/epel-9.html) + +```shell +# Install EPEL 9 - Specific For RHEL please check codeready-builder-for-rhel-9 First then install epel +dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm -y +# Install Core Utils +dnf install uutils-coreutils +# To use it: +export PATH=/usr/libexec/uutils-coreutils:$PATH +``` + ### Ubuntu -[![Ubuntu package](https://repology.org/badge/version-for-repo/ubuntu_23_04/uutils-coreutils.svg)](https://packages.ubuntu.com/source/lunar/rust-coreutils) +[![Ubuntu package](https://repology.org/badge/version-for-repo/ubuntu_25_04/uutils-coreutils.svg)](https://packages.ubuntu.com/source/plucky/rust-coreutils) ```shell apt install rust-coreutils @@ -101,8 +122,6 @@ apt install rust-coreutils export PATH=/usr/lib/cargo/bin/coreutils:$PATH ``` -> **Note**: Only available from Kinetic (Ubuntu 22.10) - ## MacOS ### Homebrew @@ -123,7 +142,7 @@ port install coreutils-uutils ## FreeBSD -[![FreeBSD port](https://repology.org/badge/version-for-repo/freebsd/uutils-coreutils.svg)](https://repology.org/project/uutils-coreutils/versions) +[![FreeBSD port](https://repology.org/badge/version-for-repo/freebsd/rust-coreutils.svg)](https://repology.org/project/rust-coreutils/versions) ```sh pkg install rust-coreutils @@ -131,9 +150,15 @@ pkg install rust-coreutils ## Windows +### Winget + +```shell +winget install uutils.coreutils +``` + ### Scoop -[![Scoop package](https://repology.org/badge/version-for-repo/scoop/uutils-coreutils.svg)](https://scoop.sh/#/apps?q=uutils-coreutils&s=0&d=1&o=true) +[Scoop package](https://scoop.sh/#/apps?q=uutils-coreutils&s=0&d=1&o=true) ```shell scoop install uutils-coreutils @@ -146,15 +171,24 @@ scoop install uutils-coreutils [Conda package](https://anaconda.org/conda-forge/uutils-coreutils) ``` -conda install -c conda-forge uutils-coreutils +conda install -c conda-forge uutils-coreutils ``` +### Yocto + +[Yocto recipe](https://github.com/openembedded/meta-openembedded/tree/master/meta-oe/recipes-core/uutils-coreutils) + +The uutils-coreutils recipe is provided as part of the meta-openembedded yocto layer. +Clone [poky](https://github.com/yoctoproject/poky) and [meta-openembedded](https://github.com/openembedded/meta-openembedded/tree/master), add +`meta-openembedded/meta-oe` as layer in your `build/conf/bblayers.conf` file, +and then either call `bitbake uutils-coreutils`, or use +`PREFERRED_PROVIDER_coreutils = "uutils-coreutils"` in your `build/conf/local.conf` file and +then build your usual yocto image. + ## Non-standard packages -### `coreutils-hybrid` (AUR) +### `coreutils-uutils` (AUR) -[![AUR package](https://repology.org/badge/version-for-repo/aur/coreutils-hybrid.svg)](https://aur.archlinux.org/packages/coreutils-hybrid) +[AUR package](https://aur.archlinux.org/packages/coreutils-uutils) -A GNU coreutils / uutils coreutils hybrid package. Uses stable uutils -programs mixed with GNU counterparts if uutils counterpart is -unfinished or buggy. +Cross-platform Rust rewrite of the GNU coreutils being used as actual system coreutils. diff --git a/docs/src/performance.md b/docs/src/performance.md new file mode 100644 index 00000000000..10c7b383447 --- /dev/null +++ b/docs/src/performance.md @@ -0,0 +1,108 @@ + + +# Performance Profiling Tutorial + +## Effective Benchmarking with Hyperfine + +[Hyperfine](https://github.com/sharkdp/hyperfine) is a powerful command-line benchmarking tool that allows you to measure and compare execution times of commands with statistical rigor. + +### Benchmarking Best Practices + +When evaluating performance improvements, always set up your benchmarks to compare: + +1. The GNU implementation as reference +2. The implementation without the change +3. The implementation with your change + +This three-way comparison provides clear insights into: +- How your implementation compares to the standard (GNU) +- The actual performance impact of your specific change + +### Example Benchmark + +First, you will need to build the binary in release mode. Debug builds are significantly slower: + +```bash +cargo build --features unix --profile profiling +``` + +```bash +# Three-way comparison benchmark +hyperfine \ + --warmup 3 \ + "/usr/bin/ls -R ." \ + "./target/profiling/coreutils.prev ls -R ." \ + "./target/profiling/coreutils ls -R ." + +# can be simplified with: +hyperfine \ + --warmup 3 \ + -L ls /usr/bin/ls,"./target/profiling/coreutils.prev ls","./target/profiling/coreutils ls" \ + "{ls} -R ." +``` + +``` +# to improve the reproducibility of the results: +taskset -c 0 +``` + +### Interpreting Results + +Hyperfine provides summary statistics including: +- Mean execution time +- Standard deviation +- Min/max times +- Relative performance comparison + +Look for consistent patterns rather than focusing on individual runs, and be aware of system noise that might affect results. + +## Using Samply for Profiling + +[Samply](https://github.com/mstange/samply) is a sampling profiler that helps you identify performance bottlenecks in your code. + +### Basic Profiling + +```bash +# Generate a flame graph for your application +samply record ./target/debug/coreutils ls -R + +# Profile with higher sampling frequency +samply record --rate 1000 ./target/debug/coreutils seq 1 1000 +``` + +The output using the `debug` profile might be easier to understand, but the performance characteristics may be somewhat different from `release` profile that we _actually_ care about. + +Consider using the `profiling` profile, that compiles in `release` mode but with debug symbols. For example: +```bash +cargo build --profile profiling -p uu_ls +samply record -r 10000 target/profiling/ls -lR /var .git .git .git > /dev/null +``` + +## Workflow: Measuring Performance Improvements + +1. **Establish baselines**: + ```bash + hyperfine --warmup 3 \ + "/usr/bin/sort large_file.txt" \ + "our-sort-v1 large_file.txt" + ``` + +2. **Identify bottlenecks**: + ```bash + samply record ./our-sort-v1 large_file.txt + ``` + +3. **Make targeted improvements** based on profiling data + +4. **Verify improvements**: + ```bash + hyperfine --warmup 3 \ + "/usr/bin/sort large_file.txt" \ + "our-sort-v1 large_file.txt" \ + "our-sort-v2 large_file.txt" + ``` + +5. **Document performance changes** with concrete numbers + ```bash + hyperfine --export-markdown file.md [...] + ``` diff --git a/docs/src/test_coverage.js b/docs/src/test_coverage.js index e601229affc..318c9934d53 100644 --- a/docs/src/test_coverage.js +++ b/docs/src/test_coverage.js @@ -19,7 +19,7 @@ function progressBar(totals) { var(--SKIP) ${skipPercentage}%` ) + (skipPercentage === 100 ? ")" : ", var(--FAIL) 0)"); - + const progress = document.createElement("div"); progress.className = "progress" progress.innerHTML = ` @@ -74,7 +74,7 @@ function parse_result(parent, obj) { return totals; } -fetch("https://raw.githubusercontent.com/uutils/coreutils-tracking/main/gnu-full-result.json") +fetch("https://raw.githubusercontent.com/uutils/coreutils-tracking/main/aggregated-result.json") .then((r) => r.json()) .then((obj) => { let parent = document.getElementById("test-cov"); diff --git a/docs/src/test_coverage.md b/docs/src/test_coverage.md index b8376058873..2bfad68bcac 100644 --- a/docs/src/test_coverage.md +++ b/docs/src/test_coverage.md @@ -18,4 +18,4 @@ or resulted in an error. ## Progress over time - + diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000000..fbf85d3df31 --- /dev/null +++ b/flake.lock @@ -0,0 +1,42 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1720633750, + "narHash": "sha256-N8apMO2pP/upWeH+JY5eM8VDp2qBAAzE+OY5LRW6qpw=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "54bc082f5a7219d122e74fe52c021cf59fed9d6f", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "systems": "systems" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000000..9c99b7b72e1 --- /dev/null +++ b/flake.nix @@ -0,0 +1,75 @@ +# spell-checker:ignore bintools gnum gperf ldflags libclang nixpkgs numtide pkgs texinfo gettext +{ + inputs = { + nixpkgs.url = "github:nixos/nixpkgs"; + + # + systems.url = "github:nix-systems/default"; + }; + + outputs = inputs: let + inherit (inputs.nixpkgs) lib legacyPackages; + eachSystem = lib.genAttrs (import inputs.systems); + pkgsFor = legacyPackages; + in { + devShells = eachSystem ( + system: let + libselinuxPath = with pkgsFor.${system}; + lib.makeLibraryPath [ + libselinux + ]; + + libaclPath = with pkgsFor.${system}; + lib.makeLibraryPath [ + acl + ]; + + build_deps = with pkgsFor.${system}; [ + clang + llvmPackages.bintools + rustup + + pre-commit + nodePackages.cspell + + # debugging + gdb + ]; + + gnu_testing_deps = with pkgsFor.${system}; [ + autoconf + automake + bison + gnum4 + gperf + gettext + texinfo + ]; + in { + default = pkgsFor.${system}.pkgs.mkShell { + packages = build_deps ++ gnu_testing_deps; + + RUSTC_VERSION = "1.85"; + LIBCLANG_PATH = pkgsFor.${system}.lib.makeLibraryPath [pkgsFor.${system}.llvmPackages_latest.libclang.lib]; + shellHook = '' + export PATH=$PATH:''${CARGO_HOME:-~/.cargo}/bin + export PATH=$PATH:''${RUSTUP_HOME:-~/.rustup}/toolchains/$RUSTC_VERSION-x86_64-unknown-linux-gnu/bin/ + ''; + + SELINUX_INCLUDE_DIR = ''${pkgsFor.${system}.libselinux.dev}/include''; + SELINUX_LIB_DIR = libselinuxPath; + SELINUX_STATIC = "0"; + + # Necessary to build GNU. + LDFLAGS = ''-L ${libselinuxPath} -L ${libaclPath}''; + + # Add precompiled library to rustc search path + RUSTFLAGS = [ + ''-L ${libselinuxPath}'' + ''-L ${libaclPath}'' + ]; + }; + } + ); + }; +} diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock new file mode 100644 index 00000000000..4495a0a5a60 --- /dev/null +++ b/fuzz/Cargo.lock @@ -0,0 +1,1804 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bigdecimal" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "binary-heap-plus" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4551d8382e911ecc0d0f0ffb602777988669be09447d536ff4388d1def11296" +dependencies = [ + "compare", +] + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "blake2b_simd" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06e903a20b159e944f91ec8499fe1e55651480c541ea0a584f5d967c49ad9d99" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "blake3" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + +[[package]] +name = "cc" +version = "1.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efdce149c370f133a071ca8ef6ea340b7b88748ab0810097a9e2976eaa34b4f3" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f10f8c9340e31fc120ff885fcdb54a0b48e474bbd77cab557f0c30a3e569402" +dependencies = [ + "parse-zoneinfo", + "phf_codegen", +] + +[[package]] +name = "clap" +version = "4.5.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "compare" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120133d4db2ec47efe2e26502ee984747630c67f51974fca0b6c1340cf2368d3" + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73" +dependencies = [ + "nix", + "windows-sys", +] + +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + +[[package]] +name = "data-encoding-macro" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47ce6c96ea0102f01122a185683611bd5ac8d99e62bc59dd12e6bda344ee673d" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "errno" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fluent" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8137a6d5a2c50d6b0ebfcb9aaa91a28154e0a70605f112d30cb0cd4a78670477" +dependencies = [ + "fluent-bundle", + "unic-langid", +] + +[[package]] +name = "fluent-bundle" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01203cb8918f5711e73891b347816d932046f95f54207710bda99beaeb423bf4" +dependencies = [ + "fluent-langneg", + "fluent-syntax", + "intl-memoizer", + "intl_pluralrules", + "rustc-hash 2.1.1", + "self_cell", + "smallvec", + "unic-langid", +] + +[[package]] +name = "fluent-langneg" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ad0989667548f06ccd0e306ed56b61bd4d35458d54df5ec7587c0e8ed5e94" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "fluent-syntax" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54f0d287c53ffd184d04d8677f590f4ac5379785529e5e08b1c8083acdd5c198" +dependencies = [ + "memchr", + "thiserror", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "intl-memoizer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "310da2e345f5eb861e7a07ee182262e94975051db9e4223e909ba90f392f163f" +dependencies = [ + "type-map", + "unic-langid", +] + +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "onig" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown", +] + +[[package]] +name = "os_display" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad5fd71b79026fb918650dde6d125000a233764f1c2f1659a1c71118e33ea08f" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + +[[package]] +name = "parse_datetime" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd3830b49ee3a0dcc8fdfadc68c6354c97d00101ac1cac5b2eee25d35c42066" +dependencies = [ + "chrono", + "nom", + "regex", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rust-ini" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" +dependencies = [ + "cfg-if", + "ordered-multimap", + "trim-in-place", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "self_cell" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + +[[package]] +name = "sm3" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb9a3b702d0a7e33bc4d85a14456633d2b165c2ad839c5fd9a8417c1ab15860" +dependencies = [ + "digest", +] + +[[package]] +name = "smallvec" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "terminal_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" +dependencies = [ + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "trim-in-place" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" + +[[package]] +name = "type-map" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb68604048ff8fa93347f02441e4487594adc20bb8a084f9e564d2b827a0a9f" +dependencies = [ + "rustc-hash 1.1.0", +] + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "tinystr", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uu_cksum" +version = "0.1.0" +dependencies = [ + "clap", + "hex", + "regex", + "uucore", +] + +[[package]] +name = "uu_cut" +version = "0.1.0" +dependencies = [ + "bstr", + "clap", + "memchr", + "uucore", +] + +[[package]] +name = "uu_date" +version = "0.1.0" +dependencies = [ + "chrono", + "clap", + "libc", + "parse_datetime", + "uucore", + "windows-sys", +] + +[[package]] +name = "uu_echo" +version = "0.1.0" +dependencies = [ + "clap", + "uucore", +] + +[[package]] +name = "uu_env" +version = "0.1.0" +dependencies = [ + "clap", + "nix", + "rust-ini", + "thiserror", + "uucore", +] + +[[package]] +name = "uu_expr" +version = "0.1.0" +dependencies = [ + "clap", + "num-bigint", + "num-traits", + "onig", + "thiserror", + "uucore", +] + +[[package]] +name = "uu_printf" +version = "0.1.0" +dependencies = [ + "clap", + "uucore", +] + +[[package]] +name = "uu_seq" +version = "0.1.0" +dependencies = [ + "bigdecimal", + "clap", + "num-bigint", + "num-traits", + "thiserror", + "uucore", +] + +[[package]] +name = "uu_sort" +version = "0.1.0" +dependencies = [ + "binary-heap-plus", + "clap", + "compare", + "ctrlc", + "fnv", + "itertools", + "memchr", + "nix", + "rand 0.9.1", + "rayon", + "self_cell", + "tempfile", + "thiserror", + "unicode-width", + "uucore", +] + +[[package]] +name = "uu_split" +version = "0.1.0" +dependencies = [ + "clap", + "memchr", + "thiserror", + "uucore", +] + +[[package]] +name = "uu_test" +version = "0.1.0" +dependencies = [ + "clap", + "libc", + "uucore", +] + +[[package]] +name = "uu_tr" +version = "0.1.0" +dependencies = [ + "clap", + "nom", + "uucore", +] + +[[package]] +name = "uu_wc" +version = "0.1.0" +dependencies = [ + "bytecount", + "clap", + "libc", + "nix", + "thiserror", + "unicode-width", + "uucore", +] + +[[package]] +name = "uucore" +version = "0.1.0" +dependencies = [ + "bigdecimal", + "blake2b_simd", + "blake3", + "chrono", + "chrono-tz", + "clap", + "crc32fast", + "data-encoding", + "data-encoding-macro", + "digest", + "dunce", + "fluent", + "fluent-bundle", + "glob", + "hex", + "iana-time-zone", + "itertools", + "libc", + "md-5", + "memchr", + "nix", + "num-traits", + "number_prefix", + "os_display", + "sha1", + "sha2", + "sha3", + "sm3", + "thiserror", + "unic-langid", + "uucore_procs", + "wild", + "winapi-util", + "windows-sys", + "z85", +] + +[[package]] +name = "uucore-fuzz" +version = "0.0.0" +dependencies = [ + "libfuzzer-sys", + "rand 0.9.1", + "uu_cksum", + "uu_cut", + "uu_date", + "uu_echo", + "uu_env", + "uu_expr", + "uu_printf", + "uu_seq", + "uu_sort", + "uu_split", + "uu_test", + "uu_tr", + "uu_wc", + "uucore", + "uufuzz", +] + +[[package]] +name = "uucore_procs" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "uuhelp_parser", +] + +[[package]] +name = "uufuzz" +version = "0.1.0" +dependencies = [ + "console", + "libc", + "rand 0.9.1", + "similar", + "tempfile", + "uucore", +] + +[[package]] +name = "uuhelp_parser" +version = "0.1.0" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wild" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3131afc8c575281e1e80f36ed6a092aa502c08b18ed7524e86fbbb12bb410e1" +dependencies = [ + "glob", +] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "z85" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3a41ce106832b4da1c065baa4c31cf640cf965fa1483816402b7f6b96f0a64" + +[[package]] +name = "zerocopy" +version = "0.8.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "zerofrom", +] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 91a85b45a0f..48da8e846b4 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -1,21 +1,36 @@ [package] name = "uucore-fuzz" version = "0.0.0" +description = "uutils ~ 'core' uutils fuzzers" +repository = "https://github.com/uutils/coreutils/tree/main/fuzz/" +edition.workspace = true publish = false -edition = "2021" + +[workspace.package] +edition = "2024" +license = "MIT" [package.metadata] cargo-fuzz = true [dependencies] -libfuzzer-sys = "0.4" -libc = "0.2" -rand = { version = "0.8", features = ["small_rng"] } - -uucore = { path = "../src/uucore/" } +libfuzzer-sys = "0.4.7" +rand = { version = "0.9.0", features = ["small_rng"] } +uufuzz = { path = "uufuzz/" } +uucore = { path = "../src/uucore/", features = ["parser"] } uu_date = { path = "../src/uu/date/" } uu_test = { path = "../src/uu/test/" } - +uu_expr = { path = "../src/uu/expr/" } +uu_printf = { path = "../src/uu/printf/" } +uu_echo = { path = "../src/uu/echo/" } +uu_seq = { path = "../src/uu/seq/" } +uu_sort = { path = "../src/uu/sort/" } +uu_wc = { path = "../src/uu/wc/" } +uu_cut = { path = "../src/uu/cut/" } +uu_split = { path = "../src/uu/split/" } +uu_tr = { path = "../src/uu/tr/" } +uu_env = { path = "../src/uu/env/" } +uu_cksum = { path = "../src/uu/cksum/" } # Prevent this from interfering with workspaces [workspace] @@ -27,12 +42,66 @@ path = "fuzz_targets/fuzz_date.rs" test = false doc = false +[[bin]] +name = "fuzz_printf" +path = "fuzz_targets/fuzz_printf.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_echo" +path = "fuzz_targets/fuzz_echo.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_seq" +path = "fuzz_targets/fuzz_seq.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_sort" +path = "fuzz_targets/fuzz_sort.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_split" +path = "fuzz_targets/fuzz_split.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_cut" +path = "fuzz_targets/fuzz_cut.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_wc" +path = "fuzz_targets/fuzz_wc.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_expr" +path = "fuzz_targets/fuzz_expr.rs" +test = false +doc = false + [[bin]] name = "fuzz_test" path = "fuzz_targets/fuzz_test.rs" test = false doc = false +[[bin]] +name = "fuzz_seq_parse_number" +path = "fuzz_targets/fuzz_seq_parse_number.rs" +test = false +doc = false + [[bin]] name = "fuzz_parse_glob" path = "fuzz_targets/fuzz_parse_glob.rs" @@ -50,3 +119,21 @@ name = "fuzz_parse_time" path = "fuzz_targets/fuzz_parse_time.rs" test = false doc = false + +[[bin]] +name = "fuzz_tr" +path = "fuzz_targets/fuzz_tr.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_env" +path = "fuzz_targets/fuzz_env.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_cksum" +path = "fuzz_targets/fuzz_cksum.rs" +test = false +doc = false diff --git a/fuzz/fuzz_targets/fuzz_cksum.rs b/fuzz/fuzz_targets/fuzz_cksum.rs new file mode 100644 index 00000000000..be93a96050e --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_cksum.rs @@ -0,0 +1,170 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore chdir + +#![no_main] +use libfuzzer_sys::fuzz_target; +use rand::Rng; +use std::env::temp_dir; +use std::ffi::OsString; +use std::fs::{self, File}; +use std::io::Write; +use std::process::Command; +use uu_cksum::uumain; +use uufuzz::{ + CommandResult, compare_result, generate_and_run_uumain, generate_random_file, + generate_random_string, + pretty_print::{print_or_empty, print_test_begin}, + replace_fuzz_binary_name, run_gnu_cmd, +}; + +static CMD_PATH: &str = "cksum"; + +fn generate_cksum_args() -> Vec { + let mut rng = rand::rng(); + let mut args = Vec::new(); + + let digests = [ + "sysv", "bsd", "crc", "md5", "sha1", "sha224", "sha256", "sha384", "sha512", "blake2b", + "sm3", + ]; + let digest_opts = [ + "--base64", + "--raw", + "--tag", + "--untagged", + "--text", + "--binary", + ]; + + if rng.random_bool(0.3) { + args.push("-a".to_string()); + args.push(digests[rng.random_range(0..digests.len())].to_string()); + } + + if rng.random_bool(0.2) { + args.push(digest_opts[rng.random_range(0..digest_opts.len())].to_string()); + } + + if rng.random_bool(0.15) { + args.push("-l".to_string()); + args.push(rng.random_range(8..513).to_string()); + } + + if rng.random_bool(0.05) { + for _ in 0..rng.random_range(0..3) { + args.push(format!("file_{}", generate_random_string(5))); + } + } else { + args.push("-c".to_string()); + } + + if rng.random_bool(0.25) { + if let Ok(file_path) = generate_random_file() { + args.push(file_path); + } + } + + if args.is_empty() || !args.iter().any(|arg| arg.starts_with("file_")) { + args.push("-a".to_string()); + args.push(digests[rng.random_range(0..digests.len())].to_string()); + + if let Ok(file_path) = generate_random_file() { + args.push(file_path); + } + } + + args +} + +fn generate_checksum_file( + algo: &str, + file_path: &str, + digest_opts: &[&str], +) -> Result { + let checksum_file_path = temp_dir().join("checksum_file"); + let mut cmd = Command::new(CMD_PATH); + cmd.arg("-a").arg(algo); + + for opt in digest_opts { + cmd.arg(opt); + } + + cmd.arg(file_path); + let output = cmd.output()?; + + let mut checksum_file = File::create(&checksum_file_path)?; + checksum_file.write_all(&output.stdout)?; + + Ok(checksum_file_path.to_str().unwrap().to_string()) +} + +fn select_random_digest_opts<'a>( + rng: &mut rand::rngs::ThreadRng, + digest_opts: &'a [&'a str], +) -> Vec<&'a str> { + digest_opts + .iter() + .filter(|_| rng.random_bool(0.5)) + .copied() + .collect() +} + +fuzz_target!(|_data: &[u8]| { + let cksum_args = generate_cksum_args(); + let mut args = vec![OsString::from("cksum")]; + args.extend(cksum_args.iter().map(OsString::from)); + + if let Ok(file_path) = generate_random_file() { + let algo = cksum_args + .iter() + .position(|arg| arg == "-a") + .map_or("md5", |index| &cksum_args[index + 1]); + + let all_digest_opts = ["--base64", "--raw", "--tag", "--untagged"]; + let mut rng = rand::rng(); + let selected_digest_opts = select_random_digest_opts(&mut rng, &all_digest_opts); + + if let Ok(checksum_file_path) = + generate_checksum_file(algo, &file_path, &selected_digest_opts) + { + print_test_begin(format!("cksum {args:?}")); + + if let Ok(content) = fs::read_to_string(&checksum_file_path) { + println!("File content ({checksum_file_path})"); + print_or_empty(&content); + } else { + eprintln!("Error reading the checksum file."); + } + let mut rust_result = generate_and_run_uumain(&args, uumain, None); + + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + // Lower the number of false positives caused by binary names + replace_fuzz_binary_name("cksum", &mut rust_result); + + compare_result( + "cksum", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, + ); + } + } +}); diff --git a/fuzz/fuzz_targets/fuzz_cut.rs b/fuzz/fuzz_targets/fuzz_cut.rs new file mode 100644 index 00000000000..4a5215f8aec --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_cut.rs @@ -0,0 +1,86 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore parens + +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_cut::uumain; + +use rand::Rng; +use std::ffi::OsString; + +use uufuzz::{ + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, +}; +static CMD_PATH: &str = "cut"; + +fn generate_cut_args() -> String { + let mut rng = rand::rng(); + let arg_count = rng.random_range(1..=6); + let mut args = Vec::new(); + + for _ in 0..arg_count { + if rng.random_bool(0.1) { + args.push(generate_random_string(rng.random_range(1..=20))); + } else { + match rng.random_range(0..=4) { + 0 => args.push(String::from("-b") + &rng.random_range(1..=10).to_string()), + 1 => args.push(String::from("-c") + &rng.random_range(1..=10).to_string()), + 2 => args.push(String::from("-d,") + &generate_random_string(1)), // Using a comma as a default delimiter + 3 => args.push(String::from("-f") + &rng.random_range(1..=5).to_string()), + _ => (), + } + } + } + + args.join(" ") +} + +fn generate_delimited_data(count: usize) -> String { + let mut rng = rand::rng(); + let mut lines = Vec::new(); + + for _ in 0..count { + let fields = (0..rng.random_range(1..=5)) + .map(|_| generate_random_string(rng.random_range(1..=10))) + .collect::>() + .join(","); + lines.push(fields); + } + + lines.join("\n") +} + +fuzz_target!(|_data: &[u8]| { + let cut_args = generate_cut_args(); + let mut args = vec![OsString::from("cut")]; + args.extend(cut_args.split_whitespace().map(OsString::from)); + + let input_lines = generate_delimited_data(10); + + let rust_result = generate_and_run_uumain(&args, uumain, Some(&input_lines)); + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, Some(&input_lines)) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "cut", + &format!("{:?}", &args[1..]), + Some(&input_lines), + &rust_result, + &gnu_result, + false, // Set to true if you want to fail on stderr diff + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_date.rs b/fuzz/fuzz_targets/fuzz_date.rs index 96c56cc6bf2..0f9cb262c06 100644 --- a/fuzz/fuzz_targets/fuzz_date.rs +++ b/fuzz/fuzz_targets/fuzz_date.rs @@ -9,6 +9,6 @@ fuzz_target!(|data: &[u8]| { let args = data .split(|b| *b == delim) .filter_map(|e| std::str::from_utf8(e).ok()) - .map(|e| OsString::from(e)); + .map(OsString::from); uumain(args); }); diff --git a/fuzz/fuzz_targets/fuzz_echo.rs b/fuzz/fuzz_targets/fuzz_echo.rs new file mode 100644 index 00000000000..e6b0ba9a6aa --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_echo.rs @@ -0,0 +1,84 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_echo::uumain; + +use rand::Rng; +use rand::prelude::IndexedRandom; +use std::ffi::OsString; + +use uufuzz::CommandResult; +use uufuzz::{compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd}; + +static CMD_PATH: &str = "echo"; + +fn generate_echo() -> String { + let mut rng = rand::rng(); + let mut echo_str = String::new(); + + // Randomly decide whether to include options + let include_n = rng.random_bool(0.1); // 10% chance + let include_e = rng.random_bool(0.1); // 10% chance + #[allow(non_snake_case)] + let include_E = rng.random_bool(0.1); // 10% chance + + if include_n { + echo_str.push_str("-n "); + } + if include_e { + echo_str.push_str("-e "); + } + if include_E { + echo_str.push_str("-E "); + } + + // Add a random string + echo_str.push_str(&generate_random_string(rng.random_range(1..=10))); + + // Include escape sequences if -e is enabled + if include_e { + // Add a 10% chance of including an escape sequence + if rng.random_bool(0.1) { + echo_str.push_str(&generate_escape_sequence(&mut rng)); + } + } + + echo_str +} + +fn generate_escape_sequence(rng: &mut impl Rng) -> String { + let escape_sequences = [ + "\\\\", "\\a", "\\b", "\\c", "\\e", "\\f", "\\n", "\\r", "\\t", "\\v", "\\0NNN", "\\xHH", + ]; + // \0NNN and \xHH need more work + escape_sequences.choose(rng).unwrap().to_string() +} + +fuzz_target!(|_data: &[u8]| { + let echo_input = generate_echo(); + let mut args = vec![OsString::from("echo")]; + args.extend(echo_input.split_whitespace().map(OsString::from)); + let rust_result = generate_and_run_uumain(&args, uumain, None); + + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "echo", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + true, + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_env.rs b/fuzz/fuzz_targets/fuzz_env.rs new file mode 100644 index 00000000000..284089f8378 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_env.rs @@ -0,0 +1,96 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore chdir + +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_env::uumain; + +use std::ffi::OsString; + +use rand::Rng; +use uufuzz::{ + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, +}; + +static CMD_PATH: &str = "env"; + +fn generate_env_args() -> Vec { + let mut rng = rand::rng(); + let mut args = Vec::new(); + + let opts = ["-i", "-0", "-v", "-vv"]; + for opt in &opts { + if rng.random_bool(0.2) { + args.push(opt.to_string()); + } + } + + if rng.random_bool(0.3) { + args.push(format!( + "-u={}", + generate_random_string(rng.random_range(3..10)) + )); + } + + if rng.random_bool(0.2) { + args.push(format!("--chdir={}", "/tmp")); // Simplified example + } + + /* + Options not implemented for now + if rng.random_bool(0.15) { + let sig_opts = ["--block-signal"];//, /*"--default-signal",*/ "--ignore-signal"]; + let chosen_sig_opt = sig_opts[rng.random_range(0..sig_opts.len())]; + args.push(chosen_sig_opt.to_string()); + // Simplify by assuming SIGPIPE for demonstration + if !chosen_sig_opt.ends_with("list-signal-handling") { + args.push(String::from("SIGPIPE")); + } + }*/ + + // Adding a few random NAME=VALUE pairs + for _ in 0..rng.random_range(0..3) { + args.push(format!( + "{}={}", + generate_random_string(5), + generate_random_string(5) + )); + } + + args +} + +fuzz_target!(|_data: &[u8]| { + let env_args = generate_env_args(); + let mut args = vec![OsString::from("env")]; + args.extend(env_args.iter().map(OsString::from)); + let input_lines = generate_random_string(10); + + let rust_result = generate_and_run_uumain(&args, uumain, Some(&input_lines)); + + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "env", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_expr.rs b/fuzz/fuzz_targets/fuzz_expr.rs new file mode 100644 index 00000000000..77ecffabc1b --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_expr.rs @@ -0,0 +1,96 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore parens + +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_expr::uumain; + +use rand::Rng; +use rand::prelude::IndexedRandom; +use std::{env, ffi::OsString}; + +use uufuzz::CommandResult; +use uufuzz::{compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd}; +static CMD_PATH: &str = "expr"; + +fn generate_expr(max_depth: u32) -> String { + let mut rng = rand::rng(); + let ops = [ + "+", "-", "*", "/", "%", "<", ">", "=", "&", "|", "!=", "<=", ">=", ":", "index", "length", + "substr", + ]; + + let mut expr = String::new(); + let mut depth = 0; + let mut last_was_operator = false; + + while depth <= max_depth { + if last_was_operator || depth == 0 { + // Add a number + expr.push_str(&rng.random_range(1..=100).to_string()); + last_was_operator = false; + } else { + // 90% chance to add an operator followed by a number + if rng.random_bool(0.9) { + let op = *ops.choose(&mut rng).unwrap(); + expr.push_str(&format!(" {op} ")); + last_was_operator = true; + } + // 10% chance to add a random string (potentially invalid syntax) + else { + let random_str = generate_random_string(rng.random_range(1..=10)); + expr.push_str(&random_str); + last_was_operator = false; + } + } + depth += 1; + } + + // Ensure the expression ends with a number if it ended with an operator + if last_was_operator { + expr.push_str(&rng.random_range(1..=100).to_string()); + } + + expr +} + +fuzz_target!(|_data: &[u8]| { + let mut rng = rand::rng(); + let expr = generate_expr(rng.random_range(0..=20)); + let mut args = vec![OsString::from("expr")]; + args.extend(expr.split_whitespace().map(OsString::from)); + + // Use C locale to avoid false positives, like in https://github.com/uutils/coreutils/issues/5378, + // because uutils expr doesn't support localization yet + // TODO remove once uutils expr supports localization + unsafe { + env::set_var("LC_COLLATE", "C"); + } + let rust_result = generate_and_run_uumain(&args, uumain, None); + + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "expr", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, // Set to true if you want to fail on stderr diff + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_parse_glob.rs b/fuzz/fuzz_targets/fuzz_parse_glob.rs index 061569bc418..66e772959e7 100644 --- a/fuzz/fuzz_targets/fuzz_parse_glob.rs +++ b/fuzz/fuzz_targets/fuzz_parse_glob.rs @@ -1,10 +1,10 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use uucore::parse_glob; +use uucore::parser::parse_glob; fuzz_target!(|data: &[u8]| { if let Ok(s) = std::str::from_utf8(data) { - _ = parse_glob::from_str(s) + _ = parse_glob::from_str(s); } }); diff --git a/fuzz/fuzz_targets/fuzz_parse_size.rs b/fuzz/fuzz_targets/fuzz_parse_size.rs index 23b3b5ea426..4e8d7e2216b 100644 --- a/fuzz/fuzz_targets/fuzz_parse_size.rs +++ b/fuzz/fuzz_targets/fuzz_parse_size.rs @@ -1,10 +1,10 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use uucore::parse_size::parse_size; +use uucore::parser::parse_size::parse_size_u64; fuzz_target!(|data: &[u8]| { if let Ok(s) = std::str::from_utf8(data) { - _ = parse_size(s); + _ = parse_size_u64(s); } }); diff --git a/fuzz/fuzz_targets/fuzz_parse_time.rs b/fuzz/fuzz_targets/fuzz_parse_time.rs index a643c6d805c..5745e5c8709 100644 --- a/fuzz/fuzz_targets/fuzz_parse_time.rs +++ b/fuzz/fuzz_targets/fuzz_parse_time.rs @@ -1,10 +1,11 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use uucore::parse_time; +use uucore::parser::parse_time; fuzz_target!(|data: &[u8]| { if let Ok(s) = std::str::from_utf8(data) { - _ = parse_time::from_str(s); + _ = parse_time::from_str(s, true); + _ = parse_time::from_str(s, false); } }); diff --git a/fuzz/fuzz_targets/fuzz_printf.rs b/fuzz/fuzz_targets/fuzz_printf.rs new file mode 100644 index 00000000000..885ebb815bf --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_printf.rs @@ -0,0 +1,109 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore parens + +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_printf::uumain; + +use rand::Rng; +use rand::seq::IndexedRandom; +use std::env; +use std::ffi::OsString; + +use uufuzz::CommandResult; +use uufuzz::{compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd}; + +static CMD_PATH: &str = "printf"; + +fn generate_escape_sequence(rng: &mut impl Rng) -> String { + let escape_sequences = [ + "\\\"", + "\\\\", + "\\a", + "\\b", + "\\c", + "\\e", + "\\f", + "\\n", + "\\r", + "\\t", + "\\v", + "\\000", + "\\x00", + "\\u0000", + "\\U00000000", + "%%", + ]; + escape_sequences.choose(rng).unwrap().to_string() +} + +fn generate_printf() -> String { + let mut rng = rand::rng(); + let format_specifiers = ["%s", "%d", "%f", "%x", "%o", "%c", "%b", "%q"]; + let mut printf_str = String::new(); + // Add a 20% chance of generating an invalid format specifier + if rng.random_bool(0.2) { + printf_str.push_str("%z"); // Invalid format specifier + } else { + let specifier = *format_specifiers.choose(&mut rng).unwrap(); + printf_str.push_str(specifier); + + // Add a 20% chance of introducing complex format strings + if rng.random_bool(0.2) { + printf_str.push_str(&format!(" %{}", rng.random_range(1..=1000))); + } else { + // Add a random string or number after the specifier + if specifier == "%s" { + printf_str.push_str(&format!( + " {}", + generate_random_string(rng.random_range(1..=10)) + )); + } else { + printf_str.push_str(&format!(" {}", rng.random_range(1..=1000))); + } + } + } + + // Add a 10% chance of including an escape sequence + if rng.random_bool(0.1) { + printf_str.push_str(&generate_escape_sequence(&mut rng)); + } + printf_str +} + +fuzz_target!(|_data: &[u8]| { + let printf_input = generate_printf(); + let mut args = vec![OsString::from("printf")]; + args.extend(printf_input.split_whitespace().map(OsString::from)); + let rust_result = generate_and_run_uumain(&args, uumain, None); + + // TODO remove once uutils printf supports localization + unsafe { + env::set_var("LC_ALL", "C"); + } + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "printf", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, // Set to true if you want to fail on stderr diff + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_seq.rs b/fuzz/fuzz_targets/fuzz_seq.rs new file mode 100644 index 00000000000..35721865e8c --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_seq.rs @@ -0,0 +1,72 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore parens + +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_seq::uumain; + +use rand::Rng; +use std::ffi::OsString; + +use uufuzz::CommandResult; +use uufuzz::{compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd}; +static CMD_PATH: &str = "seq"; + +fn generate_seq() -> String { + let mut rng = rand::rng(); + + // Generate 1 to 3 numbers for seq arguments + let arg_count = rng.random_range(1..=3); + let mut args = Vec::new(); + + for _ in 0..arg_count { + if rng.random_ratio(1, 100) { + // 1% chance to add a random string + args.push(generate_random_string(rng.random_range(1..=10))); + } else { + // 99% chance to add a numeric value + match rng.random_range(0..=3) { + 0 => args.push(rng.random_range(-10000..=10000).to_string()), // Large or small integers + 1 => args.push(rng.random_range(-100.0..100.0).to_string()), // Floating-point numbers + 2 => args.push(rng.random_range(-100..0).to_string()), // Negative integers + _ => args.push(rng.random_range(1..=100).to_string()), // Regular integers + } + } + } + + args.join(" ") +} + +fuzz_target!(|_data: &[u8]| { + let seq = generate_seq(); + let mut args = vec![OsString::from("seq")]; + args.extend(seq.split_whitespace().map(OsString::from)); + + let rust_result = generate_and_run_uumain(&args, uumain, None); + + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "seq", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, // Set to true if you want to fail on stderr diff + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_seq_parse_number.rs b/fuzz/fuzz_targets/fuzz_seq_parse_number.rs new file mode 100644 index 00000000000..04da6d47f99 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_seq_parse_number.rs @@ -0,0 +1,15 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +#![no_main] + +use libfuzzer_sys::fuzz_target; +use std::str::FromStr; +use uu_seq::number::PreciseNumber; + +fuzz_target!(|data: &[u8]| { + if let Ok(s) = std::str::from_utf8(data) { + let _ = PreciseNumber::from_str(s); + } +}); diff --git a/fuzz/fuzz_targets/fuzz_sort.rs b/fuzz/fuzz_targets/fuzz_sort.rs new file mode 100644 index 00000000000..8b38f39ec1b --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_sort.rs @@ -0,0 +1,85 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore parens + +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_sort::uumain; + +use rand::Rng; +use std::env; +use std::ffi::OsString; + +use uufuzz::CommandResult; +use uufuzz::{compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd}; +static CMD_PATH: &str = "sort"; + +fn generate_sort_args() -> String { + let mut rng = rand::rng(); + + let arg_count = rng.random_range(1..=5); + let mut args = Vec::new(); + + for _ in 0..arg_count { + match rng.random_range(0..=4) { + 0 => args.push(String::from("-r")), // Reverse the result of comparisons + 1 => args.push(String::from("-n")), // Compare according to string numerical value + 2 => args.push(String::from("-f")), // Fold lower case to upper case characters + 3 => args.push(generate_random_string(rng.random_range(1..=10))), // Random string (to simulate file names) + _ => args.push(String::from("-k") + &rng.random_range(1..=5).to_string()), // Sort via a specified field + } + } + + args.join(" ") +} + +fn generate_random_lines(count: usize) -> String { + let mut rng = rand::rng(); + let mut lines = Vec::new(); + + for _ in 0..count { + lines.push(generate_random_string(rng.random_range(1..=20))); + } + + lines.join("\n") +} + +fuzz_target!(|_data: &[u8]| { + let sort_args = generate_sort_args(); + let mut args = vec![OsString::from("sort")]; + args.extend(sort_args.split_whitespace().map(OsString::from)); + + // Generate random lines to sort + let input_lines = generate_random_lines(10); + + let rust_result = generate_and_run_uumain(&args, uumain, Some(&input_lines)); + + // TODO remove once uutils sort supports localization + unsafe { + env::set_var("LC_ALL", "C"); + } + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, Some(&input_lines)) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "sort", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, // Set to true if you want to fail on stderr diff + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_split.rs b/fuzz/fuzz_targets/fuzz_split.rs new file mode 100644 index 00000000000..70860ece731 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_split.rs @@ -0,0 +1,104 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore parens + +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_split::uumain; + +use rand::Rng; +use std::ffi::OsString; + +use uufuzz::{ + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, +}; +static CMD_PATH: &str = "split"; + +fn generate_split_args() -> String { + let mut rng = rand::rng(); + let mut args = Vec::new(); + + match rng.random_range(0..=9) { + 0 => { + args.push(String::from("-a")); // Suffix length + args.push(rng.random_range(1..=8).to_string()); + } + 1 => { + args.push(String::from("--additional-suffix")); + args.push(generate_random_string(5)); // Random suffix + } + 2 => { + args.push(String::from("-b")); // Bytes per output file + args.push(rng.random_range(1..=1024).to_string() + "K"); + } + 3 => { + args.push(String::from("-C")); // Line bytes + args.push(rng.random_range(1..=1024).to_string()); + } + 4 => args.push(String::from("-d")), // Use numeric suffixes + 5 => args.push(String::from("-x")), // Use hex suffixes + 6 => { + args.push(String::from("-l")); // Number of lines per output file + args.push(rng.random_range(1..=1000).to_string()); + } + 7 => { + args.push(String::from("--filter")); + args.push(String::from("cat > /dev/null")); // Example filter command + } + 8 => { + args.push(String::from("-t")); // Separator + args.push(String::from("\n")); // Newline as separator + } + 9 => args.push(String::from("--verbose")), // Verbose + _ => (), + } + + args.join(" ") +} + +// Function to generate a random string of lines +fn generate_random_lines(count: usize) -> String { + let mut rng = rand::rng(); + let mut lines = Vec::new(); + + for _ in 0..count { + lines.push(generate_random_string(rng.random_range(1..=20))); + } + + lines.join("\n") +} + +fuzz_target!(|_data: &[u8]| { + let split_args = generate_split_args(); + let mut args = vec![OsString::from("split")]; + args.extend(split_args.split_whitespace().map(OsString::from)); + + let input_lines = generate_random_lines(10); + + let rust_result = generate_and_run_uumain(&args, uumain, Some(&input_lines)); + + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, Some(&input_lines)) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "split", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, // Set to true if you want to fail on stderr diff + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_test.rs b/fuzz/fuzz_targets/fuzz_test.rs index 537e21abda5..894a1dcd56b 100644 --- a/fuzz/fuzz_targets/fuzz_test.rs +++ b/fuzz/fuzz_targets/fuzz_test.rs @@ -8,13 +8,14 @@ use libfuzzer_sys::fuzz_target; use uu_test::uumain; -use rand::seq::SliceRandom; use rand::Rng; +use rand::prelude::IndexedRandom; use std::ffi::OsString; -use libc::{dup, dup2, STDOUT_FILENO}; -use std::process::Command; +use uufuzz::CommandResult; +use uufuzz::{compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd}; +#[allow(clippy::upper_case_acronyms)] #[derive(PartialEq, Debug, Clone)] enum ArgType { STRING, @@ -26,41 +27,7 @@ enum ArgType { // Add any other types as needed } -fn run_gnu_test(args: &[OsString]) -> Result<(String, i32), std::io::Error> { - let mut command = Command::new("test"); - for arg in args { - command.arg(arg); - } - let output = command.output()?; - let exit_status = output.status.code().unwrap_or(-1); // Capture the exit status code - Ok(( - String::from_utf8_lossy(&output.stdout).to_string(), - exit_status, - )) -} - -fn generate_random_string(max_length: usize) -> String { - let mut rng = rand::thread_rng(); - let valid_utf8: Vec = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - .chars() - .collect(); - let invalid_utf8 = [0xC3, 0x28]; // Invalid UTF-8 sequence - let mut result = String::new(); - - for _ in 0..rng.gen_range(1..=max_length) { - if rng.gen_bool(0.9) { - let ch = valid_utf8.choose(&mut rng).unwrap(); - result.push(*ch); - } else { - let ch = invalid_utf8.choose(&mut rng).unwrap(); - if let Some(c) = char::from_u32(*ch as u32) { - result.push(c); - } - } - } - - result -} +static CMD_PATH: &str = "test"; #[derive(Debug, Clone)] struct TestArg { @@ -69,7 +36,7 @@ struct TestArg { } fn generate_random_path(rng: &mut dyn rand::RngCore) -> &'static str { - match rng.gen_range(0..=3) { + match rng.random_range(0..=3) { 0 => "/dev/null", 1 => "/dev/random", 2 => "/tmp", @@ -95,6 +62,14 @@ fn generate_test_args() -> Vec { arg: "!=".to_string(), arg_type: ArgType::STRINGSTRING, }, + TestArg { + arg: ">".to_string(), + arg_type: ArgType::STRINGSTRING, + }, + TestArg { + arg: "<".to_string(), + arg_type: ArgType::STRINGSTRING, + }, TestArg { arg: "-eq".to_string(), arg_type: ArgType::INTEGERINTEGER, @@ -143,49 +118,46 @@ fn generate_test_args() -> Vec { } fn generate_test_arg() -> String { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let test_args = generate_test_args(); let mut arg = String::new(); - let choice = rng.gen_range(0..=5); + let choice = rng.random_range(0..=5); match choice { 0 => { - arg.push_str(&rng.gen_range(-100..=100).to_string()); + arg.push_str(&rng.random_range(-100..=100).to_string()); } - 1 | 2 | 3 => { + 1..=3 => { let test_arg = test_args .choose(&mut rng) .expect("Failed to choose a random test argument"); if test_arg.arg_type == ArgType::INTEGER { arg.push_str(&format!( "{} {} {}", - &rng.gen_range(-100..=100).to_string(), + rng.random_range(-100..=100).to_string(), test_arg.arg, - &rng.gen_range(-100..=100).to_string() + rng.random_range(-100..=100).to_string() )); } else if test_arg.arg_type == ArgType::STRINGSTRING { - let random_str = generate_random_string(rng.gen_range(1..=10)); - let random_str2 = generate_random_string(rng.gen_range(1..=10)); + let random_str = generate_random_string(rng.random_range(1..=10)); + let random_str2 = generate_random_string(rng.random_range(1..=10)); - arg.push_str(&format!( - "{} {} {}", - &random_str, test_arg.arg, &random_str2 - )); + arg.push_str(&format!("{random_str} {} {random_str2}", test_arg.arg,)); } else if test_arg.arg_type == ArgType::STRING { - let random_str = generate_random_string(rng.gen_range(1..=10)); - arg.push_str(&format!("{} {}", test_arg.arg, &random_str)); + let random_str = generate_random_string(rng.random_range(1..=10)); + arg.push_str(&format!("{} {random_str}", test_arg.arg)); } else if test_arg.arg_type == ArgType::FILEFILE { let path = generate_random_path(&mut rng); let path2 = generate_random_path(&mut rng); - arg.push_str(&format!("{} {} {}", path, test_arg.arg, path2)); + arg.push_str(&format!("{path} {} {path2}", test_arg.arg)); } else if test_arg.arg_type == ArgType::FILE { let path = generate_random_path(&mut rng); - arg.push_str(&format!("{} {}", test_arg.arg, path)); + arg.push_str(&format!("{} {path}", test_arg.arg)); } } 4 => { - let random_str = generate_random_string(rng.gen_range(1..=10)); + let random_str = generate_random_string(rng.random_range(1..=10)); arg.push_str(&random_str); } _ => { @@ -198,7 +170,7 @@ fn generate_test_arg() -> String { .collect(); if let Some(test_arg) = file_test_args.choose(&mut rng) { - arg.push_str(&format!("{}{}", test_arg.arg, path)); + arg.push_str(&format!("{}{path}", test_arg.arg)); } } } @@ -207,85 +179,36 @@ fn generate_test_arg() -> String { } fuzz_target!(|_data: &[u8]| { - let mut rng = rand::thread_rng(); - let max_args = rng.gen_range(1..=6); + let mut rng = rand::rng(); + let max_args = rng.random_range(1..=6); let mut args = vec![OsString::from("test")]; - let uumain_exit_status; for _ in 0..max_args { args.push(OsString::from(generate_test_arg())); } - // Save the original stdout file descriptor - let original_stdout_fd = unsafe { dup(STDOUT_FILENO) }; - println!("Running test {:?}", &args[1..]); - // Create a pipe to capture stdout - let mut pipe_fds = [-1; 2]; - unsafe { libc::pipe(pipe_fds.as_mut_ptr()) }; - - { - // Redirect stdout to the write end of the pipe - unsafe { dup2(pipe_fds[1], STDOUT_FILENO) }; - - // Run uumain with the provided arguments - uumain_exit_status = uumain(args.clone().into_iter()); - - // Restore original stdout - unsafe { dup2(original_stdout_fd, STDOUT_FILENO) }; - unsafe { libc::close(original_stdout_fd) }; - } - // Close the write end of the pipe - unsafe { libc::close(pipe_fds[1]) }; - - // Read captured output from the read end of the pipe - let mut captured_output = Vec::new(); - let mut read_buffer = [0; 1024]; - loop { - let bytes_read = unsafe { - libc::read( - pipe_fds[0], - read_buffer.as_mut_ptr() as *mut libc::c_void, - read_buffer.len(), - ) - }; - if bytes_read <= 0 { - break; - } - captured_output.extend_from_slice(&read_buffer[..bytes_read as usize]); - } - - // Close the read end of the pipe - unsafe { libc::close(pipe_fds[0]) }; - - // Convert captured output to a string - let my_output = String::from_utf8_lossy(&captured_output) - .to_string() - .trim() - .to_owned(); - - // Run GNU test with the provided arguments and compare the output - match run_gnu_test(&args[1..]) { - Ok((gnu_output, gnu_exit_status)) => { - let gnu_output = gnu_output.trim().to_owned(); - println!("gnu_exit_status {}", gnu_exit_status); - println!("uumain_exit_status {}", uumain_exit_status); - if my_output != gnu_output || uumain_exit_status != gnu_exit_status { - println!("Discrepancy detected!"); - println!("Test: {:?}", &args[1..]); - println!("My output: {}", my_output); - println!("GNU output: {}", gnu_output); - println!("My exit status: {}", uumain_exit_status); - println!("GNU exit status: {}", gnu_exit_status); - panic!(); - } else { - println!( - "Outputs and exit statuses matched for expression {:?}", - &args[1..] - ); + let rust_result = generate_and_run_uumain(&args, uumain, None); + + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, None) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, } } - Err(_) => { - println!("GNU test execution failed for expression {:?}", &args[1..]); - } - } + }; + + compare_result( + "test", + &format!("{:?}", &args[1..]), + None, + &rust_result, + &gnu_result, + false, // Set to true if you want to fail on stderr diff + ); }); diff --git a/fuzz/fuzz_targets/fuzz_tr.rs b/fuzz/fuzz_targets/fuzz_tr.rs new file mode 100644 index 00000000000..5055ec0d748 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_tr.rs @@ -0,0 +1,72 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +#![no_main] +use libfuzzer_sys::fuzz_target; +use std::ffi::OsString; +use uu_tr::uumain; + +use rand::Rng; + +use uufuzz::{ + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, +}; +static CMD_PATH: &str = "tr"; + +fn generate_tr_args() -> Vec { + let mut rng = rand::rng(); + let mut args = Vec::new(); + + // Translate, squeeze, and/or delete characters + let opts = ["-c", "-d", "-s", "-t"]; + for opt in &opts { + if rng.random_bool(0.25) { + args.push(opt.to_string()); + } + } + + // Generating STRING1 and optionally STRING2 + let string1 = generate_random_string(rng.random_range(1..=20)); + args.push(string1); + if rng.random_bool(0.7) { + // Higher chance to add STRING2 for translation + let string2 = generate_random_string(rng.random_range(1..=20)); + args.push(string2); + } + + args +} + +fuzz_target!(|_data: &[u8]| { + let tr_args = generate_tr_args(); + let mut args = vec![OsString::from("tr")]; + args.extend(tr_args.iter().map(OsString::from)); + + let input_chars = generate_random_string(100); + + let rust_result = generate_and_run_uumain(&args, uumain, Some(&input_chars)); + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, Some(&input_chars)) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "tr", + &format!("{:?}", &args[1..]), + Some(&input_chars), + &rust_result, + &gnu_result, + false, + ); +}); diff --git a/fuzz/fuzz_targets/fuzz_wc.rs b/fuzz/fuzz_targets/fuzz_wc.rs new file mode 100644 index 00000000000..dbc046522bb --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_wc.rs @@ -0,0 +1,98 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore parens + +#![no_main] +use libfuzzer_sys::fuzz_target; +use uu_wc::uumain; + +use rand::Rng; +use std::ffi::OsString; + +use uufuzz::{ + CommandResult, compare_result, generate_and_run_uumain, generate_random_string, run_gnu_cmd, +}; +static CMD_PATH: &str = "wc"; + +fn generate_wc_args() -> String { + let mut rng = rand::rng(); + let arg_count = rng.random_range(1..=6); + let mut args = Vec::new(); + + for _ in 0..arg_count { + // Introduce a chance to add invalid arguments + if rng.random_bool(0.1) { + args.push(generate_random_string(rng.random_range(1..=20))); + } else { + match rng.random_range(0..=5) { + 0 => args.push(String::from("-c")), + 1 => args.push(String::from("-m")), + 2 => args.push(String::from("-l")), + 3 => args.push(String::from("-L")), + 4 => args.push(String::from("-w")), + // TODO + 5 => { + args.push(String::from("--files0-from")); + if rng.random_bool(0.5) { + args.push(generate_random_string(50)); // Longer invalid file name + } else { + args.push(generate_random_string(5)); + } + } + _ => (), + } + } + } + + args.join(" ") +} + +// Function to generate a random string of lines, including invalid ones +fn generate_random_lines(count: usize) -> String { + let mut rng = rand::rng(); + let mut lines = Vec::new(); + + for _ in 0..count { + if rng.random_bool(0.1) { + lines.push(generate_random_string(rng.random_range(1000..=5000))); // Very long invalid line + } else { + lines.push(generate_random_string(rng.random_range(1..=20))); + } + } + + lines.join("\n") +} + +fuzz_target!(|_data: &[u8]| { + let wc_args = generate_wc_args(); + let mut args = vec![OsString::from("wc")]; + args.extend(wc_args.split_whitespace().map(OsString::from)); + + let input_lines = generate_random_lines(10); + + let rust_result = generate_and_run_uumain(&args, uumain, Some(&input_lines)); + let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false, Some(&input_lines)) { + Ok(result) => result, + Err(error_result) => { + eprintln!("Failed to run GNU command:"); + eprintln!("Stderr: {}", error_result.stderr); + eprintln!("Exit Code: {}", error_result.exit_code); + CommandResult { + stdout: String::new(), + stderr: error_result.stderr, + exit_code: error_result.exit_code, + } + } + }; + + compare_result( + "wc", + &format!("{:?}", &args[1..]), + Some(&input_lines), + &rust_result, + &gnu_result, + false, // Set to true if you want to fail on stderr diff + ); +}); diff --git a/fuzz/uufuzz/Cargo.toml b/fuzz/uufuzz/Cargo.toml new file mode 100644 index 00000000000..d206d86319a --- /dev/null +++ b/fuzz/uufuzz/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "uufuzz" +authors = ["uutils developers"] +description = "uutils ~ 'core' uutils fuzzing library" +repository = "https://github.com/uutils/coreutils/tree/main/fuzz/uufuzz" +version = "0.1.0" +edition.workspace = true +license.workspace = true + + +[dependencies] +console = "0.15.0" +libc = "0.2.153" +rand = { version = "0.9.0", features = ["small_rng"] } +similar = "2.5.0" +uucore = { path = "../../src/uucore/", features = ["parser"] } +tempfile = "3.15.0" diff --git a/fuzz/uufuzz/src/lib.rs b/fuzz/uufuzz/src/lib.rs new file mode 100644 index 00000000000..e887bfc6755 --- /dev/null +++ b/fuzz/uufuzz/src/lib.rs @@ -0,0 +1,438 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use console::Style; +use libc::STDIN_FILENO; +use libc::{STDERR_FILENO, STDOUT_FILENO, close, dup, dup2, pipe}; +use pretty_print::{ + print_diff, print_end_with_status, print_or_empty, print_section, print_with_style, +}; +use rand::Rng; +use rand::prelude::IndexedRandom; +use std::env::temp_dir; +use std::ffi::OsString; +use std::fs::File; +use std::io::{Seek, SeekFrom, Write}; +use std::os::fd::{AsRawFd, RawFd}; +use std::process::{Command, Stdio}; +use std::sync::atomic::Ordering; +use std::sync::{Once, atomic::AtomicBool}; +use std::{io, thread}; + +pub mod pretty_print; + +/// Represents the result of running a command, including its standard output, +/// standard error, and exit code. +pub struct CommandResult { + /// The standard output (stdout) of the command as a string. + pub stdout: String, + + /// The standard error (stderr) of the command as a string. + pub stderr: String, + + /// The exit code of the command. + pub exit_code: i32, +} + +static CHECK_GNU: Once = Once::new(); +static IS_GNU: AtomicBool = AtomicBool::new(false); + +pub fn is_gnu_cmd(cmd_path: &str) -> Result<(), std::io::Error> { + CHECK_GNU.call_once(|| { + let version_output = Command::new(cmd_path).arg("--version").output().unwrap(); + + println!("version_output {version_output:#?}"); + + let version_str = String::from_utf8_lossy(&version_output.stdout).to_string(); + if version_str.contains("GNU coreutils") { + IS_GNU.store(true, Ordering::Relaxed); + } + }); + + if IS_GNU.load(Ordering::Relaxed) { + Ok(()) + } else { + panic!("Not the GNU implementation"); + } +} + +pub fn generate_and_run_uumain( + args: &[OsString], + uumain_function: F, + pipe_input: Option<&str>, +) -> CommandResult +where + F: FnOnce(std::vec::IntoIter) -> i32 + Send + 'static, +{ + // Duplicate the stdout and stderr file descriptors + let original_stdout_fd = unsafe { dup(STDOUT_FILENO) }; + let original_stderr_fd = unsafe { dup(STDERR_FILENO) }; + if original_stdout_fd == -1 || original_stderr_fd == -1 { + return CommandResult { + stdout: "".to_string(), + stderr: "Failed to duplicate STDOUT_FILENO or STDERR_FILENO".to_string(), + exit_code: -1, + }; + } + + println!("Running test {:?}", &args[0..]); + let mut pipe_stdout_fds = [-1; 2]; + let mut pipe_stderr_fds = [-1; 2]; + + // Create pipes for stdout and stderr + if unsafe { pipe(pipe_stdout_fds.as_mut_ptr()) } == -1 + || unsafe { pipe(pipe_stderr_fds.as_mut_ptr()) } == -1 + { + return CommandResult { + stdout: "".to_string(), + stderr: "Failed to create pipes".to_string(), + exit_code: -1, + }; + } + + // Redirect stdout and stderr to their respective pipes + if unsafe { dup2(pipe_stdout_fds[1], STDOUT_FILENO) } == -1 + || unsafe { dup2(pipe_stderr_fds[1], STDERR_FILENO) } == -1 + { + unsafe { + close(pipe_stdout_fds[0]); + close(pipe_stdout_fds[1]); + close(pipe_stderr_fds[0]); + close(pipe_stderr_fds[1]); + } + return CommandResult { + stdout: "".to_string(), + stderr: "Failed to redirect STDOUT_FILENO or STDERR_FILENO".to_string(), + exit_code: -1, + }; + } + + let original_stdin_fd = if let Some(input_str) = pipe_input { + // we have pipe input + let mut input_file = tempfile::tempfile().unwrap(); + write!(input_file, "{input_str}").unwrap(); + input_file.seek(SeekFrom::Start(0)).unwrap(); + + // Redirect stdin to read from the in-memory file + let original_stdin_fd = unsafe { dup(STDIN_FILENO) }; + if original_stdin_fd == -1 || unsafe { dup2(input_file.as_raw_fd(), STDIN_FILENO) } == -1 { + return CommandResult { + stdout: "".to_string(), + stderr: "Failed to set up stdin redirection".to_string(), + exit_code: -1, + }; + } + Some(original_stdin_fd) + } else { + None + }; + + let (uumain_exit_status, captured_stdout, captured_stderr) = thread::scope(|s| { + let out = s.spawn(|| read_from_fd(pipe_stdout_fds[0])); + let err = s.spawn(|| read_from_fd(pipe_stderr_fds[0])); + #[allow(clippy::unnecessary_to_owned)] + // TODO: clippy wants us to use args.iter().cloned() ? + let status = uumain_function(args.to_owned().into_iter()); + // Reset the exit code global variable in case we run another test after this one + // See https://github.com/uutils/coreutils/issues/5777 + uucore::error::set_exit_code(0); + io::stdout().flush().unwrap(); + io::stderr().flush().unwrap(); + unsafe { + close(pipe_stdout_fds[1]); + close(pipe_stderr_fds[1]); + close(STDOUT_FILENO); + close(STDERR_FILENO); + } + (status, out.join().unwrap(), err.join().unwrap()) + }); + + // Restore the original stdout and stderr + if unsafe { dup2(original_stdout_fd, STDOUT_FILENO) } == -1 + || unsafe { dup2(original_stderr_fd, STDERR_FILENO) } == -1 + { + return CommandResult { + stdout: "".to_string(), + stderr: "Failed to restore the original STDOUT_FILENO or STDERR_FILENO".to_string(), + exit_code: -1, + }; + } + unsafe { + close(original_stdout_fd); + close(original_stderr_fd); + } + + // Restore the original stdin if it was modified + if let Some(fd) = original_stdin_fd { + if unsafe { dup2(fd, STDIN_FILENO) } == -1 { + return CommandResult { + stdout: "".to_string(), + stderr: "Failed to restore the original STDIN".to_string(), + exit_code: -1, + }; + } + unsafe { close(fd) }; + } + + CommandResult { + stdout: captured_stdout, + stderr: captured_stderr + .split_once(':') + .map(|x| x.1) + .unwrap_or("") + .trim() + .to_string(), + exit_code: uumain_exit_status, + } +} + +fn read_from_fd(fd: RawFd) -> String { + let mut captured_output = Vec::new(); + let mut read_buffer = [0; 1024]; + loop { + let bytes_read = unsafe { + libc::read( + fd, + read_buffer.as_mut_ptr() as *mut libc::c_void, + read_buffer.len(), + ) + }; + + if bytes_read == -1 { + eprintln!("Failed to read from the pipe"); + break; + } + if bytes_read == 0 { + break; + } + captured_output.extend_from_slice(&read_buffer[..bytes_read as usize]); + } + + unsafe { libc::close(fd) }; + + String::from_utf8_lossy(&captured_output).into_owned() +} + +pub fn run_gnu_cmd( + cmd_path: &str, + args: &[OsString], + check_gnu: bool, + pipe_input: Option<&str>, +) -> Result { + if check_gnu { + match is_gnu_cmd(cmd_path) { + Ok(_) => {} // if the check passes, do nothing + Err(e) => { + // Convert the io::Error into the function's error type + return Err(CommandResult { + stdout: String::new(), + stderr: e.to_string(), + exit_code: -1, + }); + } + } + } + + let mut command = Command::new(cmd_path); + for arg in args { + command.arg(arg); + } + + // See https://github.com/uutils/coreutils/issues/6794 + // uutils' coreutils is not locale-aware, and aims to mirror/be compatible with GNU Core Utilities's LC_ALL=C behavior + command.env("LC_ALL", "C"); + + let output = if let Some(input_str) = pipe_input { + // We have an pipe input + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut child = command.spawn().expect("Failed to execute command"); + let child_stdin = child.stdin.as_mut().unwrap(); + child_stdin + .write_all(input_str.as_bytes()) + .expect("Failed to write to stdin"); + + match child.wait_with_output() { + Ok(output) => output, + Err(e) => { + return Err(CommandResult { + stdout: String::new(), + stderr: e.to_string(), + exit_code: -1, + }); + } + } + } else { + // Just run with args + match command.output() { + Ok(output) => output, + Err(e) => { + return Err(CommandResult { + stdout: String::new(), + stderr: e.to_string(), + exit_code: -1, + }); + } + } + }; + let exit_code = output.status.code().unwrap_or(-1); + // Here we get stdout and stderr as Strings + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + let stderr = stderr + .split_once(':') + .map(|x| x.1) + .unwrap_or("") + .trim() + .to_string(); + + if output.status.success() || !check_gnu { + Ok(CommandResult { + stdout, + stderr, + exit_code, + }) + } else { + Err(CommandResult { + stdout, + stderr, + exit_code, + }) + } +} + +/// Compare results from two different implementations of a command. +/// +/// # Arguments +/// * `test_type` - The command. +/// * `input` - The input provided to the command. +/// * `rust_result` - The result of running the command with the Rust implementation. +/// * `gnu_result` - The result of running the command with the GNU implementation. +/// * `fail_on_stderr_diff` - Whether to fail the test if there is a difference in stderr output. +pub fn compare_result( + test_type: &str, + input: &str, + pipe_input: Option<&str>, + rust_result: &CommandResult, + gnu_result: &CommandResult, + fail_on_stderr_diff: bool, +) { + print_section(format!("Compare result for: {test_type} {input}")); + + if let Some(pipe) = pipe_input { + println!("Pipe: {pipe}"); + } + + let mut discrepancies = Vec::new(); + let mut should_panic = false; + + if rust_result.stdout.trim() != gnu_result.stdout.trim() { + discrepancies.push("stdout differs"); + println!("Rust stdout:"); + print_or_empty(rust_result.stdout.as_str()); + println!("GNU stdout:"); + print_or_empty(gnu_result.stdout.as_ref()); + print_diff(&rust_result.stdout, &gnu_result.stdout); + should_panic = true; + } + + if rust_result.stderr.trim() != gnu_result.stderr.trim() { + discrepancies.push("stderr differs"); + println!("Rust stderr:"); + print_or_empty(rust_result.stderr.as_str()); + println!("GNU stderr:"); + print_or_empty(gnu_result.stderr.as_str()); + print_diff(&rust_result.stderr, &gnu_result.stderr); + if fail_on_stderr_diff { + should_panic = true; + } + } + + if rust_result.exit_code != gnu_result.exit_code { + discrepancies.push("exit code differs"); + println!( + "Different exit code: (Rust: {}, GNU: {})", + rust_result.exit_code, gnu_result.exit_code + ); + should_panic = true; + } + + if discrepancies.is_empty() { + print_end_with_status("Same behavior", true); + } else { + print_with_style( + format!("Discrepancies detected: {}", discrepancies.join(", ")), + Style::new().red(), + ); + if should_panic { + print_end_with_status( + format!("Test failed and will panic for: {test_type} {input}"), + false, + ); + panic!("Test failed for: {test_type} {input}"); + } else { + print_end_with_status( + format!("Test completed with discrepancies for: {test_type} {input}"), + false, + ); + } + } + println!(); +} + +pub fn generate_random_string(max_length: usize) -> String { + let mut rng = rand::rng(); + let valid_utf8: Vec = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + .chars() + .collect(); + let invalid_utf8 = [0xC3, 0x28]; // Invalid UTF-8 sequence + let mut result = String::new(); + + for _ in 0..rng.random_range(0..=max_length) { + if rng.random_bool(0.9) { + let ch = valid_utf8.choose(&mut rng).unwrap(); + result.push(*ch); + } else { + let ch = invalid_utf8.choose(&mut rng).unwrap(); + if let Some(c) = char::from_u32(*ch as u32) { + result.push(c); + } + } + } + + result +} + +#[allow(dead_code)] +pub fn generate_random_file() -> Result { + let mut rng = rand::rng(); + let file_name: String = (0..10) + .map(|_| rng.random_range(b'a'..=b'z') as char) + .collect(); + let mut file_path = temp_dir(); + file_path.push(file_name); + + let mut file = File::create(&file_path)?; + + let content_length = rng.random_range(10..1000); + let content: String = (0..content_length) + .map(|_| (rng.random_range(b' '..=b'~') as char)) + .collect(); + + file.write_all(content.as_bytes())?; + + Ok(file_path.to_str().unwrap().to_string()) +} + +#[allow(dead_code)] +pub fn replace_fuzz_binary_name(cmd: &str, result: &mut CommandResult) { + let fuzz_bin_name = format!("fuzz/target/x86_64-unknown-linux-gnu/release/fuzz_{cmd}"); + + result.stdout = result.stdout.replace(&fuzz_bin_name, cmd); + result.stderr = result.stderr.replace(&fuzz_bin_name, cmd); +} diff --git a/fuzz/uufuzz/src/pretty_print.rs b/fuzz/uufuzz/src/pretty_print.rs new file mode 100644 index 00000000000..ecdfccfd035 --- /dev/null +++ b/fuzz/uufuzz/src/pretty_print.rs @@ -0,0 +1,69 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::fmt; + +use console::{Style, style}; +use similar::TextDiff; + +pub fn print_section(s: S) { + println!("{}", style(format!("=== {s}")).bold()); +} + +pub fn print_subsection(s: S) { + println!("{}", style(format!("--- {s}")).bright()); +} + +#[allow(dead_code)] +pub fn print_test_begin(msg: S) { + println!( + "{} {} {}", + style("===").bold(), // Kind of gray + style("TEST").black().on_yellow().bold(), + style(msg).bold() + ); +} + +pub fn print_end_with_status(msg: S, ok: bool) { + let ok = if ok { + style(" OK ").black().on_green().bold() + } else { + style(" KO ").black().on_red().bold() + }; + + println!( + "{} {ok} {}", + style("===").bold(), // Kind of gray + style(msg).bold() + ); +} + +pub fn print_or_empty(s: &str) { + let to_print = if s.is_empty() { "(empty)" } else { s }; + + println!("{}", style(to_print).dim()); +} + +pub fn print_with_style(msg: S, style: Style) { + println!("{}", style.apply_to(msg)); +} + +pub fn print_diff(got: &str, expected: &str) { + let diff = TextDiff::from_lines(got, expected); + + print_subsection("START diff"); + + for change in diff.iter_all_changes() { + let (sign, style) = match change.tag() { + similar::ChangeTag::Equal => (" ", Style::new().dim()), + similar::ChangeTag::Delete => ("-", Style::new().red()), + similar::ChangeTag::Insert => ("+", Style::new().green()), + }; + print!("{}{}", style.apply_to(sign).bold(), style.apply_to(change)); + } + + print_subsection("END diff"); + println!(); +} diff --git a/renovate.json b/renovate.json index 9dc8c691977..8f60343a8ef 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,5 @@ { "extends": [ - "config:base" + "config:recommended" ] } diff --git a/src/bin/coreutils.rs b/src/bin/coreutils.rs index fc2cd16add2..b29e7ea2337 100644 --- a/src/bin/coreutils.rs +++ b/src/bin/coreutils.rs @@ -21,7 +21,10 @@ include!(concat!(env!("OUT_DIR"), "/uutils_map.rs")); fn usage(utils: &UtilityMap, name: &str) { println!("{name} {VERSION} (multi-call binary)\n"); - println!("Usage: {name} [function [arguments...]]\n"); + println!("Usage: {name} [function [arguments...]]"); + println!(" {name} --list\n"); + println!("Options:"); + println!(" --list lists all defined functions, one per row\n"); println!("Currently defined functions:\n"); #[allow(clippy::map_clone)] let mut utils: Vec<&str> = utils.keys().map(|&s| s).collect(); @@ -34,6 +37,8 @@ fn usage(utils: &UtilityMap, name: &str) { ); } +/// # Panics +/// Panics if the binary path cannot be determined fn binary_path(args: &mut impl Iterator) -> PathBuf { match args.next() { Some(ref s) if !s.is_empty() => PathBuf::from(s), @@ -60,7 +65,7 @@ fn main() { // binary name equals util name? if let Some(&(uumain, _)) = utils.get(binary_as_util) { - process::exit(uumain((vec![binary.into()].into_iter()).chain(args))); + process::exit(uumain(vec![binary.into()].into_iter().chain(args))); } // binary name equals prefixed util name? @@ -85,36 +90,42 @@ fn main() { process::exit(1); } - let util = match util_os.to_str() { - Some(util) => util, - None => not_found(&util_os), + let Some(util) = util_os.to_str() else { + not_found(&util_os) }; - if util == "completion" { - gen_completions(args, &utils); - } - - if util == "manpage" { - gen_manpage(args, &utils); + match util { + "completion" => gen_completions(args, &utils), + "manpage" => gen_manpage(args, &utils), + "--list" => { + let mut utils: Vec<_> = utils.keys().collect(); + utils.sort(); + for util in utils { + println!("{util}"); + } + process::exit(0); + } + // Not a special command: fallthrough to calling a util + _ => {} } match utils.get(util) { Some(&(uumain, _)) => { - process::exit(uumain((vec![util_os].into_iter()).chain(args))); + process::exit(uumain(vec![util_os].into_iter().chain(args))); } None => { if util == "--help" || util == "-h" { // see if they want help on a specific util if let Some(util_os) = args.next() { - let util = match util_os.to_str() { - Some(util) => util, - None => not_found(&util_os), + let Some(util) = util_os.to_str() else { + not_found(&util_os) }; match utils.get(util) { Some(&(uumain, _)) => { let code = uumain( - (vec![util_os, OsString::from("--help")].into_iter()) + vec![util_os, OsString::from("--help")] + .into_iter() .chain(args), ); io::stdout().flush().expect("could not flush stdout"); @@ -138,6 +149,8 @@ fn main() { } /// Prints completions for the utility in the first parameter for the shell in the second parameter to stdout +/// # Panics +/// Panics if the utility map is empty fn gen_completions( args: impl Iterator, util_map: &UtilityMap, @@ -176,6 +189,8 @@ fn gen_completions( } /// Generate the manpage for the utility in the first parameter +/// # Panics +/// Panics if the utility map is empty fn gen_manpage( args: impl Iterator, util_map: &UtilityMap, @@ -208,6 +223,8 @@ fn gen_manpage( process::exit(0); } +/// # Panics +/// Panics if the utility map is empty fn gen_coreutils_app(util_map: &UtilityMap) -> Command { let mut command = Command::new("coreutils"); for (name, (_, sub_app)) in util_map { diff --git a/src/bin/uudoc.rs b/src/bin/uudoc.rs index 77c7a2fcfdd..6a215a4ada4 100644 --- a/src/bin/uudoc.rs +++ b/src/bin/uudoc.rs @@ -13,6 +13,9 @@ use zip::ZipArchive; include!(concat!(env!("OUT_DIR"), "/uutils_map.rs")); +/// # Errors +/// Returns an error if the writer fails. +#[allow(clippy::too_many_lines)] fn main() -> io::Result<()> { let mut tldr_zip = File::open("docs/tldr.zip") .ok() @@ -20,7 +23,9 @@ fn main() -> io::Result<()> { if tldr_zip.is_none() { println!("Warning: No tldr archive found, so the documentation will not include examples."); - println!("To include examples in the documentation, download the tldr archive and put it in the docs/ folder."); + println!( + "To include examples in the documentation, download the tldr archive and put it in the docs/ folder." + ); println!(); println!(" curl https://tldr.sh/assets/tldr.zip -o docs/tldr.zip"); println!(); @@ -43,7 +48,9 @@ fn main() -> io::Result<()> { * [Installation](installation.md)\n\ * [Build from source](build.md)\n\ * [Platform support](platforms.md)\n\ - * [Contributing](contributing.md)\n\ + * [Contributing](CONTRIBUTING.md)\n\ + \t* [Development](DEVELOPMENT.md)\n\ + \t* [Code of Conduct](CODE_OF_CONDUCT.md)\n\ * [GNU test coverage](test_coverage.md)\n\ * [Extensions](extensions.md)\n\ \n\ @@ -57,7 +64,7 @@ fn main() -> io::Result<()> { for platform in ["unix", "macos", "windows", "unix_android"] { let platform_utils: Vec = String::from_utf8( std::process::Command::new("./util/show-utils.sh") - .arg(format!("--features=feat_os_{}", platform)) + .arg(format!("--features=feat_os_{platform}")) .output()? .stdout, ) @@ -107,7 +114,7 @@ fn main() -> io::Result<()> { "| util | Linux | macOS | Windows | FreeBSD | Android |\n\ | ---------------- | ----- | ----- | ------- | ------- | ------- |" )?; - for (&name, _) in &utils { + for &(&name, _) in &utils { if name == "[" { continue; } @@ -131,7 +138,7 @@ fn main() -> io::Result<()> { if name == "[" { continue; } - let p = format!("docs/src/utils/{}.md", name); + let p = format!("docs/src/utils/{name}.md"); let markdown = File::open(format!("src/uu/{name}/{name}.md")) .and_then(|mut f: File| { @@ -151,11 +158,11 @@ fn main() -> io::Result<()> { markdown, } .markdown()?; - println!("Wrote to '{}'", p); + println!("Wrote to '{p}'"); } else { - println!("Error writing to {}", p); + println!("Error writing to {p}"); } - writeln!(summary, "* [{0}](utils/{0}.md)", name)?; + writeln!(summary, "* [{name}](utils/{name}.md)")?; } Ok(()) } @@ -169,7 +176,9 @@ struct MDWriter<'a, 'b> { markdown: Option, } -impl<'a, 'b> MDWriter<'a, 'b> { +impl MDWriter<'_, '_> { + /// # Errors + /// Returns an error if the writer fails. fn markdown(&mut self) -> io::Result<()> { write!(self.w, "# {}\n\n", self.name)?; self.additional()?; @@ -180,6 +189,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { self.examples() } + /// # Errors + /// Returns an error if the writer fails. fn additional(&mut self) -> io::Result<()> { writeln!(self.w, "
")?; self.platforms()?; @@ -187,6 +198,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { writeln!(self.w, "
") } + /// # Errors + /// Returns an error if the writer fails. fn platforms(&mut self) -> io::Result<()> { writeln!(self.w, "
")?; for (feature, icon) in [ @@ -201,7 +214,7 @@ impl<'a, 'b> MDWriter<'a, 'b> { .iter() .any(|u| u == self.name) { - writeln!(self.w, "", icon)?; + writeln!(self.w, "")?; } } writeln!(self.w, "
")?; @@ -209,6 +222,10 @@ impl<'a, 'b> MDWriter<'a, 'b> { Ok(()) } + /// # Errors + /// Returns an error if the writer fails. + /// # Panics + /// Panics if the version is not found. fn version(&mut self) -> io::Result<()> { writeln!( self.w, @@ -217,19 +234,23 @@ impl<'a, 'b> MDWriter<'a, 'b> { ) } + /// # Errors + /// Returns an error if the writer fails. fn usage(&mut self) -> io::Result<()> { if let Some(markdown) = &self.markdown { let usage = uuhelp_parser::parse_usage(markdown); let usage = usage.replace("{}", self.name); writeln!(self.w, "\n```")?; - writeln!(self.w, "{}", usage)?; + writeln!(self.w, "{usage}")?; writeln!(self.w, "```") } else { Ok(()) } } + /// # Errors + /// Returns an error if the writer fails. fn about(&mut self) -> io::Result<()> { if let Some(markdown) = &self.markdown { writeln!(self.w, "{}", uuhelp_parser::parse_about(markdown)) @@ -238,6 +259,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { } } + /// # Errors + /// Returns an error if the writer fails. fn after_help(&mut self) -> io::Result<()> { if let Some(markdown) = &self.markdown { if let Some(after_help) = uuhelp_parser::parse_section("after help", markdown) { @@ -248,6 +271,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { Ok(()) } + /// # Errors + /// Returns an error if the writer fails. fn examples(&mut self) -> io::Result<()> { if let Some(zip) = self.tldr_zip { let content = if let Some(f) = @@ -268,14 +293,14 @@ impl<'a, 'b> MDWriter<'a, 'b> { writeln!(self.w)?; for line in content.lines().skip_while(|l| !l.starts_with('-')) { if let Some(l) = line.strip_prefix("- ") { - writeln!(self.w, "{}", l)?; + writeln!(self.w, "{l}")?; } else if line.starts_with('`') { writeln!(self.w, "```shell\n{}\n```", line.trim_matches('`'))?; } else if line.is_empty() { writeln!(self.w)?; } else { println!("Not sure what to do with this line:"); - println!("{}", line); + println!("{line}"); } } writeln!(self.w)?; @@ -292,6 +317,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { Ok(()) } + /// # Errors + /// Returns an error if the writer fails. fn options(&mut self) -> io::Result<()> { writeln!(self.w, "

Options

")?; write!(self.w, "
")?; @@ -305,14 +332,14 @@ impl<'a, 'b> MDWriter<'a, 'b> { write!(self.w, ", ")?; } write!(self.w, "")?; - write!(self.w, "--{}", l)?; + write!(self.w, "--{l}")?; if let Some(names) = arg.get_value_names() { write!( self.w, "={}", names .iter() - .map(|x| format!("<{}>", x)) + .map(|x| format!("<{x}>")) .collect::>() .join(" ") )?; @@ -326,14 +353,14 @@ impl<'a, 'b> MDWriter<'a, 'b> { write!(self.w, ", ")?; } write!(self.w, "")?; - write!(self.w, "-{}", s)?; + write!(self.w, "-{s}")?; if let Some(names) = arg.get_value_names() { write!( self.w, " {}", names .iter() - .map(|x| format!("<{}>", x)) + .map(|x| format!("<{x}>")) .collect::>() .join(" ") )?; @@ -354,6 +381,8 @@ impl<'a, 'b> MDWriter<'a, 'b> { } } +/// # Panics +/// Panics if the archive is not ok fn get_zip_content(archive: &mut ZipArchive, name: &str) -> Option { let mut s = String::new(); archive.by_name(name).ok()?.read_to_string(&mut s).unwrap(); diff --git a/src/uu/arch/Cargo.toml b/src/uu/arch/Cargo.toml index cd12cb4d483..611aa6845cf 100644 --- a/src/uu/arch/Cargo.toml +++ b/src/uu/arch/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_arch" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "arch ~ (uutils) display machine architecture" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/arch" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/arch.rs" diff --git a/src/uu/arch/arch.md b/src/uu/arch/arch.md deleted file mode 100644 index a4ba2e75fce..00000000000 --- a/src/uu/arch/arch.md +++ /dev/null @@ -1,11 +0,0 @@ -# arch - -``` -arch -``` - -Display machine architecture - -## After Help - -Determine architecture name for current machine. diff --git a/src/uu/arch/locales/en-US.ftl b/src/uu/arch/locales/en-US.ftl new file mode 100644 index 00000000000..1646e50030d --- /dev/null +++ b/src/uu/arch/locales/en-US.ftl @@ -0,0 +1,5 @@ +# Error message when system architecture information cannot be retrieved +cannot-get-system = cannot get system name + +arch-about = Display machine architecture +arch-after-help = Determine architecture name for current machine. diff --git a/src/uu/arch/locales/fr-FR.ftl b/src/uu/arch/locales/fr-FR.ftl new file mode 100644 index 00000000000..f08462a87b4 --- /dev/null +++ b/src/uu/arch/locales/fr-FR.ftl @@ -0,0 +1,5 @@ +# Error message when system architecture information cannot be retrieved +cannot-get-system = impossible d'obtenir le nom du système + +arch-about = Afficher l'architecture de la machine +arch-after-help = Déterminer le nom de l'architecture pour la machine actuelle. diff --git a/src/uu/arch/src/arch.rs b/src/uu/arch/src/arch.rs index 0d71a818379..82e9bb79e5c 100644 --- a/src/uu/arch/src/arch.rs +++ b/src/uu/arch/src/arch.rs @@ -5,18 +5,17 @@ use platform_info::*; -use clap::{crate_version, Command}; +use clap::Command; use uucore::error::{UResult, USimpleError}; -use uucore::{help_about, help_section}; - -static ABOUT: &str = help_about!("arch.md"); -static SUMMARY: &str = help_section!("after help", "arch.md"); +use uucore::locale::{self, get_message}; #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { + locale::setup_localization(uucore::util_name())?; uu_app().try_get_matches_from(args)?; - let uts = PlatformInfo::new().map_err(|_e| USimpleError::new(1, "cannot get system name"))?; + let uts = + PlatformInfo::new().map_err(|_e| USimpleError::new(1, get_message("cannot-get-system")))?; println!("{}", uts.machine().to_string_lossy().trim()); Ok(()) @@ -24,8 +23,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) - .about(ABOUT) - .after_help(SUMMARY) + .version(uucore::crate_version!()) + .about(get_message("arch-about")) + .after_help(get_message("arch-after-help")) .infer_long_args(true) } diff --git a/src/uu/base32/Cargo.toml b/src/uu/base32/Cargo.toml index 044b92d23fc..42421311c0e 100644 --- a/src/uu/base32/Cargo.toml +++ b/src/uu/base32/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_base32" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "base32 ~ (uutils) decode/encode input (base32-encoding)" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/base32" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/base32.rs" diff --git a/src/uu/base32/src/base32.rs b/src/uu/base32/src/base32.rs index 09250421c25..e14e83921e2 100644 --- a/src/uu/base32/src/base32.rs +++ b/src/uu/base32/src/base32.rs @@ -3,13 +3,12 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -use std::io::{stdin, Read}; +pub mod base_common; +use base_common::ReadSeek; use clap::Command; use uucore::{encoding::Format, error::UResult, help_about, help_usage}; -pub mod base_common; - const ABOUT: &str = help_about!("base32.md"); const USAGE: &str = help_usage!("base32.md"); @@ -17,20 +16,11 @@ const USAGE: &str = help_usage!("base32.md"); pub fn uumain(args: impl uucore::Args) -> UResult<()> { let format = Format::Base32; - let config: base_common::Config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; + let config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; - // Create a reference to stdin so we can return a locked stdin from - // parse_base_cmd_args - let stdin_raw = stdin(); - let mut input: Box = base_common::get_input(&config, &stdin_raw)?; + let mut input: Box = base_common::get_input(&config)?; - base_common::handle_input( - &mut input, - format, - config.wrap_cols, - config.ignore_garbage, - config.decode, - ) + base_common::handle_input(&mut input, format, config) } pub fn uu_app() -> Command { diff --git a/src/uu/base32/src/base_common.rs b/src/uu/base32/src/base_common.rs index 4a30705afcd..05bfc89b28f 100644 --- a/src/uu/base32/src/base_common.rs +++ b/src/uu/base32/src/base_common.rs @@ -3,27 +3,35 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -use std::io::{stdout, Read, Write}; +// spell-checker:ignore hexupper lsbf msbf unpadded nopad aGVsbG8sIHdvcmxkIQ +use clap::{Arg, ArgAction, Command}; +use std::fs::File; +use std::io::{self, ErrorKind, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; use uucore::display::Quotable; -use uucore::encoding::{wrap_print, Data, Format}; +use uucore::encoding::{ + BASE2LSBF, BASE2MSBF, Format, Z85Wrapper, + for_base_common::{BASE32, BASE32HEX, BASE64, BASE64_NOPAD, BASE64URL, HEXUPPER_PERMISSIVE}, +}; +use uucore::encoding::{EncodingWrapper, SupportsFastDecodeAndEncode}; use uucore::error::{FromIo, UResult, USimpleError, UUsageError}; use uucore::format_usage; -use std::fs::File; -use std::io::{BufReader, Stdin}; -use std::path::Path; - -use clap::{crate_version, Arg, ArgAction, Command}; +pub const BASE_CMD_PARSE_ERROR: i32 = 1; -pub static BASE_CMD_PARSE_ERROR: i32 = 1; +/// Encoded output will be formatted in lines of this length (the last line can be shorter) +/// +/// Other implementations default to 76 +/// +/// This default is only used if no "-w"/"--wrap" argument is passed +pub const WRAP_DEFAULT: usize = 76; -// Config. pub struct Config { pub decode: bool, pub ignore_garbage: bool, pub wrap_cols: Option, - pub to_read: Option, + pub to_read: Option, } pub mod options { @@ -35,32 +43,36 @@ pub mod options { impl Config { pub fn from(options: &clap::ArgMatches) -> UResult { - let file: Option = match options.get_many::(options::FILE) { + let to_read = match options.get_many::(options::FILE) { Some(mut values) => { let name = values.next().unwrap(); + if let Some(extra_op) = values.next() { return Err(UUsageError::new( BASE_CMD_PARSE_ERROR, - format!("extra operand {}", extra_op.quote(),), + format!("extra operand {}", extra_op.quote()), )); } if name == "-" { None } else { - if !Path::exists(Path::new(name)) { + let path = Path::new(name); + + if !path.exists() { return Err(USimpleError::new( BASE_CMD_PARSE_ERROR, - format!("{}: No such file or directory", name.maybe_quote()), + format!("{}: No such file or directory", path.maybe_quote()), )); } - Some(name.to_owned()) + + Some(path.to_owned()) } } None => None, }; - let cols = options + let wrap_cols = options .get_one::(options::WRAP) .map(|num| { num.parse::().map_err(|_| { @@ -75,8 +87,8 @@ impl Config { Ok(Self { decode: options.get_flag(options::DECODE), ignore_garbage: options.get_flag(options::IGNORE_GARBAGE), - wrap_cols: cols, - to_read: file, + wrap_cols, + to_read, }) } } @@ -87,13 +99,12 @@ pub fn parse_base_cmd_args( usage: &str, ) -> UResult { let command = base_app(about, usage); - let arg_list = args.collect_lossy(); - Config::from(&command.try_get_matches_from(arg_list)?) + Config::from(&command.try_get_matches_from(args)?) } pub fn base_app(about: &'static str, usage: &str) -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(about) .override_usage(format_usage(usage)) .infer_long_args(true) @@ -101,84 +112,735 @@ pub fn base_app(about: &'static str, usage: &str) -> Command { .arg( Arg::new(options::DECODE) .short('d') + .visible_short_alias('D') .long(options::DECODE) .help("decode data") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with(options::DECODE), ) .arg( Arg::new(options::IGNORE_GARBAGE) .short('i') .long(options::IGNORE_GARBAGE) .help("when decoding, ignore non-alphabetic characters") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with(options::IGNORE_GARBAGE), ) .arg( Arg::new(options::WRAP) .short('w') .long(options::WRAP) .value_name("COLS") - .help( - "wrap encoded lines after COLS character (default 76, 0 to disable wrapping)", - ), + .help(format!("wrap encoded lines after COLS character (default {WRAP_DEFAULT}, 0 to disable wrapping)")) + .overrides_with(options::WRAP), ) // "multiple" arguments are used to check whether there is more than one // file passed in. .arg( Arg::new(options::FILE) .index(1) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) .value_hint(clap::ValueHint::FilePath), ) } -pub fn get_input<'a>(config: &Config, stdin_ref: &'a Stdin) -> UResult> { +/// A trait alias for types that implement both `Read` and `Seek`. +pub trait ReadSeek: Read + Seek {} + +/// Automatically implement the `ReadSeek` trait for any type that implements both `Read` and `Seek`. +impl ReadSeek for T {} + +pub fn get_input(config: &Config) -> UResult> { match &config.to_read { - Some(name) => { - let file_buf = - File::open(Path::new(name)).map_err_context(|| name.maybe_quote().to_string())?; - Ok(Box::new(BufReader::new(file_buf))) // as Box + Some(path_buf) => { + // Do not buffer input, because buffering is handled by `fast_decode` and `fast_encode` + let file = + File::open(path_buf).map_err_context(|| path_buf.maybe_quote().to_string())?; + Ok(Box::new(file)) } None => { - Ok(Box::new(stdin_ref.lock())) // as Box + let mut buffer = Vec::new(); + io::stdin().read_to_end(&mut buffer)?; + Ok(Box::new(io::Cursor::new(buffer))) } } } -pub fn handle_input( - input: &mut R, +/// Determines if the input buffer ends with padding ('=') after trimming trailing whitespace. +fn has_padding(input: &mut R) -> UResult { + let mut buf = Vec::new(); + input + .read_to_end(&mut buf) + .map_err(|err| USimpleError::new(1, format_read_error(err.kind())))?; + + // Reverse iterator and skip trailing whitespace without extra collections + let has_padding = buf + .iter() + .rfind(|&&byte| !byte.is_ascii_whitespace()) + .is_some_and(|&byte| byte == b'='); + + input.seek(SeekFrom::Start(0))?; + Ok(has_padding) +} + +pub fn handle_input(input: &mut R, format: Format, config: Config) -> UResult<()> { + let has_padding = has_padding(input)?; + + let supports_fast_decode_and_encode = + get_supports_fast_decode_and_encode(format, config.decode, has_padding); + + let supports_fast_decode_and_encode_ref = supports_fast_decode_and_encode.as_ref(); + + let mut stdout_lock = io::stdout().lock(); + + if config.decode { + fast_decode::fast_decode( + input, + &mut stdout_lock, + supports_fast_decode_and_encode_ref, + config.ignore_garbage, + ) + } else { + fast_encode::fast_encode( + input, + &mut stdout_lock, + supports_fast_decode_and_encode_ref, + config.wrap_cols, + ) + } +} + +pub fn get_supports_fast_decode_and_encode( format: Format, - line_wrap: Option, - ignore_garbage: bool, decode: bool, -) -> UResult<()> { - let mut data = Data::new(input, format).ignore_garbage(ignore_garbage); - if let Some(wrap) = line_wrap { - data = data.line_wrap(wrap); - } - - if decode { - match data.decode() { - Ok(s) => { - // Silent the warning as we want to the error message - #[allow(clippy::question_mark)] - if stdout().write_all(&s).is_err() { - // on windows console, writing invalid utf8 returns an error - return Err(USimpleError::new(1, "error: cannot write non-utf8 data")); + has_padding: bool, +) -> Box { + const BASE16_VALID_DECODING_MULTIPLE: usize = 2; + const BASE2_VALID_DECODING_MULTIPLE: usize = 8; + const BASE32_VALID_DECODING_MULTIPLE: usize = 8; + const BASE64_VALID_DECODING_MULTIPLE: usize = 4; + + const BASE16_UNPADDED_MULTIPLE: usize = 1; + const BASE2_UNPADDED_MULTIPLE: usize = 1; + const BASE32_UNPADDED_MULTIPLE: usize = 5; + const BASE64_UNPADDED_MULTIPLE: usize = 3; + + match format { + Format::Base16 => Box::from(EncodingWrapper::new( + HEXUPPER_PERMISSIVE, + BASE16_VALID_DECODING_MULTIPLE, + BASE16_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"0123456789ABCDEFabcdef", + )), + Format::Base2Lsbf => Box::from(EncodingWrapper::new( + BASE2LSBF, + BASE2_VALID_DECODING_MULTIPLE, + BASE2_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"01", + )), + Format::Base2Msbf => Box::from(EncodingWrapper::new( + BASE2MSBF, + BASE2_VALID_DECODING_MULTIPLE, + BASE2_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"01", + )), + Format::Base32 => Box::from(EncodingWrapper::new( + BASE32, + BASE32_VALID_DECODING_MULTIPLE, + BASE32_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567=", + )), + Format::Base32Hex => Box::from(EncodingWrapper::new( + BASE32HEX, + BASE32_VALID_DECODING_MULTIPLE, + BASE32_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"0123456789ABCDEFGHIJKLMNOPQRSTUV=", + )), + Format::Base64 => { + let alphabet: &[u8] = if has_padding { + &b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/="[..] + } else { + &b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/"[..] + }; + let wrapper = if decode && !has_padding { + BASE64_NOPAD + } else { + BASE64 + }; + Box::from(EncodingWrapper::new( + wrapper, + BASE64_VALID_DECODING_MULTIPLE, + BASE64_UNPADDED_MULTIPLE, + alphabet, + )) + } + Format::Base64Url => Box::from(EncodingWrapper::new( + BASE64URL, + BASE64_VALID_DECODING_MULTIPLE, + BASE64_UNPADDED_MULTIPLE, + // spell-checker:disable-next-line + b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789=_-", + )), + Format::Z85 => Box::from(Z85Wrapper {}), + } +} + +pub mod fast_encode { + use crate::base_common::{WRAP_DEFAULT, format_read_error}; + use std::{ + collections::VecDeque, + io::{self, ErrorKind, Read, Write}, + num::NonZeroUsize, + }; + use uucore::{ + encoding::SupportsFastDecodeAndEncode, + error::{UResult, USimpleError}, + }; + + struct LineWrapping { + line_length: NonZeroUsize, + print_buffer: Vec, + } + + // Start of helper functions + fn encode_in_chunks_to_buffer( + supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode, + encode_in_chunks_of_size: usize, + bytes_to_steal: usize, + read_buffer: &[u8], + encoded_buffer: &mut VecDeque, + leftover_buffer: &mut VecDeque, + ) -> UResult<()> { + let bytes_to_chunk = if bytes_to_steal > 0 { + let (stolen_bytes, rest_of_read_buffer) = read_buffer.split_at(bytes_to_steal); + + leftover_buffer.extend(stolen_bytes); + + // After appending the stolen bytes to `leftover_buffer`, it should be the right size + assert_eq!(leftover_buffer.len(), encode_in_chunks_of_size); + + // Encode the old unencoded data and the stolen bytes, and add the result to + // `encoded_buffer` + supports_fast_decode_and_encode + .encode_to_vec_deque(leftover_buffer.make_contiguous(), encoded_buffer)?; + + // Reset `leftover_buffer` + leftover_buffer.clear(); + + rest_of_read_buffer + } else { + // Do not need to steal bytes from `read_buffer` + read_buffer + }; + + let chunks_exact = bytes_to_chunk.chunks_exact(encode_in_chunks_of_size); + + let remainder = chunks_exact.remainder(); + + for sl in chunks_exact { + assert_eq!(sl.len(), encode_in_chunks_of_size); + + supports_fast_decode_and_encode.encode_to_vec_deque(sl, encoded_buffer)?; + } + + leftover_buffer.extend(remainder); + + Ok(()) + } + + fn write_without_line_breaks( + encoded_buffer: &mut VecDeque, + output: &mut dyn Write, + is_cleanup: bool, + empty_wrap: bool, + ) -> io::Result<()> { + // TODO + // `encoded_buffer` only has to be a VecDeque if line wrapping is enabled + // (`make_contiguous` should be a no-op here) + // Refactoring could avoid this call + output.write_all(encoded_buffer.make_contiguous())?; + + if is_cleanup { + if !empty_wrap { + output.write_all(b"\n")?; + } + } else { + encoded_buffer.clear(); + } + + Ok(()) + } + + fn write_with_line_breaks( + &mut LineWrapping { + ref line_length, + ref mut print_buffer, + }: &mut LineWrapping, + encoded_buffer: &mut VecDeque, + output: &mut dyn Write, + is_cleanup: bool, + ) -> io::Result<()> { + let line_length = line_length.get(); + + let make_contiguous_result = encoded_buffer.make_contiguous(); + + let chunks_exact = make_contiguous_result.chunks_exact(line_length); + + let mut bytes_added_to_print_buffer = 0; + + for sl in chunks_exact { + bytes_added_to_print_buffer += sl.len(); + + print_buffer.extend_from_slice(sl); + print_buffer.push(b'\n'); + } + + output.write_all(print_buffer)?; + + // Remove the bytes that were just printed from `encoded_buffer` + drop(encoded_buffer.drain(..bytes_added_to_print_buffer)); + + if is_cleanup { + if encoded_buffer.is_empty() { + // Do not write a newline in this case, because two trailing newlines should never be printed + } else { + // Print the partial line, since this is cleanup and no more data is coming + output.write_all(encoded_buffer.make_contiguous())?; + output.write_all(b"\n")?; + } + } else { + print_buffer.clear(); + } + + Ok(()) + } + + fn write_to_output( + line_wrapping: &mut Option, + encoded_buffer: &mut VecDeque, + output: &mut dyn Write, + is_cleanup: bool, + empty_wrap: bool, + ) -> io::Result<()> { + // Write all data in `encoded_buffer` to `output` + if let &mut Some(ref mut li) = line_wrapping { + write_with_line_breaks(li, encoded_buffer, output, is_cleanup)?; + } else { + write_without_line_breaks(encoded_buffer, output, is_cleanup, empty_wrap)?; + } + + Ok(()) + } + // End of helper functions + + pub fn fast_encode( + input: &mut dyn Read, + output: &mut dyn Write, + supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode, + wrap: Option, + ) -> UResult<()> { + // Based on performance testing + const INPUT_BUFFER_SIZE: usize = 32 * 1_024; + + const ENCODE_IN_CHUNKS_OF_SIZE_MULTIPLE: usize = 1_024; + + let encode_in_chunks_of_size = + supports_fast_decode_and_encode.unpadded_multiple() * ENCODE_IN_CHUNKS_OF_SIZE_MULTIPLE; + + assert!(encode_in_chunks_of_size > 0); + + // The "data-encoding" crate supports line wrapping, but not arbitrary line wrapping, only certain widths, so + // line wrapping must be handled here. + // https://github.com/ia0/data-encoding/blob/4f42ad7ef242f6d243e4de90cd1b46a57690d00e/lib/src/lib.rs#L1710 + let mut line_wrapping = match wrap { + // Line wrapping is disabled because "-w"/"--wrap" was passed with "0" + Some(0) => None, + // A custom line wrapping value was passed + Some(an) => Some(LineWrapping { + line_length: NonZeroUsize::new(an).unwrap(), + print_buffer: Vec::::new(), + }), + // Line wrapping was not set, so the default is used + None => Some(LineWrapping { + line_length: NonZeroUsize::new(WRAP_DEFAULT).unwrap(), + print_buffer: Vec::::new(), + }), + }; + + // Start of buffers + // Data that was read from `input` + let mut input_buffer = vec![0; INPUT_BUFFER_SIZE]; + + assert!(!input_buffer.is_empty()); + + // Data that was read from `input` but has not been encoded yet + let mut leftover_buffer = VecDeque::::new(); + + // Encoded data that needs to be written to `output` + let mut encoded_buffer = VecDeque::::new(); + // End of buffers + + loop { + match input.read(&mut input_buffer) { + Ok(bytes_read_from_input) => { + if bytes_read_from_input == 0 { + break; + } + + // The part of `input_buffer` that was actually filled by the call to `read` + let read_buffer = &input_buffer[..bytes_read_from_input]; + + // How many bytes to steal from `read_buffer` to get `leftover_buffer` to the right size + let bytes_to_steal = encode_in_chunks_of_size - leftover_buffer.len(); + + if bytes_to_steal > bytes_read_from_input { + // Do not have enough data to encode a chunk, so copy data to `leftover_buffer` and read more + leftover_buffer.extend(read_buffer); + + assert!(leftover_buffer.len() < encode_in_chunks_of_size); + + continue; + } + + // Encode data in chunks, then place it in `encoded_buffer` + encode_in_chunks_to_buffer( + supports_fast_decode_and_encode, + encode_in_chunks_of_size, + bytes_to_steal, + read_buffer, + &mut encoded_buffer, + &mut leftover_buffer, + )?; + + assert!(leftover_buffer.len() < encode_in_chunks_of_size); + // Write all data in `encoded_buffer` to `output` + write_to_output( + &mut line_wrapping, + &mut encoded_buffer, + output, + false, + wrap == Some(0), + )?; + } + Err(er) => { + let kind = er.kind(); + + if kind == ErrorKind::Interrupted { + // Retry reading + continue; + } + + return Err(USimpleError::new(1, format_read_error(kind))); } - Ok(()) } - Err(_) => Err(USimpleError::new(1, "error: invalid input")), } - } else { - match data.encode() { - Ok(s) => { - wrap_print(&data, &s); - Ok(()) + + // Cleanup + // `input` has finished producing data, so the data remaining in the buffers needs to be encoded and printed + { + // Encode all remaining unencoded bytes, placing them in `encoded_buffer` + supports_fast_decode_and_encode + .encode_to_vec_deque(leftover_buffer.make_contiguous(), &mut encoded_buffer)?; + + // Write all data in `encoded_buffer` to output + // `is_cleanup` triggers special cleanup-only logic + write_to_output( + &mut line_wrapping, + &mut encoded_buffer, + output, + true, + wrap == Some(0), + )?; + } + + Ok(()) + } +} + +pub mod fast_decode { + use crate::base_common::format_read_error; + use std::io::{self, ErrorKind, Read, Write}; + use uucore::{ + encoding::SupportsFastDecodeAndEncode, + error::{UResult, USimpleError}, + }; + + // Start of helper functions + fn alphabet_to_table(alphabet: &[u8], ignore_garbage: bool) -> [bool; 256] { + // If `ignore_garbage` is enabled, all characters outside the alphabet are ignored + // If it is not enabled, only '\n' and '\r' are ignored + if ignore_garbage { + // Note: "false" here + let mut table = [false; 256]; + + // Pass through no characters except those in the alphabet + for ue in alphabet { + let us = usize::from(*ue); + + // Should not have been set yet + assert!(!table[us]); + + table[us] = true; + } + + table + } else { + // Note: "true" here + let mut table = [true; 256]; + + // Pass through all characters except '\n' and '\r' + for ue in [b'\n', b'\r'] { + let us = usize::from(ue); + + // Should not have been set yet + assert!(table[us]); + + table[us] = false; + } + + table + } + } + + fn decode_in_chunks_to_buffer( + supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode, + decode_in_chunks_of_size: usize, + bytes_to_steal: usize, + read_buffer_filtered: &[u8], + decoded_buffer: &mut Vec, + leftover_buffer: &mut Vec, + ) -> UResult<()> { + let bytes_to_chunk = if bytes_to_steal > 0 { + let (stolen_bytes, rest_of_read_buffer_filtered) = + read_buffer_filtered.split_at(bytes_to_steal); + + leftover_buffer.extend(stolen_bytes); + + // After appending the stolen bytes to `leftover_buffer`, it should be the right size + assert_eq!(leftover_buffer.len(), decode_in_chunks_of_size); + + // Decode the old un-decoded data and the stolen bytes, and add the result to + // `decoded_buffer` + supports_fast_decode_and_encode.decode_into_vec(leftover_buffer, decoded_buffer)?; + + // Reset `leftover_buffer` + leftover_buffer.clear(); + + rest_of_read_buffer_filtered + } else { + // Do not need to steal bytes from `read_buffer` + read_buffer_filtered + }; + + let chunks_exact = bytes_to_chunk.chunks_exact(decode_in_chunks_of_size); + + let remainder = chunks_exact.remainder(); + + for sl in chunks_exact { + assert_eq!(sl.len(), decode_in_chunks_of_size); + + supports_fast_decode_and_encode.decode_into_vec(sl, decoded_buffer)?; + } + + leftover_buffer.extend(remainder); + + Ok(()) + } + + fn write_to_output(decoded_buffer: &mut Vec, output: &mut dyn Write) -> io::Result<()> { + // Write all data in `decoded_buffer` to `output` + output.write_all(decoded_buffer.as_slice())?; + + decoded_buffer.clear(); + + Ok(()) + } + // End of helper functions + + pub fn fast_decode( + input: &mut dyn Read, + output: &mut dyn Write, + supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode, + ignore_garbage: bool, + ) -> UResult<()> { + // Based on performance testing + const INPUT_BUFFER_SIZE: usize = 32 * 1_024; + + const DECODE_IN_CHUNKS_OF_SIZE_MULTIPLE: usize = 1_024; + + let alphabet = supports_fast_decode_and_encode.alphabet(); + let decode_in_chunks_of_size = supports_fast_decode_and_encode.valid_decoding_multiple() + * DECODE_IN_CHUNKS_OF_SIZE_MULTIPLE; + + assert!(decode_in_chunks_of_size > 0); + + // Note that it's not worth using "data-encoding"'s ignore functionality if `ignore_garbage` is true, because + // "data-encoding"'s ignore functionality cannot discard non-ASCII bytes. The data has to be filtered before + // passing it to "data-encoding", so there is no point in doing any filtering in "data-encoding". This also + // allows execution to stay on the happy path in "data-encoding": + // https://github.com/ia0/data-encoding/blob/4f42ad7ef242f6d243e4de90cd1b46a57690d00e/lib/src/lib.rs#L754-L756 + // It is also not worth using "data-encoding"'s ignore functionality when `ignore_garbage` is + // false. + // Note that the alphabet constants above already include the padding characters + // TODO + // Precompute this + let table = alphabet_to_table(alphabet, ignore_garbage); + + // Start of buffers + // Data that was read from `input` + let mut input_buffer = vec![0; INPUT_BUFFER_SIZE]; + + assert!(!input_buffer.is_empty()); + + // Data that was read from `input` but has not been decoded yet + let mut leftover_buffer = Vec::::new(); + + // Decoded data that needs to be written to `output` + let mut decoded_buffer = Vec::::new(); + + // Buffer that will be used when `ignore_garbage` is true, and the chunk read from `input` contains garbage + // data + let mut non_garbage_buffer = Vec::::new(); + // End of buffers + + loop { + match input.read(&mut input_buffer) { + Ok(bytes_read_from_input) => { + if bytes_read_from_input == 0 { + break; + } + + let read_buffer_filtered = { + // The part of `input_buffer` that was actually filled by the call to `read` + let read_buffer = &input_buffer[..bytes_read_from_input]; + + // First just scan the data for the happy path + // Yields significant speedup when the input does not contain line endings + let found_garbage = read_buffer.iter().any(|ue| { + // Garbage, since it was not found in the table + !table[usize::from(*ue)] + }); + + if found_garbage { + non_garbage_buffer.clear(); + + for ue in read_buffer { + if table[usize::from(*ue)] { + // Not garbage, since it was found in the table + non_garbage_buffer.push(*ue); + } + } + + non_garbage_buffer.as_slice() + } else { + read_buffer + } + }; + + // How many bytes to steal from `read_buffer` to get `leftover_buffer` to the right size + let bytes_to_steal = decode_in_chunks_of_size - leftover_buffer.len(); + + if bytes_to_steal > read_buffer_filtered.len() { + // Do not have enough data to decode a chunk, so copy data to `leftover_buffer` and read more + leftover_buffer.extend(read_buffer_filtered); + + assert!(leftover_buffer.len() < decode_in_chunks_of_size); + + continue; + } + + // Decode data in chunks, then place it in `decoded_buffer` + decode_in_chunks_to_buffer( + supports_fast_decode_and_encode, + decode_in_chunks_of_size, + bytes_to_steal, + read_buffer_filtered, + &mut decoded_buffer, + &mut leftover_buffer, + )?; + + assert!(leftover_buffer.len() < decode_in_chunks_of_size); + + // Write all data in `decoded_buffer` to `output` + write_to_output(&mut decoded_buffer, output)?; + } + Err(er) => { + let kind = er.kind(); + + if kind == ErrorKind::Interrupted { + // Retry reading + continue; + } + + return Err(USimpleError::new(1, format_read_error(kind))); + } + } + } + + // Cleanup + // `input` has finished producing data, so the data remaining in the buffers needs to be decoded and printed + { + // Decode all remaining encoded bytes, placing them in `decoded_buffer` + supports_fast_decode_and_encode + .decode_into_vec(&leftover_buffer, &mut decoded_buffer)?; + + // Write all data in `decoded_buffer` to `output` + write_to_output(&mut decoded_buffer, output)?; + } + + Ok(()) + } +} + +fn format_read_error(kind: ErrorKind) -> String { + let kind_string = kind.to_string(); + + // e.g. "is a directory" -> "Is a directory" + let mut kind_string_capitalized = String::with_capacity(kind_string.len()); + + for (index, ch) in kind_string.char_indices() { + if index == 0 { + for cha in ch.to_uppercase() { + kind_string_capitalized.push(cha); } - Err(_) => Err(USimpleError::new( - 1, - "error: invalid input (length must be multiple of 4 characters)", - )), + } else { + kind_string_capitalized.push(ch); + } + } + + format!("read error: {kind_string_capitalized}") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn test_has_padding() { + let test_cases = vec![ + ("aGVsbG8sIHdvcmxkIQ==", true), + ("aGVsbG8sIHdvcmxkIQ== ", true), + ("aGVsbG8sIHdvcmxkIQ==\n", true), + ("aGVsbG8sIHdvcmxkIQ== \n", true), + ("aGVsbG8sIHdvcmxkIQ=", true), + ("aGVsbG8sIHdvcmxkIQ= ", true), + ("aGVsbG8sIHdvcmxkIQ \n", false), + ("aGVsbG8sIHdvcmxkIQ", false), + ]; + + for (input, expected) in test_cases { + let mut cursor = Cursor::new(input.as_bytes()); + assert_eq!( + has_padding(&mut cursor).unwrap(), + expected, + "Failed for input: '{input}'" + ); } } } diff --git a/src/uu/base64/Cargo.toml b/src/uu/base64/Cargo.toml index 3bc04515280..aa899f1a1e6 100644 --- a/src/uu/base64/Cargo.toml +++ b/src/uu/base64/Cargo.toml @@ -1,20 +1,24 @@ [package] name = "uu_base64" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "base64 ~ (uutils) decode/encode input (base64-encoding)" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/base64" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/base64.rs" [dependencies] +clap = { workspace = true } uucore = { workspace = true, features = ["encoding"] } uu_base32 = { workspace = true } diff --git a/src/uu/base64/src/base64.rs b/src/uu/base64/src/base64.rs index 6544638bdae..86eb75bf119 100644 --- a/src/uu/base64/src/base64.rs +++ b/src/uu/base64/src/base64.rs @@ -3,13 +3,10 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +use clap::Command; use uu_base32::base_common; -pub use uu_base32::uu_app; - use uucore::{encoding::Format, error::UResult, help_about, help_usage}; -use std::io::{stdin, Read}; - const ABOUT: &str = help_about!("base64.md"); const USAGE: &str = help_usage!("base64.md"); @@ -17,18 +14,13 @@ const USAGE: &str = help_usage!("base64.md"); pub fn uumain(args: impl uucore::Args) -> UResult<()> { let format = Format::Base64; - let config: base_common::Config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; + let config = base_common::parse_base_cmd_args(args, ABOUT, USAGE)?; - // Create a reference to stdin so we can return a locked stdin from - // parse_base_cmd_args - let stdin_raw = stdin(); - let mut input: Box = base_common::get_input(&config, &stdin_raw)?; + let mut input = base_common::get_input(&config)?; + + base_common::handle_input(&mut input, format, config) +} - base_common::handle_input( - &mut input, - format, - config.wrap_cols, - config.ignore_garbage, - config.decode, - ) +pub fn uu_app() -> Command { + base_common::base_app(ABOUT, USAGE) } diff --git a/src/uu/basename/Cargo.toml b/src/uu/basename/Cargo.toml index 878a5d9e102..5123174ae2d 100644 --- a/src/uu/basename/Cargo.toml +++ b/src/uu/basename/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_basename" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "basename ~ (uutils) display PATHNAME with leading directory components removed" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/basename" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/basename.rs" diff --git a/src/uu/basename/basename.md b/src/uu/basename/basename.md index b17cac74a00..ee87fa76d4e 100644 --- a/src/uu/basename/basename.md +++ b/src/uu/basename/basename.md @@ -1,7 +1,7 @@ # basename ``` -basename NAME [SUFFIX] +basename [-z] NAME [SUFFIX] basename OPTION... NAME... ``` diff --git a/src/uu/basename/src/basename.rs b/src/uu/basename/src/basename.rs index 6c9baca6fce..a40fcc18534 100644 --- a/src/uu/basename/src/basename.rs +++ b/src/uu/basename/src/basename.rs @@ -5,8 +5,8 @@ // spell-checker:ignore (ToDO) fullname -use clap::{crate_version, Arg, ArgAction, Command}; -use std::path::{is_separator, PathBuf}; +use clap::{Arg, ArgAction, Command}; +use std::path::{PathBuf, is_separator}; use uucore::display::Quotable; use uucore::error::{UResult, UUsageError}; use uucore::line_ending::LineEnding; @@ -27,86 +27,48 @@ pub mod options { pub fn uumain(args: impl uucore::Args) -> UResult<()> { let args = args.collect_lossy(); - // Since options have to go before names, - // if the first argument is not an option, then there is no option, - // and that implies there is exactly one name (no option => no -a option), - // so simple format is used - if args.len() > 1 && !args[1].starts_with('-') { - if args.len() > 3 { - return Err(UUsageError::new( - 1, - format!("extra operand {}", args[3].to_string().quote()), - )); - } - let suffix = if args.len() > 2 { args[2].as_ref() } else { "" }; - println!("{}", basename(&args[1], suffix)); - return Ok(()); - } - // // Argument parsing // let matches = uu_app().try_get_matches_from(args)?; - // too few arguments - if !matches.contains_id(options::NAME) { - return Err(UUsageError::new(1, "missing operand".to_string())); - } - let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO)); - let opt_suffix = matches.get_one::(options::SUFFIX).is_some(); - let opt_multiple = matches.get_flag(options::MULTIPLE); - let multiple_paths = opt_suffix || opt_multiple; - let name_args_count = matches + let mut name_args = matches .get_many::(options::NAME) - .map(|n| n.len()) - .unwrap_or(0); - - // too many arguments - if !multiple_paths && name_args_count > 2 { - return Err(UUsageError::new( - 1, - format!( - "extra operand {}", - matches - .get_many::(options::NAME) - .unwrap() - .nth(2) - .unwrap() - .quote() - ), - )); + .unwrap_or_default() + .collect::>(); + if name_args.is_empty() { + return Err(UUsageError::new(1, "missing operand".to_string())); } - - let suffix = if opt_suffix { - matches.get_one::(options::SUFFIX).unwrap() - } else if !opt_multiple && name_args_count > 1 { + let multiple_paths = + matches.get_one::(options::SUFFIX).is_some() || matches.get_flag(options::MULTIPLE); + let suffix = if multiple_paths { matches - .get_many::(options::NAME) - .unwrap() - .nth(1) - .unwrap() + .get_one::(options::SUFFIX) + .cloned() + .unwrap_or_default() } else { - "" + // "simple format" + match name_args.len() { + 0 => panic!("already checked"), + 1 => String::default(), + 2 => name_args.pop().unwrap().clone(), + _ => { + return Err(UUsageError::new( + 1, + format!("extra operand {}", name_args[2].quote()), + )); + } + } }; // // Main Program Processing // - let paths: Vec<_> = if multiple_paths { - matches.get_many::(options::NAME).unwrap().collect() - } else { - matches - .get_many::(options::NAME) - .unwrap() - .take(1) - .collect() - }; - - for path in paths { - print!("{}{}", basename(path, suffix), line_ending); + for path in name_args { + print!("{}{line_ending}", basename(path, &suffix)); } Ok(()) @@ -114,7 +76,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -123,27 +85,31 @@ pub fn uu_app() -> Command { .short('a') .long(options::MULTIPLE) .help("support multiple arguments and treat each as a NAME") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with(options::MULTIPLE), ) .arg( Arg::new(options::NAME) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) .value_hint(clap::ValueHint::AnyPath) - .hide(true), + .hide(true) + .trailing_var_arg(true), ) .arg( Arg::new(options::SUFFIX) .short('s') .long(options::SUFFIX) .value_name("SUFFIX") - .help("remove a trailing SUFFIX; implies -a"), + .help("remove a trailing SUFFIX; implies -a") + .overrides_with(options::SUFFIX), ) .arg( Arg::new(options::ZERO) .short('z') .long(options::ZERO) .help("end each output line with NUL, not newline") - .action(ArgAction::SetTrue), + .action(ArgAction::SetTrue) + .overrides_with(options::ZERO), ) } @@ -159,16 +125,13 @@ fn basename(fullname: &str, suffix: &str) -> String { // Convert to path buffer and get last path component let pb = PathBuf::from(path); - match pb.components().last() { - Some(c) => { - let name = c.as_os_str().to_str().unwrap(); - if name == suffix { - name.to_string() - } else { - name.strip_suffix(suffix).unwrap_or(name).to_string() - } - } - None => String::new(), - } + pb.components().next_back().map_or_else(String::new, |c| { + let name = c.as_os_str().to_str().unwrap(); + if name == suffix { + name.to_string() + } else { + name.strip_suffix(suffix).unwrap_or(name).to_string() + } + }) } diff --git a/src/uu/basenc/BENCHMARKING.md b/src/uu/basenc/BENCHMARKING.md new file mode 100644 index 00000000000..8248cbbc53b --- /dev/null +++ b/src/uu/basenc/BENCHMARKING.md @@ -0,0 +1,177 @@ + + +# Benchmarking base32, base64, and basenc + +Note that the functionality of the `base32` and `base64` programs is identical to that of the `basenc` program, using +the "--base32" and "--base64" options, respectively. For that reason, it is only necessary to benchmark `basenc`. + +To compare the runtime performance of the uutils implementation with the GNU Core Utilities implementation, you can +use a benchmarking tool like [hyperfine][0]. + +hyperfine currently does not measure maximum memory usage. Memory usage can be benchmarked using [poop][2], or +[toybox][3]'s "time" subcommand (both are Linux only). + +Build the `basenc` binary using the release profile: + +```Shell +cargo build --package uu_basenc --profile release +``` + +## Expected performance + +uutils' `basenc` performs streaming decoding and encoding, and therefore should perform all operations with a constant +maximum memory usage, regardless of the size of the input. Release builds currently use less than 3 mebibytes of +memory, and memory usage greater than 10 mebibytes should be considered a bug. + +As of September 2024, uutils' `basenc` has runtime performance equal to or superior to GNU Core Utilities' `basenc` in +in most scenarios. uutils' `basenc` uses slightly more memory, but given how small these quantities are in absolute +terms (see above), this is highly unlikely to be practically relevant to users. + +## Benchmark results (2024-09-27) + +### Setup + +```Shell +# Use uutils' dd to create a 1 gibibyte in-memory file filled with random bytes (Linux only). +# On other platforms, you can use /tmp instead of /dev/shm, but note that /tmp is not guaranteed to be in-memory. +coreutils dd if=/dev/urandom of=/dev/shm/one-random-gibibyte bs=1024 count=1048576 + +# Encode this file for use in decoding performance testing +/usr/bin/basenc --base32hex -- /dev/shm/one-random-gibibyte 1>/dev/shm/one-random-gibibyte-base32hex-encoded +/usr/bin/basenc --z85 -- /dev/shm/one-random-gibibyte 1>/dev/shm/one-random-gibibyte-z85-encoded +``` + +### Programs being tested + +uutils' `basenc`: + +``` +⯠git rev-list HEAD | coreutils head -n 1 -- - +a0718ef0ffd50539a2e2bc0095c9fadcd70ab857 +``` + +GNU Core Utilities' `basenc`: + +``` +⯠/usr/bin/basenc --version | coreutils head -n 1 -- - +basenc (GNU coreutils) 9.4 +``` + +### Encoding performance + +#### "--base64", default line wrapping (76 characters) + +➕ Faster than GNU Core Utilities + +``` +⯠hyperfine \ + --sort \ + command \ + -- \ + '/usr/bin/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null' \ + './target/release/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null' + +Benchmark 1: /usr/bin/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null + Time (mean ± σ): 965.1 ms ± 7.9 ms [User: 766.2 ms, System: 193.4 ms] + Range (min … max): 950.2 ms … 976.9 ms 10 runs + +Benchmark 2: ./target/release/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null + Time (mean ± σ): 696.6 ms ± 9.1 ms [User: 574.9 ms, System: 117.3 ms] + Range (min … max): 683.1 ms … 713.5 ms 10 runs + +Relative speed comparison + 1.39 ± 0.02 /usr/bin/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null + 1.00 ./target/release/basenc --base64 -- /dev/shm/one-random-gibibyte 1>/dev/null +``` + +#### "--base16", no line wrapping + +➖ Slower than GNU Core Utilities + +``` +⯠poop \ + '/usr/bin/basenc --base16 --wrap 0 -- /dev/shm/one-random-gibibyte' \ + './target/release/basenc --base16 --wrap 0 -- /dev/shm/one-random-gibibyte' + +Benchmark 1 (6 runs): /usr/bin/basenc --base16 --wrap 0 -- /dev/shm/one-random-gibibyte + measurement mean ± σ min … max outliers delta + wall_time 836ms ± 13.3ms 822ms … 855ms 0 ( 0%) 0% + peak_rss 2.05MB ± 73.0KB 1.94MB … 2.12MB 0 ( 0%) 0% + cpu_cycles 2.85G ± 32.8M 2.82G … 2.91G 0 ( 0%) 0% + instructions 14.0G ± 58.7 14.0G … 14.0G 0 ( 0%) 0% + cache_references 70.0M ± 6.48M 63.7M … 78.8M 0 ( 0%) 0% + cache_misses 582K ± 172K 354K … 771K 0 ( 0%) 0% + branch_misses 667K ± 4.55K 662K … 674K 0 ( 0%) 0% +Benchmark 2 (6 runs): ./target/release/basenc --base16 --wrap 0 -- /dev/shm/one-random-gibibyte + measurement mean ± σ min … max outliers delta + wall_time 884ms ± 6.38ms 878ms … 895ms 0 ( 0%) 💩+ 5.7% ± 1.6% + peak_rss 2.65MB ± 66.8KB 2.55MB … 2.74MB 0 ( 0%) 💩+ 29.3% ± 4.4% + cpu_cycles 3.15G ± 8.61M 3.14G … 3.16G 0 ( 0%) 💩+ 10.6% ± 1.1% + instructions 10.5G ± 275 10.5G … 10.5G 0 ( 0%) ⚡- 24.9% ± 0.0% + cache_references 93.5M ± 6.10M 87.2M … 104M 0 ( 0%) 💩+ 33.7% ± 11.6% + cache_misses 415K ± 52.3K 363K … 474K 0 ( 0%) - 28.8% ± 28.0% + branch_misses 1.43M ± 4.82K 1.42M … 1.43M 0 ( 0%) 💩+113.9% ± 0.9% +``` + +### Decoding performance + +#### "--base32hex" + +➕ Faster than GNU Core Utilities + +``` +⯠hyperfine \ + --sort \ + command \ + -- \ + '/usr/bin/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null' \ + './target/release/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null' + +Benchmark 1: /usr/bin/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null + Time (mean ± σ): 7.154 s ± 0.082 s [User: 6.802 s, System: 0.323 s] + Range (min … max): 7.051 s … 7.297 s 10 runs + +Benchmark 2: ./target/release/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null + Time (mean ± σ): 2.679 s ± 0.025 s [User: 2.446 s, System: 0.221 s] + Range (min … max): 2.649 s … 2.718 s 10 runs + +Relative speed comparison + 2.67 ± 0.04 /usr/bin/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null + 1.00 ./target/release/basenc --base32hex --decode -- /dev/shm/one-random-gibibyte-base32hex-encoded 1>/dev/null +``` + +#### "--z85", with "--ignore-garbage" + +➕ Faster than GNU Core Utilities + +``` +⯠poop \ + '/usr/bin/basenc --decode --ignore-garbage --z85 -- /dev/shm/one-random-gibibyte-z85-encoded' \ + './target/release/basenc --decode --ignore-garbage --z85 -- /dev/shm/one-random-gibibyte-z85-encoded' + +Benchmark 1 (3 runs): /usr/bin/basenc --decode --ignore-garbage --z85 -- /dev/shm/one-random-gibibyte-z85-encoded + measurement mean ± σ min … max outliers delta + wall_time 14.4s ± 68.4ms 14.3s … 14.4s 0 ( 0%) 0% + peak_rss 1.98MB ± 10.8KB 1.97MB … 1.99MB 0 ( 0%) 0% + cpu_cycles 58.4G ± 211M 58.3G … 58.7G 0 ( 0%) 0% + instructions 74.7G ± 64.0 74.7G … 74.7G 0 ( 0%) 0% + cache_references 41.8M ± 624K 41.2M … 42.4M 0 ( 0%) 0% + cache_misses 693K ± 118K 567K … 802K 0 ( 0%) 0% + branch_misses 1.24G ± 183K 1.24G … 1.24G 0 ( 0%) 0% +Benchmark 2 (3 runs): ./target/release/basenc --decode --ignore-garbage --z85 -- /dev/shm/one-random-gibibyte-z85-encoded + measurement mean ± σ min … max outliers delta + wall_time 2.80s ± 17.9ms 2.79s … 2.82s 0 ( 0%) ⚡- 80.5% ± 0.8% + peak_rss 2.61MB ± 67.4KB 2.57MB … 2.69MB 0 ( 0%) 💩+ 31.9% ± 5.5% + cpu_cycles 10.8G ± 27.9M 10.8G … 10.9G 0 ( 0%) ⚡- 81.5% ± 0.6% + instructions 39.0G ± 353 39.0G … 39.0G 0 ( 0%) ⚡- 47.7% ± 0.0% + cache_references 114M ± 2.43M 112M … 116M 0 ( 0%) 💩+173.3% ± 9.6% + cache_misses 1.06M ± 288K 805K … 1.37M 0 ( 0%) + 52.6% ± 72.0% + branch_misses 1.18M ± 14.7K 1.16M … 1.19M 0 ( 0%) ⚡- 99.9% ± 0.0% +``` + +[0]: https://github.com/sharkdp/hyperfine +[1]: https://github.com/sharkdp/hyperfine?tab=readme-ov-file#installation +[2]: https://github.com/andrewrk/poop +[3]: https://landley.net/toybox/ diff --git a/src/uu/basenc/Cargo.toml b/src/uu/basenc/Cargo.toml index 92912e5dfbf..2f78a95751c 100644 --- a/src/uu/basenc/Cargo.toml +++ b/src/uu/basenc/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_basenc" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "basenc ~ (uutils) decode/encode input" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/basenc" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/basenc.rs" diff --git a/src/uu/basenc/basenc.md b/src/uu/basenc/basenc.md index 17916bd4ab8..001babe9e6b 100644 --- a/src/uu/basenc/basenc.md +++ b/src/uu/basenc/basenc.md @@ -1,7 +1,7 @@ # basenc ``` -basenc [OPTION]... [FILE]" +basenc [OPTION]... [FILE] ``` Encode/decode data and print to standard output diff --git a/src/uu/basenc/src/basenc.rs b/src/uu/basenc/src/basenc.rs index ff512b17652..10090765232 100644 --- a/src/uu/basenc/src/basenc.rs +++ b/src/uu/basenc/src/basenc.rs @@ -3,19 +3,15 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -//spell-checker:ignore (args) lsbf msbf +// spell-checker:ignore lsbf msbf use clap::{Arg, ArgAction, Command}; -use uu_base32::base_common::{self, Config, BASE_CMD_PARSE_ERROR}; - +use uu_base32::base_common::{self, BASE_CMD_PARSE_ERROR, Config}; +use uucore::error::UClapError; use uucore::{ encoding::Format, error::{UResult, UUsageError}, }; - -use std::io::{stdin, Read}; -use uucore::error::UClapError; - use uucore::{help_about, help_usage}; const ABOUT: &str = help_about!("basenc.md"); @@ -53,12 +49,14 @@ const ENCODINGS: &[(&str, Format, &str)] = &[ pub fn uu_app() -> Command { let mut command = base_common::base_app(ABOUT, USAGE); for encoding in ENCODINGS { - command = command.arg( - Arg::new(encoding.0) - .long(encoding.0) - .help(encoding.2) - .action(ArgAction::SetTrue), - ); + let raw_arg = Arg::new(encoding.0) + .long(encoding.0) + .help(encoding.2) + .action(ArgAction::SetTrue); + let overriding_arg = ENCODINGS + .iter() + .fold(raw_arg, |arg, enc| arg.overrides_with(enc.0)); + command = command.arg(overriding_arg); } command } @@ -79,16 +77,8 @@ fn parse_cmd_args(args: impl uucore::Args) -> UResult<(Config, Format)> { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let (config, format) = parse_cmd_args(args)?; - // Create a reference to stdin so we can return a locked stdin from - // parse_base_cmd_args - let stdin_raw = stdin(); - let mut input: Box = base_common::get_input(&config, &stdin_raw)?; - base_common::handle_input( - &mut input, - format, - config.wrap_cols, - config.ignore_garbage, - config.decode, - ) + let mut input = base_common::get_input(&config)?; + + base_common::handle_input(&mut input, format, config) } diff --git a/src/uu/cat/Cargo.toml b/src/uu/cat/Cargo.toml index d67cd1b254d..f5ac6a64eff 100644 --- a/src/uu/cat/Cargo.toml +++ b/src/uu/cat/Cargo.toml @@ -1,24 +1,27 @@ [package] name = "uu_cat" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "cat ~ (uutils) concatenate and display input" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/cat" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/cat.rs" [dependencies] clap = { workspace = true } +memchr = { workspace = true } thiserror = { workspace = true } -is-terminal = { workspace = true } -uucore = { workspace = true, features = ["fs", "pipes"] } +uucore = { workspace = true, features = ["fast-inc", "fs", "pipes"] } [target.'cfg(unix)'.dependencies] nix = { workspace = true } diff --git a/src/uu/cat/src/cat.rs b/src/uu/cat/src/cat.rs index 8ce8052517b..45fbe6cebf3 100644 --- a/src/uu/cat/src/cat.rs +++ b/src/uu/cat/src/cat.rs @@ -3,37 +3,88 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) nonprint nonblank nonprinting +// spell-checker:ignore (ToDO) nonprint nonblank nonprinting ELOOP +use std::fs::{File, metadata}; +use std::io::{self, BufWriter, IsTerminal, Read, Write}; +/// Unix domain socket support +#[cfg(unix)] +use std::net::Shutdown; +#[cfg(unix)] +use std::os::fd::AsFd; +#[cfg(unix)] +use std::os::unix::fs::FileTypeExt; +#[cfg(unix)] +use std::os::unix::net::UnixStream; -// last synced with: cat (GNU coreutils) 8.13 -use clap::{crate_version, Arg, ArgAction, Command}; -use is_terminal::IsTerminal; -use std::fs::{metadata, File}; -use std::io::{self, Read, Write}; +use clap::{Arg, ArgAction, Command}; +use memchr::memchr2; +#[cfg(unix)] +use nix::fcntl::{FcntlArg, fcntl}; use thiserror::Error; use uucore::display::Quotable; use uucore::error::UResult; use uucore::fs::FileInformation; - -#[cfg(unix)] -use std::os::unix::io::AsRawFd; +use uucore::{fast_inc::fast_inc_one, format_usage, help_about, help_usage}; /// Linux splice support #[cfg(any(target_os = "linux", target_os = "android"))] mod splice; -/// Unix domain socket support -#[cfg(unix)] -use std::net::Shutdown; -#[cfg(unix)] -use std::os::unix::fs::FileTypeExt; -#[cfg(unix)] -use std::os::unix::net::UnixStream; -use uucore::{format_usage, help_about, help_usage}; - const USAGE: &str = help_usage!("cat.md"); const ABOUT: &str = help_about!("cat.md"); +// Allocate 32 digits for the line number. +// An estimate is that we can print about 1e8 lines/seconds, so 32 digits +// would be enough for billions of universe lifetimes. +const LINE_NUMBER_BUF_SIZE: usize = 32; + +struct LineNumber { + buf: [u8; LINE_NUMBER_BUF_SIZE], + print_start: usize, + num_start: usize, + num_end: usize, +} + +// Logic to store a string for the line number. Manually incrementing the value +// represented in a buffer like this is significantly faster than storing +// a `usize` and using the standard Rust formatting macros to format a `usize` +// to a string each time it's needed. +// Buffer is initialized to " 1\t" and incremented each time `increment` is +// called, using uucore's fast_inc function that operates on strings. +impl LineNumber { + fn new() -> Self { + let mut buf = [b'0'; LINE_NUMBER_BUF_SIZE]; + + let init_str = " 1\t"; + let print_start = buf.len() - init_str.len(); + let num_start = buf.len() - 2; + let num_end = buf.len() - 1; + + buf[print_start..].copy_from_slice(init_str.as_bytes()); + + LineNumber { + buf, + print_start, + num_start, + num_end, + } + } + + fn increment(&mut self) { + fast_inc_one(&mut self.buf, &mut self.num_start, self.num_end); + self.print_start = self.print_start.min(self.num_start); + } + + #[inline] + fn to_str(&self) -> &[u8] { + &self.buf[self.print_start..] + } + + fn write(&self, writer: &mut impl Write) -> io::Result<()> { + writer.write_all(self.to_str()) + } +} + #[derive(Error, Debug)] enum CatError { /// Wrapper around `io::Error` @@ -44,7 +95,7 @@ enum CatError { #[error("{0}")] Nix(#[from] nix::Error), /// Unknown file type; it's not a regular file, socket, etc. - #[error("unknown filetype: {}", ft_debug)] + #[error("unknown filetype: {ft_debug}")] UnknownFiletype { /// A debug print of the file type ft_debug: String, @@ -53,6 +104,8 @@ enum CatError { IsDirectory, #[error("input file is output file")] OutputIsInput, + #[error("Too many levels of symbolic links")] + TooManySymlinks, } type CatResult = Result; @@ -83,19 +136,11 @@ struct OutputOptions { impl OutputOptions { fn tab(&self) -> &'static str { - if self.show_tabs { - "^I" - } else { - "\t" - } + if self.show_tabs { "^I" } else { "\t" } } fn end_of_line(&self) -> &'static str { - if self.show_ends { - "$\n" - } else { - "\n" - } + if self.show_ends { "$\n" } else { "\n" } } /// We can write fast if we can simply copy the contents of the file to @@ -113,7 +158,7 @@ impl OutputOptions { /// when we can't write fast. struct OutputState { /// The current line number - line_number: usize, + line_number: LineNumber, /// Whether the output cursor is at the beginning of a new line at_line_start: bool, @@ -126,12 +171,12 @@ struct OutputState { } #[cfg(unix)] -trait FdReadable: Read + AsRawFd {} +trait FdReadable: Read + AsFd {} #[cfg(not(unix))] trait FdReadable: Read {} #[cfg(unix)] -impl FdReadable for T where T: Read + AsRawFd {} +impl FdReadable for T where T: Read + AsFd {} #[cfg(not(unix))] impl FdReadable for T where T: Read {} @@ -171,12 +216,11 @@ mod options { pub static SHOW_NONPRINTING_TABS: &str = "t"; pub static SHOW_TABS: &str = "show-tabs"; pub static SHOW_NONPRINTING: &str = "show-nonprinting"; + pub static IGNORED_U: &str = "ignored-u"; } #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_ignore(); - let matches = uu_app().try_get_matches_from(args)?; let number_mode = if matches.get_flag(options::NUMBER_NONBLANK) { @@ -214,7 +258,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let squeeze_blank = matches.get_flag(options::SQUEEZE_BLANK); let files: Vec = match matches.get_many::(options::FILE) { - Some(v) => v.clone().map(|v| v.to_owned()).collect(), + Some(v) => v.cloned().collect(), None => vec!["-".to_owned()], }; @@ -230,14 +274,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .override_usage(format_usage(USAGE)) .about(ABOUT) .infer_long_args(true) + .args_override_self(true) .arg( Arg::new(options::FILE) .hide(true) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) .value_hint(clap::ValueHint::FilePath), ) .arg( @@ -252,7 +297,8 @@ pub fn uu_app() -> Command { .short('b') .long(options::NUMBER_NONBLANK) .help("number nonempty output lines, overrides -n") - .overrides_with(options::NUMBER) + // Note: This MUST NOT .overrides_with(options::NUMBER)! + // In clap, overriding is symmetric, so "-b -n" counts as "-n", which is not what we want. .action(ArgAction::SetTrue), ) .arg( @@ -302,6 +348,12 @@ pub fn uu_app() -> Command { .help("use ^ and M- notation, except for LF (\\n) and TAB (\\t)") .action(ArgAction::SetTrue), ) + .arg( + Arg::new(options::IGNORED_U) + .short('u') + .help("(ignored)") + .action(ArgAction::SetTrue), + ) } fn cat_handle( @@ -316,6 +368,23 @@ fn cat_handle( } } +/// Whether this process is appending to stdout. +#[cfg(unix)] +fn is_appending() -> bool { + let stdout = io::stdout(); + let Ok(flags) = fcntl(stdout.as_fd(), FcntlArg::F_GETFL) else { + return false; + }; + // TODO Replace `1 << 10` with `nix::fcntl::Oflag::O_APPEND`. + let o_append = 1 << 10; + (flags & o_append) > 0 +} + +#[cfg(not(unix))] +fn is_appending() -> bool { + false +} + fn cat_path( path: &str, options: &OutputOptions, @@ -325,10 +394,16 @@ fn cat_path( match get_input_type(path)? { InputType::StdIn => { let stdin = io::stdin(); + let in_info = FileInformation::from_file(&stdin)?; let mut handle = InputHandle { reader: stdin, - is_interactive: std::io::stdin().is_terminal(), + is_interactive: io::stdin().is_terminal(), }; + if let Some(out_info) = out_info { + if in_info == *out_info && is_appending() { + return Err(CatError::OutputIsInput); + } + } cat_handle(&mut handle, options, state) } InputType::Directory => Err(CatError::IsDirectory), @@ -363,10 +438,10 @@ fn cat_path( } fn cat_files(files: &[String], options: &OutputOptions) -> UResult<()> { - let out_info = FileInformation::from_file(&std::io::stdout()).ok(); + let out_info = FileInformation::from_file(&io::stdout()).ok(); let mut state = OutputState { - line_number: 1, + line_number: LineNumber::new(), at_line_start: true, skipped_carriage_return: false, one_blank_kept: false, @@ -375,7 +450,7 @@ fn cat_files(files: &[String], options: &OutputOptions) -> UResult<()> { for path in files { if let Err(err) = cat_path(path, options, &mut state, out_info.as_ref()) { - error_messages.push(format!("{}: {}", path.maybe_quote(), err)); + error_messages.push(format!("{}: {err}", path.maybe_quote())); } } if state.skipped_carriage_return { @@ -404,7 +479,23 @@ fn get_input_type(path: &str) -> CatResult { return Ok(InputType::StdIn); } - let ft = metadata(path)?.file_type(); + let ft = match metadata(path) { + Ok(md) => md.file_type(), + Err(e) => { + if let Some(raw_error) = e.raw_os_error() { + // On Unix-like systems, the error code for "Too many levels of symbolic links" is 40 (ELOOP). + // we want to provide a proper error message in this case. + #[cfg(not(any(target_os = "macos", target_os = "freebsd")))] + let too_many_symlink_code = 40; + #[cfg(any(target_os = "macos", target_os = "freebsd"))] + let too_many_symlink_code = 62; + if raw_error == too_many_symlink_code { + return Err(CatError::TooManySymlinks); + } + } + return Err(CatError::Io(e)); + } + }; match ft { #[cfg(unix)] ft if ft.is_block_device() => Ok(InputType::BlockDevice), @@ -445,12 +536,18 @@ fn write_fast(handle: &mut InputHandle) -> CatResult<()> { } stdout_lock.write_all(&buf[..n])?; } + + // If the splice() call failed and there has been some data written to + // stdout via while loop above AND there will be second splice() call + // that will succeed, data pushed through splice will be output before + // the data buffered in stdout.lock. Therefore additional explicit flush + // is required here. + stdout_lock.flush()?; Ok(()) } /// Outputs file contents to stdout in a line-by-line fashion, /// propagating any errors that might occur. -#[allow(clippy::cognitive_complexity)] fn write_lines( handle: &mut InputHandle, options: &OutputOptions, @@ -458,7 +555,9 @@ fn write_lines( ) -> CatResult<()> { let mut in_buf = [0; 1024 * 31]; let stdout = io::stdout(); - let mut writer = stdout.lock(); + let stdout = stdout.lock(); + // Add a 32K buffer for stdout - this greatly improves performance. + let mut writer = BufWriter::with_capacity(32 * 1024, stdout); while let Ok(n) = handle.reader.read(&mut in_buf) { if n == 0 { @@ -469,22 +568,7 @@ fn write_lines( while pos < n { // skip empty line_number enumerating them if needed if in_buf[pos] == b'\n' { - // \r followed by \n is printed as ^M when show_ends is enabled, so that \r\n prints as ^M$ - if state.skipped_carriage_return && options.show_ends { - writer.write_all(b"^M")?; - state.skipped_carriage_return = false; - } - if !state.at_line_start || !options.squeeze_blank || !state.one_blank_kept { - state.one_blank_kept = true; - if state.at_line_start && options.number == NumberingMode::All { - write!(writer, "{0:6}\t", state.line_number)?; - state.line_number += 1; - } - writer.write_all(options.end_of_line().as_bytes())?; - if handle.is_interactive { - writer.flush()?; - } - } + write_new_line(&mut writer, options, state, handle.is_interactive)?; state.at_line_start = true; pos += 1; continue; @@ -496,18 +580,13 @@ fn write_lines( } state.one_blank_kept = false; if state.at_line_start && options.number != NumberingMode::None { - write!(writer, "{0:6}\t", state.line_number)?; - state.line_number += 1; + state.line_number.write(&mut writer)?; + state.line_number.increment(); } // print to end of line or end of buffer - let offset = if options.show_nonprint { - write_nonprint_to_end(&in_buf[pos..], &mut writer, options.tab().as_bytes()) - } else if options.show_tabs { - write_tab_to_end(&in_buf[pos..], &mut writer) - } else { - write_to_end(&in_buf[pos..], &mut writer) - }; + let offset = write_end(&mut writer, &in_buf[pos..], options); + // end of buffer? if offset + pos == in_buf.len() { state.at_line_start = false; @@ -518,26 +597,75 @@ fn write_lines( } else { assert_eq!(in_buf[pos + offset], b'\n'); // print suitable end of line - writer.write_all(options.end_of_line().as_bytes())?; - if handle.is_interactive { - writer.flush()?; - } + write_end_of_line( + &mut writer, + options.end_of_line().as_bytes(), + handle.is_interactive, + )?; state.at_line_start = true; } pos += offset + 1; } + // We need to flush the buffer each time around the loop in order to pass GNU tests. + // When we are reading the input from a pipe, the `handle.reader.read` call at the top + // of this loop will block (indefinitely) whist waiting for more data. The expectation + // however is that anything that's ready for output should show up in the meantime, + // and not be buffered internally to the `cat` process. + // Hence it's necessary to flush our buffer before every time we could potentially block + // on a `std::io::Read::read` call. + writer.flush()?; } Ok(()) } +// \r followed by \n is printed as ^M when show_ends is enabled, so that \r\n prints as ^M$ +fn write_new_line( + writer: &mut W, + options: &OutputOptions, + state: &mut OutputState, + is_interactive: bool, +) -> CatResult<()> { + if state.skipped_carriage_return { + if options.show_ends { + writer.write_all(b"^M")?; + } else { + writer.write_all(b"\r")?; + } + state.skipped_carriage_return = false; + + write_end_of_line(writer, options.end_of_line().as_bytes(), is_interactive)?; + return Ok(()); + } + if !state.at_line_start || !options.squeeze_blank || !state.one_blank_kept { + state.one_blank_kept = true; + if state.at_line_start && options.number == NumberingMode::All { + state.line_number.write(writer)?; + state.line_number.increment(); + } + write_end_of_line(writer, options.end_of_line().as_bytes(), is_interactive)?; + } + Ok(()) +} + +fn write_end(writer: &mut W, in_buf: &[u8], options: &OutputOptions) -> usize { + if options.show_nonprint { + write_nonprint_to_end(in_buf, writer, options.tab().as_bytes()) + } else if options.show_tabs { + write_tab_to_end(in_buf, writer) + } else { + write_to_end(in_buf, writer) + } +} + // write***_to_end methods // Write all symbols till \n or \r or end of buffer is reached // We need to stop at \r because it may be written as ^M depending on the byte after and settings; // however, write_nonprint_to_end doesn't need to stop at \r because it will always write \r as ^M. // Return the number of written symbols fn write_to_end(in_buf: &[u8], writer: &mut W) -> usize { - match in_buf.iter().position(|c| *c == b'\n' || *c == b'\r') { + // using memchr2 significantly improves performances + match memchr2(b'\n', b'\r', in_buf) { Some(p) => { writer.write_all(&in_buf[..p]).unwrap(); p @@ -569,7 +697,7 @@ fn write_tab_to_end(mut in_buf: &[u8], writer: &mut W) -> usize { } None => { writer.write_all(in_buf).unwrap(); - return in_buf.len(); + return in_buf.len() + count; } }; } @@ -586,10 +714,10 @@ fn write_nonprint_to_end(in_buf: &[u8], writer: &mut W, tab: &[u8]) -> 9 => writer.write_all(tab), 0..=8 | 10..=31 => writer.write_all(&[b'^', byte + 64]), 32..=126 => writer.write_all(&[byte]), - 127 => writer.write_all(&[b'^', b'?']), + 127 => writer.write_all(b"^?"), 128..=159 => writer.write_all(&[b'M', b'-', b'^', byte - 64]), 160..=254 => writer.write_all(&[b'M', b'-', byte - 128]), - _ => writer.write_all(&[b'M', b'-', b'^', b'?']), + _ => writer.write_all(b"M-^?"), } .unwrap(); count += 1; @@ -597,9 +725,35 @@ fn write_nonprint_to_end(in_buf: &[u8], writer: &mut W, tab: &[u8]) -> count } +fn write_end_of_line( + writer: &mut W, + end_of_line: &[u8], + is_interactive: bool, +) -> CatResult<()> { + writer.write_all(end_of_line)?; + if is_interactive { + writer.flush()?; + } + Ok(()) +} + #[cfg(test)] mod tests { - use std::io::{stdout, BufWriter}; + use std::io::{BufWriter, stdout}; + + #[test] + fn test_write_tab_to_end_with_newline() { + let mut writer = BufWriter::with_capacity(1024 * 64, stdout()); + let in_buf = b"a\tb\tc\n"; + assert_eq!(super::write_tab_to_end(in_buf, &mut writer), 5); + } + + #[test] + fn test_write_tab_to_end_no_newline() { + let mut writer = BufWriter::with_capacity(1024 * 64, stdout()); + let in_buf = b"a\tb\tc"; + assert_eq!(super::write_tab_to_end(in_buf, &mut writer), 5); + } #[test] fn test_write_nonprint_to_end_new_line() { @@ -640,4 +794,25 @@ mod tests { assert_eq!(writer.buffer(), [b'^', byte + 64]); } } + + #[test] + fn test_incrementing_string() { + let mut incrementing_string = super::LineNumber::new(); + assert_eq!(b" 1\t", incrementing_string.to_str()); + incrementing_string.increment(); + assert_eq!(b" 2\t", incrementing_string.to_str()); + // Run through to 100 + for _ in 3..=100 { + incrementing_string.increment(); + } + assert_eq!(b" 100\t", incrementing_string.to_str()); + // Run through until we overflow the original size. + for _ in 101..=1_000_000 { + incrementing_string.increment(); + } + // Confirm that the start position moves when we overflow the original size. + assert_eq!(b"1000000\t", incrementing_string.to_str()); + incrementing_string.increment(); + assert_eq!(b"1000001\t", incrementing_string.to_str()); + } } diff --git a/src/uu/cat/src/splice.rs b/src/uu/cat/src/splice.rs index 5a9e8738ed2..ca5265d2bf8 100644 --- a/src/uu/cat/src/splice.rs +++ b/src/uu/cat/src/splice.rs @@ -5,7 +5,7 @@ use super::{CatResult, FdReadable, InputHandle}; use nix::unistd; -use std::os::unix::io::{AsRawFd, RawFd}; +use std::os::{fd::AsFd, unix::io::AsRawFd}; use uucore::pipes::{pipe, splice, splice_exact}; @@ -20,9 +20,9 @@ const BUF_SIZE: usize = 1024 * 16; /// The `bool` in the result value indicates if we need to fall back to normal /// copying or not. False means we don't have to. #[inline] -pub(super) fn write_fast_using_splice( - handle: &mut InputHandle, - write_fd: &impl AsRawFd, +pub(super) fn write_fast_using_splice( + handle: &InputHandle, + write_fd: &S, ) -> CatResult { let (pipe_rd, pipe_wr) = pipe()?; @@ -38,7 +38,7 @@ pub(super) fn write_fast_using_splice( // we can recover by copying the data that we have from the // intermediate pipe to stdout using normal read/write. Then // we tell the caller to fall back. - copy_exact(pipe_rd.as_raw_fd(), write_fd.as_raw_fd(), n)?; + copy_exact(&pipe_rd, write_fd, n)?; return Ok(true); } } @@ -52,7 +52,7 @@ pub(super) fn write_fast_using_splice( /// Move exactly `num_bytes` bytes from `read_fd` to `write_fd`. /// /// Panics if not enough bytes can be read. -fn copy_exact(read_fd: RawFd, write_fd: RawFd, num_bytes: usize) -> nix::Result<()> { +fn copy_exact(read_fd: &impl AsFd, write_fd: &impl AsFd, num_bytes: usize) -> nix::Result<()> { let mut left = num_bytes; let mut buf = [0; BUF_SIZE]; while left > 0 { diff --git a/src/uu/chcon/Cargo.toml b/src/uu/chcon/Cargo.toml index c83c8abfc14..ccf36056339 100644 --- a/src/uu/chcon/Cargo.toml +++ b/src/uu/chcon/Cargo.toml @@ -1,14 +1,18 @@ [package] name = "uu_chcon" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "chcon ~ (uutils) change file security context" -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chcon" keywords = ["coreutils", "uutils", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chcon.rs" diff --git a/src/uu/chcon/src/chcon.rs b/src/uu/chcon/src/chcon.rs index ec111c853fd..2b1ff2e8f97 100644 --- a/src/uu/chcon/src/chcon.rs +++ b/src/uu/chcon/src/chcon.rs @@ -3,13 +3,14 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. // spell-checker:ignore (vars) RFILE +#![cfg(target_os = "linux")] #![allow(clippy::upper_case_acronyms)] use clap::builder::ValueParser; use uucore::error::{UResult, USimpleError, UUsageError}; use uucore::{display::Quotable, format_usage, help_about, help_usage, show_error, show_warning}; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use selinux::{OpaqueSecurityContext, SecurityContext}; use std::borrow::Cow; @@ -149,11 +150,12 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) .disable_help_flag(true) + .args_override_self(true) .arg( Arg::new(options::HELP) .long(options::HELP) @@ -163,7 +165,7 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::dereference::DEREFERENCE) .long(options::dereference::DEREFERENCE) - .conflicts_with(options::dereference::NO_DEREFERENCE) + .overrides_with(options::dereference::NO_DEREFERENCE) .help( "Affect the referent of each symbolic link (this is the default), \ rather than the symbolic link itself.", @@ -180,7 +182,7 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::preserve_root::PRESERVE_ROOT) .long(options::preserve_root::PRESERVE_ROOT) - .conflicts_with(options::preserve_root::NO_PRESERVE_ROOT) + .overrides_with(options::preserve_root::NO_PRESERVE_ROOT) .help("Fail to operate recursively on '/'.") .action(ArgAction::SetTrue), ) @@ -310,7 +312,7 @@ struct Options { files: Vec, } -fn parse_command_line(config: clap::Command, args: impl uucore::Args) -> Result { +fn parse_command_line(config: Command, args: impl uucore::Args) -> Result { let matches = config.try_get_matches_from(args)?; let verbose = matches.get_flag(options::VERBOSE); @@ -606,7 +608,7 @@ fn process_file( if result.is_ok() { if options.verbose { println!( - "{}: Changing security context of: {}", + "{}: changing security context of {}", uucore::util_name(), file_full_name.quote() ); @@ -726,7 +728,7 @@ fn get_root_dev_ino() -> Result { } fn root_dev_ino_check(root_dev_ino: Option, dir_dev_ino: DeviceAndINode) -> bool { - root_dev_ino.map_or(false, |root_dev_ino| root_dev_ino == dir_dev_ino) + root_dev_ino == Some(dir_dev_ino) } fn root_dev_ino_warn(dir_name: &Path) { @@ -776,7 +778,7 @@ enum SELinuxSecurityContext<'t> { String(Option), } -impl<'t> SELinuxSecurityContext<'t> { +impl SELinuxSecurityContext<'_> { fn to_c_string(&self) -> Result>> { match self { Self::File(context) => context diff --git a/src/uu/chcon/src/errors.rs b/src/uu/chcon/src/errors.rs index 10d5735a0c6..b8f720a3920 100644 --- a/src/uu/chcon/src/errors.rs +++ b/src/uu/chcon/src/errors.rs @@ -2,6 +2,8 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +#![cfg(target_os = "linux")] + use std::ffi::OsString; use std::fmt::Write; use std::io; diff --git a/src/uu/chcon/src/fts.rs b/src/uu/chcon/src/fts.rs index a81cb39b658..c9a8599fa2a 100644 --- a/src/uu/chcon/src/fts.rs +++ b/src/uu/chcon/src/fts.rs @@ -2,11 +2,12 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +#![cfg(target_os = "linux")] + use std::ffi::{CStr, CString, OsStr}; use std::marker::PhantomData; use std::os::raw::{c_int, c_long, c_short}; use std::path::Path; -use std::ptr::NonNull; use std::{io, iter, ptr, slice}; use crate::errors::{Error, Result}; @@ -69,7 +70,7 @@ impl FTS { // pointer assumed to be valid. let new_entry = unsafe { fts_sys::fts_read(self.fts.as_ptr()) }; - self.entry = NonNull::new(new_entry); + self.entry = ptr::NonNull::new(new_entry); if self.entry.is_none() { let r = io::Error::last_os_error(); if let Some(0) = r.raw_os_error() { @@ -159,7 +160,7 @@ impl<'fts> EntryRef<'fts> { return None; } - NonNull::new(entry.fts_path) + ptr::NonNull::new(entry.fts_path) .map(|path_ptr| { let path_size = usize::from(entry.fts_pathlen).saturating_add(1); diff --git a/src/uu/chcon/src/main.rs b/src/uu/chcon/src/main.rs index d93d7d1da2b..d1354d840af 100644 --- a/src/uu/chcon/src/main.rs +++ b/src/uu/chcon/src/main.rs @@ -1 +1,2 @@ +#![cfg(target_os = "linux")] uucore::bin!(uu_chcon); diff --git a/src/uu/chgrp/Cargo.toml b/src/uu/chgrp/Cargo.toml index 7f43e846625..7f23eec34d7 100644 --- a/src/uu/chgrp/Cargo.toml +++ b/src/uu/chgrp/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_chgrp" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "chgrp ~ (uutils) change the group ownership of FILE" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chgrp" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chgrp.rs" diff --git a/src/uu/chgrp/src/chgrp.rs b/src/uu/chgrp/src/chgrp.rs index fba2cef1611..1763bbfeb73 100644 --- a/src/uu/chgrp/src/chgrp.rs +++ b/src/uu/chgrp/src/chgrp.rs @@ -8,10 +8,10 @@ use uucore::display::Quotable; pub use uucore::entries; use uucore::error::{FromIo, UResult, USimpleError}; -use uucore::perms::{chown_base, options, GidUidOwnerFilter, IfFrom}; +use uucore::perms::{GidUidOwnerFilter, IfFrom, chown_base, options}; use uucore::{format_usage, help_about, help_usage}; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; use std::fs; use std::os::unix::fs::MetadataExt; @@ -19,8 +19,26 @@ use std::os::unix::fs::MetadataExt; const ABOUT: &str = help_about!("chgrp.md"); const USAGE: &str = help_usage!("chgrp.md"); -fn parse_gid_and_uid(matches: &ArgMatches) -> UResult { - let mut raw_group: String = String::new(); +fn parse_gid_from_str(group: &str) -> Result { + if let Some(gid_str) = group.strip_prefix(':') { + // Handle :gid format + gid_str + .parse::() + .map_err(|_| format!("invalid group id: '{gid_str}'")) + } else { + // Try as group name first + match entries::grp2gid(group) { + Ok(g) => Ok(g), + // If group name lookup fails, try parsing as raw number + Err(_) => group + .parse::() + .map_err(|_| format!("invalid group: '{group}'")), + } + } +} + +fn get_dest_gid(matches: &ArgMatches) -> UResult<(Option, String)> { + let mut raw_group = String::new(); let dest_gid = if let Some(file) = matches.get_one::(options::REFERENCE) { fs::metadata(file) .map(|meta| { @@ -38,22 +56,38 @@ fn parse_gid_and_uid(matches: &ArgMatches) -> UResult { if group.is_empty() { None } else { - match entries::grp2gid(group) { + match parse_gid_from_str(group) { Ok(g) => Some(g), - _ => { - return Err(USimpleError::new( - 1, - format!("invalid group: {}", group.quote()), - )) - } + Err(e) => return Err(USimpleError::new(1, e)), } } }; + Ok((dest_gid, raw_group)) +} + +fn parse_gid_and_uid(matches: &ArgMatches) -> UResult { + let (dest_gid, raw_group) = get_dest_gid(matches)?; + + // Handle --from option + let filter = if let Some(from_group) = matches.get_one::(options::FROM) { + match parse_gid_from_str(from_group) { + Ok(g) => IfFrom::Group(g), + Err(_) => { + return Err(USimpleError::new( + 1, + format!("invalid user: '{from_group}'"), + )); + } + } + } else { + IfFrom::All + }; + Ok(GidUidOwnerFilter { dest_gid, dest_uid: None, raw_owner: raw_group, - filter: IfFrom::All, + filter, }) } @@ -64,7 +98,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -73,7 +107,7 @@ pub fn uu_app() -> Command { Arg::new(options::HELP) .long(options::HELP) .help("Print help information.") - .action(ArgAction::Help) + .action(ArgAction::Help), ) .arg( Arg::new(options::verbosity::CHANGES) @@ -101,20 +135,6 @@ pub fn uu_app() -> Command { .help("output a diagnostic for every file processed") .action(ArgAction::SetTrue), ) - .arg( - Arg::new(options::dereference::DEREFERENCE) - .long(options::dereference::DEREFERENCE) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::dereference::NO_DEREFERENCE) - .short('h') - .long(options::dereference::NO_DEREFERENCE) - .help( - "affect symbolic links instead of any referenced file (useful only on systems that can change the ownership of a symlink)", - ) - .action(ArgAction::SetTrue), - ) .arg( Arg::new(options::preserve_root::PRESERVE) .long(options::preserve_root::PRESERVE) @@ -134,6 +154,12 @@ pub fn uu_app() -> Command { .value_hint(clap::ValueHint::FilePath) .help("use RFILE's group rather than specifying GROUP values"), ) + .arg( + Arg::new(options::FROM) + .long(options::FROM) + .value_name("GROUP") + .help("change the group only if its current group matches GROUP"), + ) .arg( Arg::new(options::RECURSIVE) .short('R') @@ -141,23 +167,6 @@ pub fn uu_app() -> Command { .help("operate on files and directories recursively") .action(ArgAction::SetTrue), ) - .arg( - Arg::new(options::traverse::TRAVERSE) - .short(options::traverse::TRAVERSE.chars().next().unwrap()) - .help("if a command line argument is a symbolic link to a directory, traverse it") - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::traverse::NO_TRAVERSE) - .short(options::traverse::NO_TRAVERSE.chars().next().unwrap()) - .help("do not traverse any symbolic links (default)") - .overrides_with_all([options::traverse::TRAVERSE, options::traverse::EVERY]) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::traverse::EVERY) - .short(options::traverse::EVERY.chars().next().unwrap()) - .help("traverse every symbolic link to a directory encountered") - .action(ArgAction::SetTrue), - ) + // Add common arguments with chgrp, chown & chmod + .args(uucore::perms::common_args()) } diff --git a/src/uu/chmod/Cargo.toml b/src/uu/chmod/Cargo.toml index 2a9c73d9db3..09f1c531a90 100644 --- a/src/uu/chmod/Cargo.toml +++ b/src/uu/chmod/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_chmod" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "chmod ~ (uutils) change mode of FILE" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chmod" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chmod.rs" @@ -17,7 +20,7 @@ path = "src/chmod.rs" [dependencies] clap = { workspace = true } libc = { workspace = true } -uucore = { workspace = true, features = ["fs", "mode"] } +uucore = { workspace = true, features = ["entries", "fs", "mode", "perms"] } [[bin]] name = "chmod" diff --git a/src/uu/chmod/chmod.md b/src/uu/chmod/chmod.md index d6c2ed2d8e3..10ddb48a2ed 100644 --- a/src/uu/chmod/chmod.md +++ b/src/uu/chmod/chmod.md @@ -13,4 +13,4 @@ With --reference, change the mode of each FILE to that of RFILE. ## After Help -Each MODE is of the form '[ugoa]*([-+=]([rwxXst]*|[ugo]))+|[-+=]?[0-7]+'. +Each MODE is of the form `[ugoa]*([-+=]([rwxXst]*|[ugo]))+|[-+=]?[0-7]+`. diff --git a/src/uu/chmod/src/chmod.rs b/src/uu/chmod/src/chmod.rs index b007bb1d735..dfe30485919 100644 --- a/src/uu/chmod/src/chmod.rs +++ b/src/uu/chmod/src/chmod.rs @@ -5,17 +5,18 @@ // spell-checker:ignore (ToDO) Chmoder cmode fmode fperm fref ugoa RFILE RFILE's -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use std::ffi::OsString; use std::fs; use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::path::Path; use uucore::display::Quotable; -use uucore::error::{set_exit_code, ExitCode, UResult, USimpleError, UUsageError}; +use uucore::error::{ExitCode, UResult, USimpleError, UUsageError, set_exit_code}; use uucore::fs::display_permissions_unix; use uucore::libc::mode_t; #[cfg(not(windows))] use uucore::mode; +use uucore::perms::{TraverseSymlinks, configure_symlink_and_recursion}; use uucore::{format_usage, help_about, help_section, help_usage, show, show_error}; const ABOUT: &str = help_about!("chmod.md"); @@ -23,6 +24,7 @@ const USAGE: &str = help_usage!("chmod.md"); const LONG_USAGE: &str = help_section!("after help", "chmod.md"); mod options { + pub const HELP: &str = "help"; pub const CHANGES: &str = "changes"; pub const QUIET: &str = "quiet"; // visible_alias("silent") pub const VERBOSE: &str = "verbose"; @@ -98,15 +100,14 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let quiet = matches.get_flag(options::QUIET); let verbose = matches.get_flag(options::VERBOSE); let preserve_root = matches.get_flag(options::PRESERVE_ROOT); - let recursive = matches.get_flag(options::RECURSIVE); let fmode = match matches.get_one::(options::REFERENCE) { Some(fref) => match fs::metadata(fref) { - Ok(meta) => Some(meta.mode()), + Ok(meta) => Some(meta.mode() & 0o7777), Err(err) => { return Err(USimpleError::new( 1, - format!("cannot stat attributes of {}: {}", fref.quote(), err), - )) + format!("cannot stat attributes of {}: {err}", fref.quote()), + )); } }, None => None, @@ -137,6 +138,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { return Err(UUsageError::new(1, "missing operand".to_string())); } + let (recursive, dereference, traverse_symlinks) = + configure_symlink_and_recursion(&matches, TraverseSymlinks::First)?; + let chmoder = Chmoder { changes, quiet, @@ -145,6 +149,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { recursive, fmode, cmode, + traverse_symlinks, + dereference, }; chmoder.chmod(&files) @@ -152,12 +158,19 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .args_override_self(true) .infer_long_args(true) .no_binary_name(true) + .disable_help_flag(true) + .arg( + Arg::new(options::HELP) + .long(options::HELP) + .help("Print help information.") + .action(ArgAction::Help), + ) .arg( Arg::new(options::CHANGES) .long(options::CHANGES) @@ -206,9 +219,10 @@ pub fn uu_app() -> Command { .help("use RFILE's mode instead of MODE values"), ) .arg( - Arg::new(options::MODE).required_unless_present(options::REFERENCE), // It would be nice if clap could parse with delimiter, e.g. "g-x,u+x", - // however .multiple_occurrences(true) cannot be used here because FILE already needs that. - // Only one positional argument with .multiple_occurrences(true) set is allowed per command + Arg::new(options::MODE).required_unless_present(options::REFERENCE), + // It would be nice if clap could parse with delimiter, e.g. "g-x,u+x", + // however .multiple_occurrences(true) cannot be used here because FILE already needs that. + // Only one positional argument with .multiple_occurrences(true) set is allowed per command ) .arg( Arg::new(options::FILE) @@ -216,6 +230,8 @@ pub fn uu_app() -> Command { .action(ArgAction::Append) .value_hint(clap::ValueHint::AnyPath), ) + // Add common arguments with chgrp, chown & chmod + .args(uucore::perms::common_args()) } struct Chmoder { @@ -226,6 +242,8 @@ struct Chmoder { recursive: bool, fmode: Option, cmode: Option, + traverse_symlinks: TraverseSymlinks, + dereference: bool, } impl Chmoder { @@ -237,12 +255,23 @@ impl Chmoder { let file = Path::new(filename); if !file.exists() { if file.is_symlink() { + if !self.dereference && !self.recursive { + // The file is a symlink and we should not follow it + // Don't try to change the mode of the symlink itself + continue; + } + if self.recursive && self.traverse_symlinks == TraverseSymlinks::None { + continue; + } + if !self.quiet { show!(USimpleError::new( 1, format!("cannot operate on dangling symlink {}", filename.quote()), )); + set_exit_code(1); } + if self.verbose { println!( "failed to change mode of {} from 0000 (---------) to 1500 (r-x-----T)", @@ -262,14 +291,19 @@ impl Chmoder { // So we set the exit code, because it hasn't been set yet if `self.quiet` is true. set_exit_code(1); continue; + } else if !self.dereference && file.is_symlink() { + // The file is a symlink and we should not follow it + // chmod 755 --no-dereference a/link + // should not change the permissions in this case + continue; } if self.recursive && self.preserve_root && filename == "/" { return Err(USimpleError::new( 1, format!( - "it is dangerous to operate recursively on {}\nuse --no-preserve-root to override this failsafe", + "it is dangerous to operate recursively on {}\nchmod: use --no-preserve-root to override this failsafe", filename.quote() - ) + ), )); } if self.recursive { @@ -283,11 +317,23 @@ impl Chmoder { fn walk_dir(&self, file_path: &Path) -> UResult<()> { let mut r = self.chmod_file(file_path); - if !file_path.is_symlink() && file_path.is_dir() { + // Determine whether to traverse symlinks based on `self.traverse_symlinks` + let should_follow_symlink = match self.traverse_symlinks { + TraverseSymlinks::All => true, + TraverseSymlinks::First => { + file_path == file_path.canonicalize().unwrap_or(file_path.to_path_buf()) + } + TraverseSymlinks::None => false, + }; + + // If the path is a directory (or we should follow symlinks), recurse into it + if (!file_path.is_symlink() || should_follow_symlink) && file_path.is_dir() { for dir_entry in file_path.read_dir()? { let path = dir_entry?.path(); if !path.is_symlink() { r = self.walk_dir(path.as_path()); + } else if should_follow_symlink { + r = self.chmod_file(path.as_path()).and(r); } } } @@ -303,31 +349,36 @@ impl Chmoder { } #[cfg(unix)] fn chmod_file(&self, file: &Path) -> UResult<()> { - use uucore::mode::get_umask; + use uucore::{mode::get_umask, perms::get_metadata}; + + let metadata = get_metadata(file, self.dereference); - let fperm = match fs::metadata(file) { + let fperm = match metadata { Ok(meta) => meta.mode() & 0o7777, Err(err) => { - if file.is_symlink() { + // Handle dangling symlinks or other errors + return if file.is_symlink() && !self.dereference { if self.verbose { println!( "neither symbolic link {} nor referent has been changed", file.quote() ); } - return Ok(()); + Ok(()) // Skip dangling symlinks } else if err.kind() == std::io::ErrorKind::PermissionDenied { // These two filenames would normally be conditionally // quoted, but GNU's tests expect them to always be quoted - return Err(USimpleError::new( + Err(USimpleError::new( 1, format!("{}: Permission denied", file.quote()), - )); + )) } else { - return Err(USimpleError::new(1, format!("{}: {}", file.quote(), err))); - } + Err(USimpleError::new(1, format!("{}: {err}", file.quote()))) + }; } }; + + // Determine the new permissions to apply match self.fmode { Some(mode) => self.change_file(fperm, mode, file)?, None => { @@ -335,9 +386,7 @@ impl Chmoder { let mut new_mode = fperm; let mut naively_expected_new_mode = new_mode; for mode in cmode_unwrapped.split(',') { - // cmode is guaranteed to be Some in this case - let arr: &[char] = &['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']; - let result = if mode.contains(arr) { + let result = if mode.chars().any(|c| c.is_ascii_digit()) { mode::parse_numeric(new_mode, mode, file.is_dir()).map(|v| (v, v)) } else { mode::parse_symbolic(new_mode, mode, get_umask(), file.is_dir()).map(|m| { @@ -352,20 +401,22 @@ impl Chmoder { (m, naive_mode) }) }; + match result { Ok((mode, naive_mode)) => { new_mode = mode; naively_expected_new_mode = naive_mode; } Err(f) => { - if self.quiet { - return Err(ExitCode::new(1)); + return if self.quiet { + Err(ExitCode::new(1)) } else { - return Err(USimpleError::new(1, f)); - } + Err(USimpleError::new(1, f)) + }; } } } + self.change_file(fperm, new_mode, file)?; // if a permission would have been removed if umask was 0, but it wasn't because umask was not 0, print an error and fail if (new_mode & !naively_expected_new_mode) != 0 { @@ -390,24 +441,21 @@ impl Chmoder { if fperm == mode { if self.verbose && !self.changes { println!( - "mode of {} retained as {:04o} ({})", + "mode of {} retained as {fperm:04o} ({})", file.quote(), - fperm, display_permissions_unix(fperm as mode_t, false), ); } Ok(()) } else if let Err(err) = fs::set_permissions(file, fs::Permissions::from_mode(mode)) { if !self.quiet { - show_error!("{}", err); + show_error!("{err}"); } if self.verbose { println!( - "failed to change mode of file {} from {:04o} ({}) to {:04o} ({})", + "failed to change mode of file {} from {fperm:04o} ({}) to {mode:04o} ({})", file.quote(), - fperm, display_permissions_unix(fperm as mode_t, false), - mode, display_permissions_unix(mode as mode_t, false) ); } @@ -415,11 +463,9 @@ impl Chmoder { } else { if self.verbose || self.changes { println!( - "mode of {} changed from {:04o} ({}) to {:04o} ({})", + "mode of {} changed from {fperm:04o} ({}) to {mode:04o} ({})", file.quote(), - fperm, display_permissions_unix(fperm as mode_t, false), - mode, display_permissions_unix(mode as mode_t, false) ); } diff --git a/src/uu/chown/Cargo.toml b/src/uu/chown/Cargo.toml index 82b32875d65..dcf7c445412 100644 --- a/src/uu/chown/Cargo.toml +++ b/src/uu/chown/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_chown" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "chown ~ (uutils) change the ownership of FILE" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chown" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chown.rs" diff --git a/src/uu/chown/src/chown.rs b/src/uu/chown/src/chown.rs index 8e97d565242..4389d92f663 100644 --- a/src/uu/chown/src/chown.rs +++ b/src/uu/chown/src/chown.rs @@ -7,12 +7,12 @@ use uucore::display::Quotable; pub use uucore::entries::{self, Group, Locate, Passwd}; -use uucore::perms::{chown_base, options, GidUidOwnerFilter, IfFrom}; +use uucore::perms::{GidUidOwnerFilter, IfFrom, chown_base, options}; use uucore::{format_usage, help_about, help_usage}; use uucore::error::{FromIo, UResult, USimpleError}; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; use std::fs; use std::os::unix::fs::MetadataExt; @@ -78,7 +78,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -96,25 +96,6 @@ pub fn uu_app() -> Command { .help("like verbose but report only when a change is made") .action(ArgAction::SetTrue), ) - .arg( - Arg::new(options::dereference::DEREFERENCE) - .long(options::dereference::DEREFERENCE) - .help( - "affect the referent of each symbolic link (this is the default), \ - rather than the symbolic link itself", - ) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::dereference::NO_DEREFERENCE) - .short('h') - .long(options::dereference::NO_DEREFERENCE) - .help( - "affect symbolic links instead of any referenced file \ - (useful only on systems that can change the ownership of a symlink)", - ) - .action(ArgAction::SetTrue), - ) .arg( Arg::new(options::FROM) .long(options::FROM) @@ -165,27 +146,6 @@ pub fn uu_app() -> Command { .long(options::verbosity::SILENT) .action(ArgAction::SetTrue), ) - .arg( - Arg::new(options::traverse::TRAVERSE) - .short(options::traverse::TRAVERSE.chars().next().unwrap()) - .help("if a command line argument is a symbolic link to a directory, traverse it") - .overrides_with_all([options::traverse::EVERY, options::traverse::NO_TRAVERSE]) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::traverse::EVERY) - .short(options::traverse::EVERY.chars().next().unwrap()) - .help("traverse every symbolic link to a directory encountered") - .overrides_with_all([options::traverse::TRAVERSE, options::traverse::NO_TRAVERSE]) - .action(ArgAction::SetTrue), - ) - .arg( - Arg::new(options::traverse::NO_TRAVERSE) - .short(options::traverse::NO_TRAVERSE.chars().next().unwrap()) - .help("do not traverse any symbolic links (default)") - .overrides_with_all([options::traverse::TRAVERSE, options::traverse::EVERY]) - .action(ArgAction::SetTrue), - ) .arg( Arg::new(options::verbosity::VERBOSE) .long(options::verbosity::VERBOSE) @@ -193,6 +153,55 @@ pub fn uu_app() -> Command { .help("output a diagnostic for every file processed") .action(ArgAction::SetTrue), ) + // Add common arguments with chgrp, chown & chmod + .args(uucore::perms::common_args()) +} + +/// Parses the user string to extract the UID. +fn parse_uid(user: &str, spec: &str, sep: char) -> UResult> { + if user.is_empty() { + return Ok(None); + } + match Passwd::locate(user) { + Ok(u) => Ok(Some(u.uid)), // We have been able to get the uid + Err(_) => { + // we have NOT been able to find the uid + // but we could be in the case where we have user.group + if spec.contains('.') && !spec.contains(':') && sep == ':' { + // but the input contains a '.' but not a ':' + // we might have something like username.groupname + // So, try to parse it this way + parse_spec(spec, '.').map(|(uid, _)| uid) + } else { + // It's possible that the `user` string contains a + // numeric user ID, in which case, we respect that. + match user.parse() { + Ok(uid) => Ok(Some(uid)), + Err(_) => Err(USimpleError::new( + 1, + format!("invalid user: {}", spec.quote()), + )), + } + } + } + } +} + +/// Parses the group string to extract the GID. +fn parse_gid(group: &str, spec: &str) -> UResult> { + if group.is_empty() { + return Ok(None); + } + match Group::locate(group) { + Ok(g) => Ok(Some(g.gid)), + Err(_) => match group.parse() { + Ok(gid) => Ok(Some(gid)), + Err(_) => Err(USimpleError::new( + 1, + format!("invalid group: {}", spec.quote()), + )), + }, + } } /// Parse the owner/group specifier string into a user ID and a group ID. @@ -213,52 +222,8 @@ fn parse_spec(spec: &str, sep: char) -> UResult<(Option, Option)> { let user = args.next().unwrap_or(""); let group = args.next().unwrap_or(""); - let uid = if user.is_empty() { - None - } else { - Some(match Passwd::locate(user) { - Ok(u) => u.uid, // We have been able to get the uid - Err(_) => - // we have NOT been able to find the uid - // but we could be in the case where we have user.group - { - if spec.contains('.') && !spec.contains(':') && sep == ':' { - // but the input contains a '.' but not a ':' - // we might have something like username.groupname - // So, try to parse it this way - return parse_spec(spec, '.'); - } else { - // It's possible that the `user` string contains a - // numeric user ID, in which case, we respect that. - match user.parse() { - Ok(uid) => uid, - Err(_) => { - return Err(USimpleError::new( - 1, - format!("invalid user: {}", spec.quote()), - )) - } - } - } - } - }) - }; - let gid = if group.is_empty() { - None - } else { - Some(match Group::locate(group) { - Ok(g) => g.gid, - Err(_) => match group.parse() { - Ok(gid) => gid, - Err(_) => { - return Err(USimpleError::new( - 1, - format!("invalid group: {}", spec.quote()), - )); - } - }, - }) - }; + let uid = parse_uid(user, spec, sep)?; + let gid = parse_gid(group, spec)?; if user.chars().next().map(char::is_numeric).unwrap_or(false) && group.is_empty() diff --git a/src/uu/chroot/Cargo.toml b/src/uu/chroot/Cargo.toml index 3c836add869..4d302d95f06 100644 --- a/src/uu/chroot/Cargo.toml +++ b/src/uu/chroot/Cargo.toml @@ -1,21 +1,25 @@ [package] name = "uu_chroot" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "chroot ~ (uutils) run COMMAND under a new root directory" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/chroot" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/chroot.rs" [dependencies] clap = { workspace = true } +thiserror = { workspace = true } uucore = { workspace = true, features = ["entries", "fs"] } [[bin]] diff --git a/src/uu/chroot/src/chroot.rs b/src/uu/chroot/src/chroot.rs index 6366775c36d..15c7bac4d8d 100644 --- a/src/uu/chroot/src/chroot.rs +++ b/src/uu/chroot/src/chroot.rs @@ -7,51 +7,172 @@ mod error; use crate::error::ChrootError; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use std::ffi::CString; use std::io::Error; use std::os::unix::prelude::OsStrExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process; -use uucore::error::{set_exit_code, UClapError, UResult, UUsageError}; -use uucore::fs::{canonicalize, MissingHandling, ResolveMode}; +use uucore::entries::{Locate, Passwd, grp2gid, usr2uid}; +use uucore::error::{UClapError, UResult, UUsageError, set_exit_code}; +use uucore::fs::{MissingHandling, ResolveMode, canonicalize}; use uucore::libc::{self, chroot, setgid, setgroups, setuid}; -use uucore::{entries, format_usage, help_about, help_usage}; +use uucore::{format_usage, help_about, help_usage, show}; static ABOUT: &str = help_about!("chroot.md"); static USAGE: &str = help_usage!("chroot.md"); mod options { pub const NEWROOT: &str = "newroot"; - pub const USER: &str = "user"; - pub const GROUP: &str = "group"; pub const GROUPS: &str = "groups"; pub const USERSPEC: &str = "userspec"; pub const COMMAND: &str = "command"; pub const SKIP_CHDIR: &str = "skip-chdir"; } +/// A user and group specification, where each is optional. +enum UserSpec { + NeitherGroupNorUser, + UserOnly(String), + GroupOnly(String), + UserAndGroup(String, String), +} + +struct Options { + /// Path to the new root directory. + newroot: PathBuf, + /// Whether to change to the new root directory. + skip_chdir: bool, + /// List of groups under which the command will be run. + groups: Option>, + /// The user and group (each optional) under which the command will be run. + userspec: Option, +} + +/// Parse a user and group from the argument to `--userspec`. +/// +/// The `spec` must be of the form `[USER][:[GROUP]]`, otherwise an +/// error is returned. +fn parse_userspec(spec: &str) -> UserSpec { + match spec.split_once(':') { + // "" + None if spec.is_empty() => UserSpec::NeitherGroupNorUser, + // "usr" + None => UserSpec::UserOnly(spec.to_string()), + // ":" + Some(("", "")) => UserSpec::NeitherGroupNorUser, + // ":grp" + Some(("", grp)) => UserSpec::GroupOnly(grp.to_string()), + // "usr:" + Some((usr, "")) => UserSpec::UserOnly(usr.to_string()), + // "usr:grp" + Some((usr, grp)) => UserSpec::UserAndGroup(usr.to_string(), grp.to_string()), + } +} + +// Pre-condition: `list_str` is non-empty. +fn parse_group_list(list_str: &str) -> Result, ChrootError> { + let split: Vec<&str> = list_str.split(',').collect(); + if split.len() == 1 { + let name = split[0].trim(); + if name.is_empty() { + // --groups=" " + // chroot: invalid group ‘ ’ + Err(ChrootError::InvalidGroup(name.to_string())) + } else { + // --groups="blah" + Ok(vec![name.to_string()]) + } + } else if split.iter().all(|s| s.is_empty()) { + // --groups="," + // chroot: invalid group list ‘,’ + Err(ChrootError::InvalidGroupList(list_str.to_string())) + } else { + let mut result = vec![]; + let mut err = false; + for name in split { + let trimmed_name = name.trim(); + if trimmed_name.is_empty() { + if name.is_empty() { + // --groups="," + continue; + } else { + // --groups=", " + // chroot: invalid group ‘ ’ + show!(ChrootError::InvalidGroup(name.to_string())); + err = true; + } + } else { + // TODO Figure out a better condition here. + if trimmed_name.starts_with(char::is_numeric) + && trimmed_name.ends_with(|c: char| !c.is_numeric()) + { + // --groups="0trail" + // chroot: invalid group ‘0trail’ + show!(ChrootError::InvalidGroup(name.to_string())); + err = true; + } else { + result.push(trimmed_name.to_string()); + } + } + } + if err { + Err(ChrootError::GroupsParsingFailed) + } else { + Ok(result) + } + } +} + +impl Options { + /// Parse parameters from the command-line arguments. + fn from(matches: &clap::ArgMatches) -> UResult { + let newroot = match matches.get_one::(options::NEWROOT) { + Some(v) => Path::new(v).to_path_buf(), + None => return Err(ChrootError::MissingNewRoot.into()), + }; + let groups = match matches.get_one::(options::GROUPS) { + None => None, + Some(s) => { + if s.is_empty() { + Some(vec![]) + } else { + Some(parse_group_list(s)?) + } + } + }; + let skip_chdir = matches.get_flag(options::SKIP_CHDIR); + let userspec = matches + .get_one::(options::USERSPEC) + .map(|s| parse_userspec(s)); + Ok(Self { + newroot, + skip_chdir, + groups, + userspec, + }) + } +} + #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_lossy(); - let matches = uu_app().try_get_matches_from(args).with_exit_code(125)?; let default_shell: &'static str = "/bin/sh"; let default_option: &'static str = "-i"; let user_shell = std::env::var("SHELL"); - let newroot: &Path = match matches.get_one::(options::NEWROOT) { - Some(v) => Path::new(v), - None => return Err(ChrootError::MissingNewRoot.into()), - }; + let options = Options::from(&matches)?; - let skip_chdir = matches.get_flag(options::SKIP_CHDIR); // We are resolving the path in case it is a symlink or /. or /../ - if skip_chdir - && canonicalize(newroot, MissingHandling::Normal, ResolveMode::Logical) - .unwrap() - .to_str() + if options.skip_chdir + && canonicalize( + &options.newroot, + MissingHandling::Normal, + ResolveMode::Logical, + ) + .unwrap() + .to_str() != Some("/") { return Err(UUsageError::new( @@ -60,8 +181,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { )); } - if !newroot.is_dir() { - return Err(ChrootError::NoSuchDirectory(format!("{}", newroot.display())).into()); + if !options.newroot.is_dir() { + return Err(ChrootError::NoSuchDirectory(format!("{}", options.newroot.display())).into()); } let commands = match matches.get_many::(options::COMMAND) { @@ -87,7 +208,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let chroot_args = &command[1..]; // NOTE: Tests can only trigger code beyond this point if they're invoked with root permissions - set_context(newroot, &matches)?; + set_context(&options)?; let pstatus = match process::Command::new(chroot_command) .args(chroot_args) @@ -100,7 +221,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } else { ChrootError::CommandFailed(command[0].to_string(), e) } - .into()) + .into()); } }; @@ -115,7 +236,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -127,35 +248,17 @@ pub fn uu_app() -> Command { .required(true) .index(1), ) - .arg( - Arg::new(options::USER) - .short('u') - .long(options::USER) - .help("User (ID or name) to switch before running the program") - .value_name("USER"), - ) - .arg( - Arg::new(options::GROUP) - .short('g') - .long(options::GROUP) - .help("Group (ID or name) to switch to") - .value_name("GROUP"), - ) .arg( Arg::new(options::GROUPS) - .short('G') .long(options::GROUPS) + .overrides_with(options::GROUPS) .help("Comma-separated list of groups to switch to") .value_name("GROUP1,GROUP2..."), ) .arg( Arg::new(options::USERSPEC) .long(options::USERSPEC) - .help( - "Colon-separated user and group to switch to. \ - Same as -u USER -g GROUP. \ - Userspec has higher preference than -u and/or -g", - ) + .help("Colon-separated user and group to switch to.") .value_name("USER:GROUP"), ) .arg( @@ -177,119 +280,178 @@ pub fn uu_app() -> Command { ) } -fn set_context(root: &Path, options: &clap::ArgMatches) -> UResult<()> { - let userspec_str = options.get_one::(options::USERSPEC); - let user_str = options - .get_one::(options::USER) - .map(|s| s.as_str()) - .unwrap_or_default(); - let group_str = options - .get_one::(options::GROUP) - .map(|s| s.as_str()) - .unwrap_or_default(); - let groups_str = options - .get_one::(options::GROUPS) - .map(|s| s.as_str()) - .unwrap_or_default(); - let skip_chdir = options.contains_id(options::SKIP_CHDIR); - let userspec = match userspec_str { - Some(u) => { - let s: Vec<&str> = u.split(':').collect(); - if s.len() != 2 || s.iter().any(|&spec| spec.is_empty()) { - return Err(ChrootError::InvalidUserspec(u.to_string()).into()); - }; - s - } - None => Vec::new(), - }; - - let (user, group) = if userspec.is_empty() { - (user_str, group_str) - } else { - (userspec[0], userspec[1]) - }; +/// Get the UID for the given username, falling back to numeric parsing. +/// +/// According to the documentation of GNU `chroot`, "POSIX requires that +/// these commands first attempt to resolve the specified string as a +/// name, and only once that fails, then try to interpret it as an ID." +fn name_to_uid(name: &str) -> Result { + match usr2uid(name) { + Ok(uid) => Ok(uid), + Err(_) => name + .parse::() + .map_err(|_| ChrootError::NoSuchUser), + } +} - enter_chroot(root, skip_chdir)?; +/// Get the GID for the given group name, falling back to numeric parsing. +/// +/// According to the documentation of GNU `chroot`, "POSIX requires that +/// these commands first attempt to resolve the specified string as a +/// name, and only once that fails, then try to interpret it as an ID." +fn name_to_gid(name: &str) -> Result { + match grp2gid(name) { + Ok(gid) => Ok(gid), + Err(_) => name + .parse::() + .map_err(|_| ChrootError::NoSuchGroup), + } +} - set_groups_from_str(groups_str)?; - set_main_group(group)?; - set_user(user)?; - Ok(()) +/// Get the list of group IDs for the given user. +/// +/// According to the GNU documentation, "the supplementary groups are +/// set according to the system defined list for that user". This +/// function gets that list. +fn supplemental_gids(uid: libc::uid_t) -> Vec { + match Passwd::locate(uid) { + Err(_) => vec![], + Ok(passwd) => passwd.belongs_to(), + } } -fn enter_chroot(root: &Path, skip_chdir: bool) -> UResult<()> { - let err = unsafe { - chroot( - CString::new(root.as_os_str().as_bytes().to_vec()) - .unwrap() - .as_bytes_with_nul() - .as_ptr() as *const libc::c_char, - ) - }; +/// Set the supplemental group IDs for this process. +fn set_supplemental_gids(gids: &[libc::gid_t]) -> std::io::Result<()> { + #[cfg(any(target_vendor = "apple", target_os = "freebsd", target_os = "openbsd"))] + let n = gids.len() as libc::c_int; + #[cfg(any(target_os = "linux", target_os = "android"))] + let n = gids.len() as libc::size_t; + let err = unsafe { setgroups(n, gids.as_ptr()) }; + if err == 0 { + Ok(()) + } else { + Err(Error::last_os_error()) + } +} +/// Set the group ID of this process. +fn set_gid(gid: libc::gid_t) -> std::io::Result<()> { + let err = unsafe { setgid(gid) }; if err == 0 { - if !skip_chdir { - std::env::set_current_dir(root).unwrap(); - } Ok(()) } else { - Err(ChrootError::CannotEnter(format!("{}", root.display()), Error::last_os_error()).into()) + Err(Error::last_os_error()) } } -fn set_main_group(group: &str) -> UResult<()> { - if !group.is_empty() { - let group_id = match entries::grp2gid(group) { - Ok(g) => g, - _ => return Err(ChrootError::NoSuchGroup(group.to_string()).into()), - }; - let err = unsafe { setgid(group_id) }; - if err != 0 { - return Err( - ChrootError::SetGidFailed(group_id.to_string(), Error::last_os_error()).into(), - ); - } +/// Set the user ID of this process. +fn set_uid(uid: libc::uid_t) -> std::io::Result<()> { + let err = unsafe { setuid(uid) }; + if err == 0 { + Ok(()) + } else { + Err(Error::last_os_error()) } - Ok(()) } -#[cfg(any(target_vendor = "apple", target_os = "freebsd"))] -fn set_groups(groups: &[libc::gid_t]) -> libc::c_int { - unsafe { setgroups(groups.len() as libc::c_int, groups.as_ptr()) } +/// What to do when the `--groups` argument is missing. +enum Strategy { + /// Do nothing. + Nothing, + /// Use the list of supplemental groups for the given user. + /// + /// If the `bool` parameter is `false` and the list of groups for + /// the given user is empty, then this will result in an error. + FromUID(libc::uid_t, bool), } -#[cfg(any(target_os = "linux", target_os = "android"))] -fn set_groups(groups: &[libc::gid_t]) -> libc::c_int { - unsafe { setgroups(groups.len() as libc::size_t, groups.as_ptr()) } +/// Set supplemental groups when the `--groups` argument is not specified. +fn handle_missing_groups(strategy: Strategy) -> Result<(), ChrootError> { + match strategy { + Strategy::Nothing => Ok(()), + Strategy::FromUID(uid, false) => { + let gids = supplemental_gids(uid); + if gids.is_empty() { + Err(ChrootError::NoGroupSpecified(uid)) + } else { + set_supplemental_gids(&gids).map_err(ChrootError::SetGroupsFailed) + } + } + Strategy::FromUID(uid, true) => { + let gids = supplemental_gids(uid); + set_supplemental_gids(&gids).map_err(ChrootError::SetGroupsFailed) + } + } } -fn set_groups_from_str(groups: &str) -> UResult<()> { - if !groups.is_empty() { - let mut groups_vec = vec![]; - for group in groups.split(',') { - let gid = match entries::grp2gid(group) { - Ok(g) => g, - Err(_) => return Err(ChrootError::NoSuchGroup(group.to_string()).into()), - }; - groups_vec.push(gid); +/// Set supplemental groups for this process. +fn set_supplemental_gids_with_strategy( + strategy: Strategy, + groups: Option<&Vec>, +) -> Result<(), ChrootError> { + match groups { + None => handle_missing_groups(strategy), + Some(groups) => { + let mut gids = vec![]; + for group in groups { + gids.push(name_to_gid(group)?); + } + set_supplemental_gids(&gids).map_err(ChrootError::SetGroupsFailed) + } + } +} + +/// Change the root, set the user ID, and set the group IDs for this process. +fn set_context(options: &Options) -> UResult<()> { + enter_chroot(&options.newroot, options.skip_chdir)?; + match &options.userspec { + None | Some(UserSpec::NeitherGroupNorUser) => { + let strategy = Strategy::Nothing; + set_supplemental_gids_with_strategy(strategy, options.groups.as_ref())?; } - let err = set_groups(&groups_vec); - if err != 0 { - return Err(ChrootError::SetGroupsFailed(Error::last_os_error()).into()); + Some(UserSpec::UserOnly(user)) => { + let uid = name_to_uid(user)?; + let gid = uid as libc::gid_t; + let strategy = Strategy::FromUID(uid, false); + set_supplemental_gids_with_strategy(strategy, options.groups.as_ref())?; + set_gid(gid).map_err(|e| ChrootError::SetGidFailed(user.to_string(), e))?; + set_uid(uid).map_err(|e| ChrootError::SetUserFailed(user.to_string(), e))?; + } + Some(UserSpec::GroupOnly(group)) => { + let gid = name_to_gid(group)?; + let strategy = Strategy::Nothing; + set_supplemental_gids_with_strategy(strategy, options.groups.as_ref())?; + set_gid(gid).map_err(|e| ChrootError::SetGidFailed(group.to_string(), e))?; + } + Some(UserSpec::UserAndGroup(user, group)) => { + let uid = name_to_uid(user)?; + let gid = name_to_gid(group)?; + let strategy = Strategy::FromUID(uid, true); + set_supplemental_gids_with_strategy(strategy, options.groups.as_ref())?; + set_gid(gid).map_err(|e| ChrootError::SetGidFailed(group.to_string(), e))?; + set_uid(uid).map_err(|e| ChrootError::SetUserFailed(user.to_string(), e))?; } } Ok(()) } -fn set_user(user: &str) -> UResult<()> { - if !user.is_empty() { - let user_id = entries::usr2uid(user).unwrap(); - let err = unsafe { setuid(user_id as libc::uid_t) }; - if err != 0 { - return Err( - ChrootError::SetUserFailed(user.to_string(), Error::last_os_error()).into(), - ); +fn enter_chroot(root: &Path, skip_chdir: bool) -> UResult<()> { + let err = unsafe { + chroot( + CString::new(root.as_os_str().as_bytes().to_vec()) + .map_err(|e| ChrootError::CannotEnter("root".to_string(), e.into()))? + .as_bytes_with_nul() + .as_ptr() + .cast::(), + ) + }; + + if err == 0 { + if !skip_chdir { + std::env::set_current_dir("/")?; } + Ok(()) + } else { + Err(ChrootError::CannotEnter(format!("{}", root.display()), Error::last_os_error()).into()) } - Ok(()) } diff --git a/src/uu/chroot/src/error.rs b/src/uu/chroot/src/error.rs index 526f1a75a43..78fd7ad64e7 100644 --- a/src/uu/chroot/src/error.rs +++ b/src/uu/chroot/src/error.rs @@ -4,47 +4,71 @@ // file that was distributed with this source code. // spell-checker:ignore NEWROOT Userspec userspec //! Errors returned by chroot. -use std::fmt::Display; use std::io::Error; +use thiserror::Error; use uucore::display::Quotable; use uucore::error::UError; +use uucore::libc; /// Errors that can happen while executing chroot. -#[derive(Debug)] +#[derive(Debug, Error)] pub enum ChrootError { /// Failed to enter the specified directory. - CannotEnter(String, Error), + #[error("cannot chroot to {dir}: {err}", dir = .0.quote(), err = .1)] + CannotEnter(String, #[source] Error), /// Failed to execute the specified command. - CommandFailed(String, Error), + #[error("failed to run command {cmd}: {err}", cmd = .0.to_string().quote(), err = .1)] + CommandFailed(String, #[source] Error), /// Failed to find the specified command. - CommandNotFound(String, Error), + #[error("failed to run command {cmd}: {err}", cmd = .0.to_string().quote(), err = .1)] + CommandNotFound(String, #[source] Error), - /// The given user and group specification was invalid. - InvalidUserspec(String), + #[error("--groups parsing failed")] + GroupsParsingFailed, + + #[error("invalid group: {group}", group = .0.quote())] + InvalidGroup(String), + + #[error("invalid group list: {list}", list = .0.quote())] + InvalidGroupList(String), /// The new root directory was not given. + #[error( + "Missing operand: NEWROOT\nTry '{0} --help' for more information.", + uucore::execution_phrase() + )] MissingNewRoot, + #[error("no group specified for unknown uid: {0}")] + NoGroupSpecified(libc::uid_t), + + /// Failed to find the specified user. + #[error("invalid user")] + NoSuchUser, + /// Failed to find the specified group. - NoSuchGroup(String), + #[error("invalid group")] + NoSuchGroup, /// The given directory does not exist. + #[error("cannot change root directory to {dir}: no such directory", dir = .0.quote())] NoSuchDirectory(String), /// The call to `setgid()` failed. - SetGidFailed(String, Error), + #[error("cannot set gid to {gid}: {err}", gid = .0, err = .1)] + SetGidFailed(String, #[source] Error), /// The call to `setgroups()` failed. + #[error("cannot set groups: {0}")] SetGroupsFailed(Error), /// The call to `setuid()` failed. - SetUserFailed(String, Error), + #[error("cannot set user to {user}: {err}", user = .0.maybe_quote(), err = .1)] + SetUserFailed(String, #[source] Error), } -impl std::error::Error for ChrootError {} - impl UError for ChrootError { // 125 if chroot itself fails // 126 if command is found but cannot be invoked @@ -57,31 +81,3 @@ impl UError for ChrootError { } } } - -impl Display for ChrootError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::CannotEnter(s, e) => write!(f, "cannot chroot to {}: {}", s.quote(), e,), - Self::CommandFailed(s, e) | Self::CommandNotFound(s, e) => { - write!(f, "failed to run command {}: {}", s.to_string().quote(), e,) - } - Self::InvalidUserspec(s) => write!(f, "invalid userspec: {}", s.quote(),), - Self::MissingNewRoot => write!( - f, - "Missing operand: NEWROOT\nTry '{} --help' for more information.", - uucore::execution_phrase(), - ), - Self::NoSuchGroup(s) => write!(f, "no such group: {}", s.maybe_quote(),), - Self::NoSuchDirectory(s) => write!( - f, - "cannot change root directory to {}: no such directory", - s.quote(), - ), - Self::SetGidFailed(s, e) => write!(f, "cannot set gid to {s}: {e}"), - Self::SetGroupsFailed(e) => write!(f, "cannot set groups: {e}"), - Self::SetUserFailed(s, e) => { - write!(f, "cannot set user to {}: {}", s.maybe_quote(), e) - } - } - } -} diff --git a/src/uu/cksum/Cargo.toml b/src/uu/cksum/Cargo.toml index 49a881f706f..c49288aa9d4 100644 --- a/src/uu/cksum/Cargo.toml +++ b/src/uu/cksum/Cargo.toml @@ -1,23 +1,27 @@ [package] name = "uu_cksum" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "cksum ~ (uutils) display CRC and size of input" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/cksum" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/cksum.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true, features = ["sum"] } +uucore = { workspace = true, features = ["checksum", "encoding", "sum"] } hex = { workspace = true } +regex = { workspace = true } [[bin]] name = "cksum" diff --git a/src/uu/cksum/cksum.md b/src/uu/cksum/cksum.md index 4b0d25f32c3..5ca83b40150 100644 --- a/src/uu/cksum/cksum.md +++ b/src/uu/cksum/cksum.md @@ -13,6 +13,7 @@ DIGEST determines the digest algorithm and default output format: - `sysv`: (equivalent to sum -s) - `bsd`: (equivalent to sum -r) - `crc`: (equivalent to cksum) +- `crc32b`: (only available through cksum) - `md5`: (equivalent to md5sum) - `sha1`: (equivalent to sha1sum) - `sha224`: (equivalent to sha224sum) diff --git a/src/uu/cksum/src/cksum.rs b/src/uu/cksum/src/cksum.rs index 6c9c795821b..a1a9115d9a0 100644 --- a/src/uu/cksum/src/cksum.rs +++ b/src/uu/cksum/src/cksum.rs @@ -4,104 +4,48 @@ // file that was distributed with this source code. // spell-checker:ignore (ToDO) fname, algo -use clap::{crate_version, Arg, ArgAction, Command}; -use hex::encode; -use std::ffi::OsStr; +use clap::builder::ValueParser; +use clap::{Arg, ArgAction, Command, value_parser}; +use std::ffi::{OsStr, OsString}; use std::fs::File; -use std::io::{self, stdin, BufReader, Read}; +use std::io::{self, BufReader, Read, Write, stdin, stdout}; use std::iter; use std::path::Path; +use uucore::checksum::{ + ALGORITHM_OPTIONS_BLAKE2B, ALGORITHM_OPTIONS_BSD, ALGORITHM_OPTIONS_CRC, + ALGORITHM_OPTIONS_CRC32B, ALGORITHM_OPTIONS_SYSV, ChecksumError, ChecksumOptions, + ChecksumVerbose, SUPPORTED_ALGORITHMS, calculate_blake2b_length, detect_algo, digest_reader, + perform_checksum_validation, +}; use uucore::{ - error::{FromIo, UResult}, + encoding, + error::{FromIo, UResult, USimpleError}, format_usage, help_about, help_section, help_usage, - sum::{ - div_ceil, Blake2b, Digest, DigestWriter, Md5, Sha1, Sha224, Sha256, Sha384, Sha512, Sm3, - BSD, CRC, SYSV, - }, + line_ending::LineEnding, + os_str_as_bytes, show, + sum::Digest, }; const USAGE: &str = help_usage!("cksum.md"); const ABOUT: &str = help_about!("cksum.md"); const AFTER_HELP: &str = help_section!("after help", "cksum.md"); -const ALGORITHM_OPTIONS_SYSV: &str = "sysv"; -const ALGORITHM_OPTIONS_BSD: &str = "bsd"; -const ALGORITHM_OPTIONS_CRC: &str = "crc"; -const ALGORITHM_OPTIONS_MD5: &str = "md5"; -const ALGORITHM_OPTIONS_SHA1: &str = "sha1"; -const ALGORITHM_OPTIONS_SHA224: &str = "sha224"; -const ALGORITHM_OPTIONS_SHA256: &str = "sha256"; -const ALGORITHM_OPTIONS_SHA384: &str = "sha384"; -const ALGORITHM_OPTIONS_SHA512: &str = "sha512"; -const ALGORITHM_OPTIONS_BLAKE2B: &str = "blake2b"; -const ALGORITHM_OPTIONS_SM3: &str = "sm3"; - -fn detect_algo(program: &str) -> (&'static str, Box, usize) { - match program { - ALGORITHM_OPTIONS_SYSV => ( - ALGORITHM_OPTIONS_SYSV, - Box::new(SYSV::new()) as Box, - 512, - ), - ALGORITHM_OPTIONS_BSD => ( - ALGORITHM_OPTIONS_BSD, - Box::new(BSD::new()) as Box, - 1024, - ), - ALGORITHM_OPTIONS_CRC => ( - ALGORITHM_OPTIONS_CRC, - Box::new(CRC::new()) as Box, - 256, - ), - ALGORITHM_OPTIONS_MD5 => ( - ALGORITHM_OPTIONS_MD5, - Box::new(Md5::new()) as Box, - 128, - ), - ALGORITHM_OPTIONS_SHA1 => ( - ALGORITHM_OPTIONS_SHA1, - Box::new(Sha1::new()) as Box, - 160, - ), - ALGORITHM_OPTIONS_SHA224 => ( - ALGORITHM_OPTIONS_SHA224, - Box::new(Sha224::new()) as Box, - 224, - ), - ALGORITHM_OPTIONS_SHA256 => ( - ALGORITHM_OPTIONS_SHA256, - Box::new(Sha256::new()) as Box, - 256, - ), - ALGORITHM_OPTIONS_SHA384 => ( - ALGORITHM_OPTIONS_SHA384, - Box::new(Sha384::new()) as Box, - 384, - ), - ALGORITHM_OPTIONS_SHA512 => ( - ALGORITHM_OPTIONS_SHA512, - Box::new(Sha512::new()) as Box, - 512, - ), - ALGORITHM_OPTIONS_BLAKE2B => ( - ALGORITHM_OPTIONS_BLAKE2B, - Box::new(Blake2b::new()) as Box, - 512, - ), - ALGORITHM_OPTIONS_SM3 => ( - ALGORITHM_OPTIONS_SM3, - Box::new(Sm3::new()) as Box, - 512, - ), - _ => unreachable!("unknown algorithm: clap should have prevented this case"), - } +#[derive(Debug, PartialEq)] +enum OutputFormat { + Hexadecimal, + Raw, + Base64, } struct Options { algo_name: &'static str, digest: Box, output_bits: usize, - untagged: bool, + tag: bool, // will cover the --untagged option + length: Option, + output_format: OutputFormat, + asterisk: bool, // if we display an asterisk or not (--binary/--text) + line_ending: LineEnding, } /// Calculate checksum @@ -115,131 +59,280 @@ fn cksum<'a, I>(mut options: Options, files: I) -> UResult<()> where I: Iterator, { + let files: Vec<_> = files.collect(); + if options.output_format == OutputFormat::Raw && files.len() > 1 { + return Err(Box::new(ChecksumError::RawMultipleFiles)); + } + for filename in files { let filename = Path::new(filename); let stdin_buf; let file_buf; let not_file = filename == OsStr::new("-"); + + // Handle the file input let mut file = BufReader::new(if not_file { stdin_buf = stdin(); Box::new(stdin_buf) as Box } else if filename.is_dir() { Box::new(BufReader::new(io::empty())) as Box } else { - file_buf = - File::open(filename).map_err_context(|| filename.to_str().unwrap().to_string())?; + file_buf = match File::open(filename) { + Ok(file) => file, + Err(err) => { + show!(err.map_err_context(|| filename.to_string_lossy().to_string())); + continue; + } + }; Box::new(file_buf) as Box }); - let (sum, sz) = digest_read(&mut options.digest, &mut file, options.output_bits) - .map_err_context(|| "failed to read input".to_string())?; + if filename.is_dir() { + show!(USimpleError::new( + 1, + format!("{}: Is a directory", filename.display()) + )); + continue; + } + + let (sum_hex, sz) = + digest_reader(&mut options.digest, &mut file, false, options.output_bits) + .map_err_context(|| "failed to read input".to_string())?; + + let sum = match options.output_format { + OutputFormat::Raw => { + let bytes = match options.algo_name { + ALGORITHM_OPTIONS_CRC => sum_hex.parse::().unwrap().to_be_bytes().to_vec(), + ALGORITHM_OPTIONS_SYSV | ALGORITHM_OPTIONS_BSD => { + sum_hex.parse::().unwrap().to_be_bytes().to_vec() + } + _ => hex::decode(sum_hex).unwrap(), + }; + // Cannot handle multiple files anyway, output immediately. + stdout().write_all(&bytes)?; + return Ok(()); + } + OutputFormat::Hexadecimal => sum_hex, + OutputFormat::Base64 => match options.algo_name { + ALGORITHM_OPTIONS_CRC + | ALGORITHM_OPTIONS_CRC32B + | ALGORITHM_OPTIONS_SYSV + | ALGORITHM_OPTIONS_BSD => sum_hex, + _ => encoding::for_cksum::BASE64.encode(&hex::decode(sum_hex).unwrap()), + }, + }; // The BSD checksum output is 5 digit integer let bsd_width = 5; - match (options.algo_name, not_file) { - (ALGORITHM_OPTIONS_SYSV, true) => println!( - "{} {}", - sum.parse::().unwrap(), - div_ceil(sz, options.output_bits) - ), - (ALGORITHM_OPTIONS_SYSV, false) => println!( - "{} {} {}", - sum.parse::().unwrap(), - div_ceil(sz, options.output_bits), - filename.display() + let (before_filename, should_print_filename, after_filename) = match options.algo_name { + ALGORITHM_OPTIONS_SYSV => ( + format!( + "{} {}{}", + sum.parse::().unwrap(), + sz.div_ceil(options.output_bits), + if not_file { "" } else { " " } + ), + !not_file, + String::new(), ), - (ALGORITHM_OPTIONS_BSD, true) => println!( - "{:0bsd_width$} {:bsd_width$}", - sum.parse::().unwrap(), - div_ceil(sz, options.output_bits) + ALGORITHM_OPTIONS_BSD => ( + format!( + "{:0bsd_width$} {:bsd_width$}{}", + sum.parse::().unwrap(), + sz.div_ceil(options.output_bits), + if not_file { "" } else { " " } + ), + !not_file, + String::new(), ), - (ALGORITHM_OPTIONS_BSD, false) => println!( - "{:0bsd_width$} {:bsd_width$} {}", - sum.parse::().unwrap(), - div_ceil(sz, options.output_bits), - filename.display() + ALGORITHM_OPTIONS_CRC | ALGORITHM_OPTIONS_CRC32B => ( + format!("{sum} {sz}{}", if not_file { "" } else { " " }), + !not_file, + String::new(), ), - (ALGORITHM_OPTIONS_CRC, true) => println!("{sum} {sz}"), - (ALGORITHM_OPTIONS_CRC, false) => println!("{sum} {sz} {}", filename.display()), - (ALGORITHM_OPTIONS_BLAKE2B, _) if !options.untagged => { - println!("BLAKE2b ({}) = {sum}", filename.display()); + ALGORITHM_OPTIONS_BLAKE2B if options.tag => { + ( + if let Some(length) = options.length { + // Multiply by 8 here, as we want to print the length in bits. + format!("BLAKE2b-{} (", length * 8) + } else { + "BLAKE2b (".to_owned() + }, + true, + format!(") = {sum}"), + ) } _ => { - if options.untagged { - println!("{sum} {}", filename.display()); + if options.tag { + ( + format!("{} (", options.algo_name.to_ascii_uppercase()), + true, + format!(") = {sum}"), + ) } else { - println!( - "{} ({}) = {sum}", - options.algo_name.to_ascii_uppercase(), - filename.display() - ); + let prefix = if options.asterisk { "*" } else { " " }; + (format!("{sum} {prefix}"), true, String::new()) } } + }; + print!("{before_filename}"); + if should_print_filename { + // The filename might not be valid UTF-8, and filename.display() would mangle the names. + // Therefore, emit the bytes directly to stdout, without any attempt at encoding them. + let _dropped_result = stdout().write_all(os_str_as_bytes(filename.as_os_str())?); } + print!("{after_filename}{}", options.line_ending); } Ok(()) } -fn digest_read( - digest: &mut Box, - reader: &mut BufReader, - output_bits: usize, -) -> io::Result<(String, usize)> { - digest.reset(); - - // Read bytes from `reader` and write those bytes to `digest`. - // - // If `binary` is `false` and the operating system is Windows, then - // `DigestWriter` replaces "\r\n" with "\n" before it writes the - // bytes into `digest`. Otherwise, it just inserts the bytes as-is. - // - // In order to support replacing "\r\n", we must call `finalize()` - // in order to support the possibility that the last character read - // from the reader was "\r". (This character gets buffered by - // `DigestWriter` and only written if the following character is - // "\n". But when "\r" is the last character read, we need to force - // it to be written.) - let mut digest_writer = DigestWriter::new(digest, true); - let output_size = std::io::copy(reader, &mut digest_writer)? as usize; - digest_writer.finalize(); - - if digest.output_bits() > 0 { - Ok((digest.result_str(), output_size)) - } else { - // Assume it's SHAKE. result_str() doesn't work with shake (as of 8/30/2016) - let mut bytes = Vec::new(); - bytes.resize((output_bits + 7) / 8, 0); - digest.hash_finalize(&mut bytes); - Ok((encode(bytes), output_size)) - } -} - mod options { pub const ALGORITHM: &str = "algorithm"; pub const FILE: &str = "file"; pub const UNTAGGED: &str = "untagged"; + pub const TAG: &str = "tag"; + pub const LENGTH: &str = "length"; + pub const RAW: &str = "raw"; + pub const BASE64: &str = "base64"; + pub const CHECK: &str = "check"; + pub const STRICT: &str = "strict"; + pub const TEXT: &str = "text"; + pub const BINARY: &str = "binary"; + pub const STATUS: &str = "status"; + pub const WARN: &str = "warn"; + pub const IGNORE_MISSING: &str = "ignore-missing"; + pub const QUIET: &str = "quiet"; + pub const ZERO: &str = "zero"; +} + +/*** + * cksum has a bunch of legacy behavior. + * We handle this in this function to make sure they are self contained + * and "easier" to understand + */ +fn handle_tag_text_binary_flags>( + args: impl Iterator, +) -> UResult<(bool, bool)> { + let mut tag = true; + let mut binary = false; + + // --binary, --tag and --untagged are tight together: none of them + // conflicts with each other but --tag will reset "binary" and set "tag". + + for arg in args { + let arg = arg.as_ref(); + if arg == "-b" || arg == "--binary" { + binary = true; + } else if arg == "--tag" { + tag = true; + binary = false; + } else if arg == "--untagged" { + tag = false; + } + } + + Ok((tag, !tag && binary)) } #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_ignore(); - let matches = uu_app().try_get_matches_from(args)?; + let check = matches.get_flag(options::CHECK); + let algo_name: &str = match matches.get_one::(options::ALGORITHM) { Some(v) => v, - None => ALGORITHM_OPTIONS_CRC, + None => { + if check { + // if we are doing a --check, we should not default to crc + "" + } else { + ALGORITHM_OPTIONS_CRC + } + } + }; + + let input_length = matches.get_one::(options::LENGTH); + + let length = match input_length { + Some(length) => { + if algo_name == ALGORITHM_OPTIONS_BLAKE2B { + calculate_blake2b_length(*length)? + } else { + return Err(ChecksumError::LengthOnlyForBlake2b.into()); + } + } + None => None, + }; + + if ["bsd", "crc", "sysv", "crc32b"].contains(&algo_name) && check { + return Err(ChecksumError::AlgorithmNotSupportedWithCheck.into()); + } + + if check { + let text_flag = matches.get_flag(options::TEXT); + let binary_flag = matches.get_flag(options::BINARY); + let strict = matches.get_flag(options::STRICT); + let status = matches.get_flag(options::STATUS); + let warn = matches.get_flag(options::WARN); + let ignore_missing = matches.get_flag(options::IGNORE_MISSING); + let quiet = matches.get_flag(options::QUIET); + let tag = matches.get_flag(options::TAG); + + if tag || binary_flag || text_flag { + return Err(ChecksumError::BinaryTextConflict.into()); + } + // Determine the appropriate algorithm option to pass + let algo_option = if algo_name.is_empty() { + None + } else { + Some(algo_name) + }; + + // Execute the checksum validation based on the presence of files or the use of stdin + + let files = matches.get_many::(options::FILE).map_or_else( + || iter::once(OsStr::new("-")).collect::>(), + |files| files.map(OsStr::new).collect::>(), + ); + + let verbose = ChecksumVerbose::new(status, quiet, warn); + + let opts = ChecksumOptions { + binary: binary_flag, + ignore_missing, + strict, + verbose, + }; + + return perform_checksum_validation(files.iter().copied(), algo_option, length, opts); + } + + let (tag, asterisk) = handle_tag_text_binary_flags(std::env::args_os())?; + + let algo = detect_algo(algo_name, length)?; + let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO)); + + let output_format = if matches.get_flag(options::RAW) { + OutputFormat::Raw + } else if matches.get_flag(options::BASE64) { + OutputFormat::Base64 + } else { + OutputFormat::Hexadecimal }; - let (name, algo, bits) = detect_algo(algo_name); let opts = Options { - algo_name: name, - digest: algo, - output_bits: bits, - untagged: matches.get_flag(options::UNTAGGED), + algo_name: algo.name, + digest: (algo.create_fn)(), + output_bits: algo.bits, + length, + tag, + output_format, + asterisk, + line_ending, }; - match matches.get_many::(options::FILE) { + match matches.get_many::(options::FILE) { Some(files) => cksum(opts, files.map(OsStr::new))?, None => cksum(opts, iter::once(OsStr::new("-")))?, }; @@ -249,14 +342,16 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) + .args_override_self(true) .arg( Arg::new(options::FILE) .hide(true) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) + .value_parser(ValueParser::os_string()) .value_hint(clap::ValueHint::FilePath), ) .arg( @@ -265,25 +360,130 @@ pub fn uu_app() -> Command { .short('a') .help("select the digest type to use. See DIGEST below") .value_name("ALGORITHM") - .value_parser([ - ALGORITHM_OPTIONS_SYSV, - ALGORITHM_OPTIONS_BSD, - ALGORITHM_OPTIONS_CRC, - ALGORITHM_OPTIONS_MD5, - ALGORITHM_OPTIONS_SHA1, - ALGORITHM_OPTIONS_SHA224, - ALGORITHM_OPTIONS_SHA256, - ALGORITHM_OPTIONS_SHA384, - ALGORITHM_OPTIONS_SHA512, - ALGORITHM_OPTIONS_BLAKE2B, - ALGORITHM_OPTIONS_SM3, - ]), + .value_parser(SUPPORTED_ALGORITHMS), ) .arg( Arg::new(options::UNTAGGED) .long(options::UNTAGGED) .help("create a reversed style checksum, without digest type") + .action(ArgAction::SetTrue) + .overrides_with(options::TAG), + ) + .arg( + Arg::new(options::TAG) + .long(options::TAG) + .help("create a BSD style checksum, undo --untagged (default)") + .action(ArgAction::SetTrue) + .overrides_with(options::UNTAGGED), + ) + .arg( + Arg::new(options::LENGTH) + .long(options::LENGTH) + .value_parser(value_parser!(usize)) + .short('l') + .help( + "digest length in bits; must not exceed the max for the blake2 algorithm \ + and must be a multiple of 8", + ) + .action(ArgAction::Set), + ) + .arg( + Arg::new(options::RAW) + .long(options::RAW) + .help("emit a raw binary digest, not hexadecimal") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::STRICT) + .long(options::STRICT) + .help("exit non-zero for improperly formatted checksum lines") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::CHECK) + .short('c') + .long(options::CHECK) + .help("read hashsums from the FILEs and check them") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::BASE64) + .long(options::BASE64) + .help("emit a base64 digest, not hexadecimal") + .action(ArgAction::SetTrue) + // Even though this could easily just override an earlier '--raw', + // GNU cksum does not permit these flags to be combined: + .conflicts_with(options::RAW), + ) + .arg( + Arg::new(options::TEXT) + .long(options::TEXT) + .short('t') + .hide(true) + .overrides_with(options::BINARY) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::BINARY) + .long(options::BINARY) + .short('b') + .hide(true) + .overrides_with(options::TEXT) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::WARN) + .short('w') + .long("warn") + .help("warn about improperly formatted checksum lines") + .action(ArgAction::SetTrue) + .overrides_with_all([options::STATUS, options::QUIET]), + ) + .arg( + Arg::new(options::STATUS) + .long("status") + .help("don't output anything, status code shows success") + .action(ArgAction::SetTrue) + .overrides_with_all([options::WARN, options::QUIET]), + ) + .arg( + Arg::new(options::QUIET) + .long(options::QUIET) + .help("don't print OK for each successfully verified file") + .action(ArgAction::SetTrue) + .overrides_with_all([options::WARN, options::STATUS]), + ) + .arg( + Arg::new(options::IGNORE_MISSING) + .long(options::IGNORE_MISSING) + .help("don't fail or report status for missing files") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::ZERO) + .long(options::ZERO) + .short('z') + .help( + "end each output line with NUL, not newline,\n and disable file name escaping", + ) .action(ArgAction::SetTrue), ) .after_help(AFTER_HELP) } + +#[cfg(test)] +mod tests { + use crate::calculate_blake2b_length; + + #[test] + fn test_calculate_length() { + assert_eq!(calculate_blake2b_length(256).unwrap(), Some(32)); + assert_eq!(calculate_blake2b_length(512).unwrap(), None); + assert_eq!(calculate_blake2b_length(256).unwrap(), Some(32)); + calculate_blake2b_length(255).unwrap_err(); + + calculate_blake2b_length(33).unwrap_err(); + + calculate_blake2b_length(513).unwrap_err(); + } +} diff --git a/src/uu/comm/Cargo.toml b/src/uu/comm/Cargo.toml index e4a1dac82fa..71617428039 100644 --- a/src/uu/comm/Cargo.toml +++ b/src/uu/comm/Cargo.toml @@ -1,22 +1,25 @@ [package] name = "uu_comm" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "comm ~ (uutils) compare sorted inputs" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/comm" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/comm.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["fs"] } [[bin]] name = "comm" diff --git a/src/uu/comm/src/comm.rs b/src/uu/comm/src/comm.rs index e6977142ef5..11752c331a5 100644 --- a/src/uu/comm/src/comm.rs +++ b/src/uu/comm/src/comm.rs @@ -3,17 +3,17 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) delim mkdelim +// spell-checker:ignore (ToDO) delim mkdelim pairable use std::cmp::Ordering; -use std::fs::File; -use std::io::{self, stdin, BufRead, BufReader, Stdin}; -use std::path::Path; -use uucore::error::{FromIo, UResult}; +use std::fs::{File, metadata}; +use std::io::{self, BufRead, BufReader, Read, Stdin, stdin}; +use uucore::error::{FromIo, UResult, USimpleError}; +use uucore::fs::paths_refer_to_same_file; use uucore::line_ending::LineEnding; use uucore::{format_usage, help_about, help_usage}; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; const ABOUT: &str = help_about!("comm.md"); const USAGE: &str = help_usage!("comm.md"); @@ -28,6 +28,30 @@ mod options { pub const FILE_2: &str = "FILE2"; pub const TOTAL: &str = "total"; pub const ZERO_TERMINATED: &str = "zero-terminated"; + pub const CHECK_ORDER: &str = "check-order"; + pub const NO_CHECK_ORDER: &str = "nocheck-order"; +} + +#[derive(Debug, Clone, Copy)] +enum FileNumber { + One, + Two, +} + +impl FileNumber { + fn as_str(&self) -> &'static str { + match self { + FileNumber::One => "1", + FileNumber::Two => "2", + } + } +} + +struct OrderChecker { + last_line: Vec, + file_num: FileNumber, + check_order: bool, + has_error: bool, } enum Input { @@ -42,7 +66,7 @@ struct LineReader { impl LineReader { fn new(input: Input, line_ending: LineEnding) -> Self { - Self { input, line_ending } + Self { line_ending, input } } fn read_line(&mut self, buf: &mut Vec) -> io::Result { @@ -61,12 +85,74 @@ impl LineReader { } } -fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { - let delim = match opts.get_one::(options::DELIMITER).unwrap().as_str() { - "" => "\0", - delim => delim, - }; +impl OrderChecker { + fn new(file_num: FileNumber, check_order: bool) -> Self { + Self { + last_line: Vec::new(), + file_num, + check_order, + has_error: false, + } + } + + fn verify_order(&mut self, current_line: &[u8]) -> bool { + if self.last_line.is_empty() { + self.last_line = current_line.to_vec(); + return true; + } + + let is_ordered = current_line >= &self.last_line; + if !is_ordered && !self.has_error { + eprintln!( + "comm: file {} is not in sorted order", + self.file_num.as_str() + ); + self.has_error = true; + } + + self.last_line = current_line.to_vec(); + is_ordered || !self.check_order + } +} + +// Check if two files are identical by comparing their contents +pub fn are_files_identical(path1: &str, path2: &str) -> io::Result { + // First compare file sizes + let metadata1 = metadata(path1)?; + let metadata2 = metadata(path2)?; + + if metadata1.len() != metadata2.len() { + return Ok(false); + } + + let file1 = File::open(path1)?; + let file2 = File::open(path2)?; + + let mut reader1 = BufReader::new(file1); + let mut reader2 = BufReader::new(file2); + + let mut buffer1 = [0; 8192]; + let mut buffer2 = [0; 8192]; + + loop { + let bytes1 = reader1.read(&mut buffer1)?; + let bytes2 = reader2.read(&mut buffer2)?; + + if bytes1 != bytes2 { + return Ok(false); + } + + if bytes1 == 0 { + return Ok(true); + } + + if buffer1[..bytes1] != buffer2[..bytes2] { + return Ok(false); + } + } +} +fn comm(a: &mut LineReader, b: &mut LineReader, delim: &str, opts: &ArgMatches) -> UResult<()> { let width_col_1 = usize::from(!opts.get_flag(options::COLUMN_1)); let width_col_2 = usize::from(!opts.get_flag(options::COLUMN_2)); @@ -82,6 +168,26 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { let mut total_col_2 = 0; let mut total_col_3 = 0; + let check_order = opts.get_flag(options::CHECK_ORDER); + let no_check_order = opts.get_flag(options::NO_CHECK_ORDER); + + // Determine if we should perform order checking + let should_check_order = !no_check_order + && (check_order + || if let (Some(file1), Some(file2)) = ( + opts.get_one::(options::FILE_1), + opts.get_one::(options::FILE_2), + ) { + !(paths_refer_to_same_file(file1, file2, true) + || are_files_identical(file1, file2).unwrap_or(false)) + } else { + true + }); + + let mut checker1 = OrderChecker::new(FileNumber::One, check_order); + let mut checker2 = OrderChecker::new(FileNumber::Two, check_order); + let mut input_error = false; + while na.is_ok() || nb.is_ok() { let ord = match (na.is_ok(), nb.is_ok()) { (false, true) => Ordering::Greater, @@ -97,6 +203,9 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { match ord { Ordering::Less => { + if should_check_order && !checker1.verify_order(ra) { + break; + } if !opts.get_flag(options::COLUMN_1) { print!("{}", String::from_utf8_lossy(ra)); } @@ -105,6 +214,9 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { total_col_1 += 1; } Ordering::Greater => { + if should_check_order && !checker2.verify_order(rb) { + break; + } if !opts.get_flag(options::COLUMN_2) { print!("{delim_col_2}{}", String::from_utf8_lossy(rb)); } @@ -113,6 +225,10 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { total_col_2 += 1; } Ordering::Equal => { + if should_check_order && (!checker1.verify_order(ra) || !checker2.verify_order(rb)) + { + break; + } if !opts.get_flag(options::COLUMN_3) { print!("{delim_col_3}{}", String::from_utf8_lossy(ra)); } @@ -123,19 +239,37 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { total_col_3 += 1; } } + + // Track if we've seen any order errors + if (checker1.has_error || checker2.has_error) && !input_error && !check_order { + input_error = true; + } } if opts.get_flag(options::TOTAL) { let line_ending = LineEnding::from_zero_flag(opts.get_flag(options::ZERO_TERMINATED)); print!("{total_col_1}{delim}{total_col_2}{delim}{total_col_3}{delim}total{line_ending}"); } + + if should_check_order && (checker1.has_error || checker2.has_error) { + // Print the input error message once at the end + if input_error { + eprintln!("comm: input is not in sorted order"); + } + Err(USimpleError::new(1, "")) + } else { + Ok(()) + } } fn open_file(name: &str, line_ending: LineEnding) -> io::Result { if name == "-" { Ok(LineReader::new(Input::Stdin(stdin()), line_ending)) } else { - let f = File::open(Path::new(name))?; + if metadata(name)?.is_dir() { + return Err(io::Error::other("Is a directory")); + } + let f = File::open(name)?; Ok(LineReader::new( Input::FileIn(BufReader::new(f)), line_ending, @@ -145,8 +279,6 @@ fn open_file(name: &str, line_ending: LineEnding) -> io::Result { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_lossy(); - let matches = uu_app().try_get_matches_from(args)?; let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO_TERMINATED)); let filename1 = matches.get_one::(options::FILE_1).unwrap(); @@ -154,16 +286,38 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let mut f1 = open_file(filename1, line_ending).map_err_context(|| filename1.to_string())?; let mut f2 = open_file(filename2, line_ending).map_err_context(|| filename2.to_string())?; - comm(&mut f1, &mut f2, &matches); - Ok(()) + // Due to default_value(), there must be at least one value here, thus unwrap() must not panic. + let all_delimiters = matches + .get_many::(options::DELIMITER) + .unwrap() + .map(String::from) + .collect::>(); + for delim in &all_delimiters[1..] { + // Note that this check is very different from ".conflicts_with_self(true).action(ArgAction::Set)", + // as this accepts duplicate *identical* arguments. + if delim != &all_delimiters[0] { + // Note: This intentionally deviate from the GNU error message by inserting the word "conflicting". + return Err(USimpleError::new( + 1, + "multiple conflicting output delimiters specified", + )); + } + } + let delim = match &*all_delimiters[0] { + "" => "\0", + delim => delim, + }; + + comm(&mut f1, &mut f2, delim, &matches) } pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) + .args_override_self(true) .arg( Arg::new(options::COLUMN_1) .short('1') @@ -188,6 +342,8 @@ pub fn uu_app() -> Command { .help("separate columns with STR") .value_name("STR") .default_value(options::DELIMITER_DEFAULT) + .allow_hyphen_values(true) + .action(ArgAction::Append) .hide_default_value(true), ) .arg( @@ -214,4 +370,17 @@ pub fn uu_app() -> Command { .help("output a summary") .action(ArgAction::SetTrue), ) + .arg( + Arg::new(options::CHECK_ORDER) + .long(options::CHECK_ORDER) + .help("check that the input is correctly sorted, even if all input lines are pairable") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::NO_CHECK_ORDER) + .long(options::NO_CHECK_ORDER) + .help("do not check that the input is correctly sorted") + .action(ArgAction::SetTrue) + .conflicts_with(options::CHECK_ORDER), + ) } diff --git a/src/uu/cp/Cargo.toml b/src/uu/cp/Cargo.toml index 177525ff5f2..fd5b4696e03 100644 --- a/src/uu/cp/Cargo.toml +++ b/src/uu/cp/Cargo.toml @@ -1,19 +1,18 @@ [package] name = "uu_cp" -version = "0.0.21" -authors = [ - "Jordy Dickinson ", - "Joshua S. Miller ", - "uutils developers", -] -license = "MIT" description = "cp ~ (uutils) copy SOURCE to DESTINATION" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/cp" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/cp.rs" @@ -22,12 +21,16 @@ path = "src/cp.rs" clap = { workspace = true } filetime = { workspace = true } libc = { workspace = true } +linux-raw-sys = { workspace = true } quick-error = { workspace = true } selinux = { workspace = true, optional = true } uucore = { workspace = true, features = [ "backup-control", + "buf-copy", "entries", "fs", + "fsxattr", + "parser", "perms", "mode", "update-control", @@ -44,5 +47,5 @@ name = "cp" path = "src/main.rs" [features] -feat_selinux = ["selinux"] +feat_selinux = ["selinux", "uucore/selinux"] feat_acl = ["exacl"] diff --git a/src/uu/cp/src/copydir.rs b/src/uu/cp/src/copydir.rs index 430548d0cf3..d2e367c5c19 100644 --- a/src/uu/cp/src/copydir.rs +++ b/src/uu/cp/src/copydir.rs @@ -2,13 +2,13 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore TODO canonicalizes direntry pathbuf symlinked +// spell-checker:ignore TODO canonicalizes direntry pathbuf symlinked IRWXO IRWXG //! Recursively copy the contents of a directory. //! //! See the [`copy_directory`] function for more information. #[cfg(windows)] use std::borrow::Cow; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::env; use std::fs; use std::io; @@ -17,15 +17,17 @@ use std::path::{Path, PathBuf, StripPrefixError}; use indicatif::ProgressBar; use uucore::display::Quotable; use uucore::error::UIoError; -use uucore::fs::{canonicalize, FileInformation, MissingHandling, ResolveMode}; +use uucore::fs::{ + FileInformation, MissingHandling, ResolveMode, canonicalize, path_ends_with_terminator, +}; use uucore::show; use uucore::show_error; use uucore::uio_error; use walkdir::{DirEntry, WalkDir}; use crate::{ - aligned_ancestors, context_for, copy_attributes, copy_file, copy_link, preserve_hardlinks, - CopyResult, Error, Options, + CopyResult, Error, Options, aligned_ancestors, context_for, copy_attributes, copy_file, + copy_link, }; /// Ensure a Windows path starts with a `\\?`. @@ -40,8 +42,9 @@ fn adjust_canonicalization(p: &Path) -> Cow { .components() .next() .and_then(|comp| comp.as_os_str().to_str()) - .map(|p_str| p_str.starts_with(VERBATIM_PREFIX) || p_str.starts_with(DEVICE_NS_PREFIX)) - .unwrap_or_default(); + .is_some_and(|p_str| { + p_str.starts_with(VERBATIM_PREFIX) || p_str.starts_with(DEVICE_NS_PREFIX) + }); if has_prefix { p.into() @@ -77,6 +80,12 @@ fn get_local_to_root_parent( } } +/// Given an iterator, return all its items except the last. +fn skip_last(mut iter: impl Iterator) -> impl Iterator { + let last = iter.next(); + iter.scan(last, |state, item| state.replace(item)) +} + /// Paths that are invariant throughout the traversal when copying a directory. struct Context<'a> { /// The current working directory at the time of starting the traversal. @@ -87,10 +96,13 @@ struct Context<'a> { /// The target path to which the directory will be copied. target: &'a Path, + + /// The source path from which the directory will be copied. + root: &'a Path, } impl<'a> Context<'a> { - fn new(root: &'a Path, target: &'a Path) -> std::io::Result { + fn new(root: &'a Path, target: &'a Path) -> io::Result { let current_dir = env::current_dir()?; let root_path = current_dir.join(root); let root_parent = if target.exists() && !root.to_str().unwrap().ends_with("/.") { @@ -102,6 +114,7 @@ impl<'a> Context<'a> { current_dir, root_parent, target, + root, }) } } @@ -156,11 +169,27 @@ struct Entry { } impl Entry { - fn new(context: &Context, direntry: &DirEntry) -> Result { - let source_relative = direntry.path().to_path_buf(); + fn new>( + context: &Context, + source: A, + no_target_dir: bool, + ) -> Result { + let source = source.as_ref(); + let source_relative = source.to_path_buf(); let source_absolute = context.current_dir.join(&source_relative); - let descendant = + let mut descendant = get_local_to_root_parent(&source_absolute, context.root_parent.as_deref())?; + if no_target_dir { + let source_is_dir = source.is_dir(); + if path_ends_with_terminator(context.target) && source_is_dir { + if let Err(e) = fs::create_dir_all(context.target) { + eprintln!("Failed to create directory: {e}"); + } + } else { + descendant = descendant.strip_prefix(context.root)?.to_path_buf(); + } + } + let local_to_target = context.target.join(descendant); let target_is_file = context.target.is_file(); Ok(Self { @@ -172,35 +201,16 @@ impl Entry { } } -/// Decide whether the given path ends with `/.`. -/// -/// # Examples -/// -/// ```rust,ignore -/// assert!(ends_with_slash_dot("/.")); -/// assert!(ends_with_slash_dot("./.")); -/// assert!(ends_with_slash_dot("a/.")); -/// -/// assert!(!ends_with_slash_dot(".")); -/// assert!(!ends_with_slash_dot("./")); -/// assert!(!ends_with_slash_dot("a/..")); -/// ``` -fn ends_with_slash_dot

(path: P) -> bool -where - P: AsRef, -{ - // `path.ends_with(".")` does not seem to work - path.as_ref().display().to_string().ends_with("/.") -} - +#[allow(clippy::too_many_arguments)] /// Copy a single entry during a directory traversal. fn copy_direntry( - progress_bar: &Option, + progress_bar: Option<&ProgressBar>, entry: Entry, options: &Options, symlinked_files: &mut HashSet, preserve_hard_links: bool, - hard_links: &mut Vec<(String, u64)>, + copied_destinations: &HashSet, + copied_files: &mut HashMap, ) -> CopyResult<()> { let Entry { source_absolute, @@ -217,79 +227,54 @@ fn copy_direntry( // If the source is a directory and the destination does not // exist, ... - if source_absolute.is_dir() - && !ends_with_slash_dot(&source_absolute) - && !local_to_target.exists() - { - if target_is_file { - return Err("cannot overwrite non-directory with directory".into()); + if source_absolute.is_dir() && !local_to_target.exists() { + return if target_is_file { + Err("cannot overwrite non-directory with directory".into()) } else { - // TODO Since the calling code is traversing from the root - // of the directory structure, I don't think - // `create_dir_all()` will have any benefit over - // `create_dir()`, since all the ancestor directories - // should have already been created. - fs::create_dir_all(&local_to_target)?; + build_dir(&local_to_target, false, options, Some(&source_absolute))?; if options.verbose { println!("{}", context_for(&source_relative, &local_to_target)); } - return Ok(()); - } + Ok(()) + }; } // If the source is not a directory, then we need to copy the file. if !source_absolute.is_dir() { - if preserve_hard_links { - let dest = local_to_target.as_path().to_path_buf(); - let found_hard_link = preserve_hardlinks(hard_links, &source_absolute, &dest)?; - if !found_hard_link { - match copy_file( - progress_bar, - &source_absolute, - local_to_target.as_path(), - options, - symlinked_files, - false, - ) { - Ok(_) => Ok(()), - Err(err) => { - if source_absolute.is_symlink() { - // silent the error with a symlink - // In case we do --archive, we might copy the symlink - // before the file itself - Ok(()) - } else { - Err(err) - } + if let Err(err) = copy_file( + progress_bar, + &source_absolute, + local_to_target.as_path(), + options, + symlinked_files, + copied_destinations, + copied_files, + false, + ) { + if preserve_hard_links { + if !source_absolute.is_symlink() { + return Err(err); + } + // silent the error with a symlink + // In case we do --archive, we might copy the symlink + // before the file itself + } else { + // At this point, `path` is just a plain old file. + // Terminate this function immediately if there is any + // kind of error *except* a "permission denied" error. + // + // TODO What other kinds of errors, if any, should + // cause us to continue walking the directory? + match err { + Error::IoErrContext(e, _) if e.kind() == io::ErrorKind::PermissionDenied => { + show!(uio_error!( + e, + "cannot open {} for reading", + source_relative.quote(), + )); } - }?; - } - } else { - // At this point, `path` is just a plain old file. - // Terminate this function immediately if there is any - // kind of error *except* a "permission denied" error. - // - // TODO What other kinds of errors, if any, should - // cause us to continue walking the directory? - match copy_file( - progress_bar, - &source_absolute, - local_to_target.as_path(), - options, - symlinked_files, - false, - ) { - Ok(_) => {} - Err(Error::IoErrContext(e, _)) - if e.kind() == std::io::ErrorKind::PermissionDenied => - { - show!(uio_error!( - e, - "cannot open {} for reading", - source_relative.quote(), - )); + e => return Err(e), } - Err(e) => return Err(e), } } } @@ -304,18 +289,17 @@ fn copy_direntry( /// /// Any errors encountered copying files in the tree will be logged but /// will not cause a short-circuit. +#[allow(clippy::too_many_arguments)] pub(crate) fn copy_directory( - progress_bar: &Option, + progress_bar: Option<&ProgressBar>, root: &Path, target: &Path, options: &Options, symlinked_files: &mut HashSet, + copied_destinations: &HashSet, + copied_files: &mut HashMap, source_in_command_line: bool, ) -> CopyResult<()> { - if !options.recursive { - return Err(format!("omitting directory {}", root.quote()).into()); - } - // if no-dereference is enabled and this is a symlink, copy it as a file if !options.dereference(source_in_command_line) && root.is_symlink() { return copy_file( @@ -324,10 +308,16 @@ pub(crate) fn copy_directory( target, options, symlinked_files, + copied_destinations, + copied_files, source_in_command_line, ); } + if !options.recursive { + return Err(format!("-r not specified; omitting directory {}", root.quote()).into()); + } + // check if root is a prefix of target if path_has_prefix(target, root)? { return Err(format!( @@ -349,8 +339,7 @@ pub(crate) fn copy_directory( let tmp = if options.parents { if let Some(parent) = root.parent() { let new_target = target.join(parent); - std::fs::create_dir_all(&new_target)?; - + build_dir(&new_target, true, options, None)?; if options.verbose { // For example, if copying file `a/b/c` and its parents // to directory `d/`, then print @@ -372,7 +361,6 @@ pub(crate) fn copy_directory( }; let target = tmp.as_path(); - let mut hard_links: Vec<(String, u64)> = vec![]; let preserve_hard_links = options.preserve_hard_links(); // Collect some paths here that are invariant during the traversal @@ -383,6 +371,9 @@ pub(crate) fn copy_directory( Err(e) => return Err(format!("failed to get current directory {e}").into()), }; + // The directory we were in during the previous iteration + let mut last_iter: Option = None; + // Traverse the contents of the directory, copying each one. for direntry_result in WalkDir::new(root) .same_file_system(options.one_file_system) @@ -390,30 +381,104 @@ pub(crate) fn copy_directory( { match direntry_result { Ok(direntry) => { - let entry = Entry::new(&context, &direntry)?; + let entry = Entry::new(&context, direntry.path(), options.no_target_dir)?; + copy_direntry( progress_bar, entry, options, symlinked_files, preserve_hard_links, - &mut hard_links, + copied_destinations, + copied_files, )?; + + // We omit certain permissions when creating directories + // to prevent other users from accessing them before they're done. + // We thus need to fix the permissions of each directory we copy + // once it's contents are ready. + // This "fixup" is implemented here in a memory-efficient manner. + // + // We detect iterations where we "walk up" the directory tree, + // and fix permissions on all the directories we exited. + // (Note that there can be more than one! We might step out of + // `./a/b/c` into `./a/`, in which case we'll need to fix the + // permissions of both `./a/b/c` and `./a/b`, in that order.) + if direntry.file_type().is_dir() { + // If true, last_iter is not a parent of this iter. + // The means we just exited a directory. + let went_up = if let Some(last_iter) = &last_iter { + last_iter.path().strip_prefix(direntry.path()).is_ok() + } else { + false + }; + + if went_up { + // Compute the "difference" between `last_iter` and `direntry`. + // For example, if... + // - last_iter = `a/b/c/d` + // - direntry = `a/b` + // then diff = `c/d` + // + // All the unwraps() here are unreachable. + let last_iter = last_iter.as_ref().unwrap(); + let diff = last_iter.path().strip_prefix(direntry.path()).unwrap(); + + // Fix permissions for every entry in `diff`, inside-out. + // We skip the last directory (which will be `.`) because + // its permissions will be fixed when we walk _out_ of it. + // (at this point, we might not be done copying `.`!) + for p in skip_last(diff.ancestors()) { + let src = direntry.path().join(p); + let entry = Entry::new(&context, &src, options.no_target_dir)?; + + copy_attributes( + &entry.source_absolute, + &entry.local_to_target, + &options.attributes, + )?; + } + } + + last_iter = Some(direntry); + } } + // Print an error message, but continue traversing the directory. - Err(e) => show_error!("{}", e), + Err(e) => show_error!("{e}"), } } - // Copy the attributes from the root directory to the target directory. + // Handle final directory permission fixes. + // This is almost the same as the permission-fixing code above, + // with minor differences (commented) + if let Some(last_iter) = last_iter { + let diff = last_iter.path().strip_prefix(root).unwrap(); + + // Do _not_ skip `.` this time, since we know we're done. + // This is where we fix the permissions of the top-level + // directory we just copied. + for p in diff.ancestors() { + let src = root.join(p); + let entry = Entry::new(&context, &src, options.no_target_dir)?; + + copy_attributes( + &entry.source_absolute, + &entry.local_to_target, + &options.attributes, + )?; + } + } + + // Also fix permissions for parent directories, + // if we were asked to create them. if options.parents { let dest = target.join(root.file_name().unwrap()); - copy_attributes(root, dest.as_path(), &options.attributes)?; for (x, y) in aligned_ancestors(root, dest.as_path()) { - copy_attributes(x, y, &options.attributes)?; + if let Ok(src) = canonicalize(x, MissingHandling::Normal, ResolveMode::Physical) { + copy_attributes(&src, y, &options.attributes)?; + } } - } else { - copy_attributes(root, target, &options.attributes)?; } Ok(()) @@ -443,24 +508,61 @@ pub fn path_has_prefix(p1: &Path, p2: &Path) -> io::Result { Ok(pathbuf1.starts_with(pathbuf2)) } -#[cfg(test)] -mod tests { - use super::ends_with_slash_dot; - - #[test] - fn test_ends_with_slash_dot() { - assert!(ends_with_slash_dot("/.")); - assert!(ends_with_slash_dot("./.")); - assert!(ends_with_slash_dot("../.")); - assert!(ends_with_slash_dot("a/.")); - assert!(ends_with_slash_dot("/a/.")); - - assert!(!ends_with_slash_dot("")); - assert!(!ends_with_slash_dot(".")); - assert!(!ends_with_slash_dot("./")); - assert!(!ends_with_slash_dot("..")); - assert!(!ends_with_slash_dot("/..")); - assert!(!ends_with_slash_dot("a/..")); - assert!(!ends_with_slash_dot("/a/..")); +/// Builds a directory at the specified path with the given options. +/// +/// # Notes +/// - If `copy_attributes_from` is `Some`, the new directory's attributes will be +/// copied from the provided file. Otherwise, the new directory will have the default +/// attributes for the current user. +/// - This method excludes certain permissions if ownership or special mode bits could +/// potentially change. (See `test_dir_perm_race_with_preserve_mode_and_ownership``) +/// - The `recursive` flag determines whether parent directories should be created +/// if they do not already exist. +// we need to allow unused_variable since `options` might be unused in non unix systems +#[allow(unused_variables)] +fn build_dir( + path: &PathBuf, + recursive: bool, + options: &Options, + copy_attributes_from: Option<&Path>, +) -> CopyResult<()> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(recursive); + + // To prevent unauthorized access before the folder is ready, + // exclude certain permissions if ownership or special mode bits + // could potentially change. + #[cfg(unix)] + { + use crate::Preserve; + use std::os::unix::fs::PermissionsExt; + + // we need to allow trivial casts here because some systems like linux have u32 constants in + // in libc while others don't. + #[allow(clippy::unnecessary_cast)] + let mut excluded_perms = if matches!(options.attributes.ownership, Preserve::Yes { .. }) { + libc::S_IRWXG | libc::S_IRWXO // exclude rwx for group and other + } else if matches!(options.attributes.mode, Preserve::Yes { .. }) { + libc::S_IWGRP | libc::S_IWOTH //exclude w for group and other + } else { + 0 + } as u32; + + let umask = if copy_attributes_from.is_some() + && matches!(options.attributes.mode, Preserve::Yes { .. }) + { + !fs::symlink_metadata(copy_attributes_from.unwrap())? + .permissions() + .mode() + } else { + uucore::mode::get_umask() + }; + + excluded_perms |= umask; + let mode = !excluded_perms & 0o777; //use only the last three octet bits + std::os::unix::fs::DirBuilderExt::mode(&mut builder, mode); } + + builder.create(path)?; + Ok(()) } diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index 7dbf8aedb50..06f0b79657d 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -2,47 +2,42 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) copydir ficlone fiemap ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs xattrs symlinked deduplicated advcpmv nushell -#![allow(clippy::missing_safety_doc)] -#![allow(clippy::extra_unused_lifetimes)] +// spell-checker:ignore (ToDO) copydir ficlone fiemap ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs xattrs symlinked deduplicated advcpmv nushell IRWXG IRWXO IRWXU IRWXUGO IRWXU IRWXG IRWXO IRWXUGO use quick_error::quick_error; -use std::borrow::Cow; use std::cmp::Ordering; -use std::collections::HashSet; -use std::env; -#[cfg(not(windows))] -use std::ffi::CString; -use std::fs::{self, File, OpenOptions}; +use std::collections::{HashMap, HashSet}; +use std::ffi::OsString; +use std::fs::{self, Metadata, OpenOptions, Permissions}; use std::io; #[cfg(unix)] -use std::os::unix::ffi::OsStrExt; -#[cfg(unix)] use std::os::unix::fs::{FileTypeExt, PermissionsExt}; use std::path::{Path, PathBuf, StripPrefixError}; -use std::string::ToString; +#[cfg(all(unix, not(target_os = "android")))] +use uucore::fsxattr::copy_xattrs; -use clap::{builder::ValueParser, crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command, builder::ValueParser, value_parser}; use filetime::FileTime; use indicatif::{ProgressBar, ProgressStyle}; -#[cfg(unix)] -use libc::mkfifo; use quick_error::ResultExt; use platform::copy_on_write; use uucore::display::Quotable; -use uucore::error::{set_exit_code, UClapError, UError, UResult, UUsageError}; +use uucore::error::{UClapError, UError, UResult, UUsageError, set_exit_code}; +#[cfg(unix)] +use uucore::fs::make_fifo; use uucore::fs::{ - canonicalize, is_symlink_loop, paths_refer_to_same_file, FileInformation, MissingHandling, - ResolveMode, + FileInformation, MissingHandling, ResolveMode, are_hardlinks_to_same_file, canonicalize, + get_filename, is_symlink_loop, normalize_path, path_ends_with_terminator, + paths_refer_to_same_file, }; use uucore::{backup_control, update_control}; // These are exposed for projects (e.g. nushell) that want to create an `Options` value, which // requires these enum. pub use uucore::{backup_control::BackupMode, update_control::UpdateMode}; use uucore::{ - crash, format_usage, help_about, help_section, help_usage, prompt_yes, show_error, - show_warning, util_name, + format_usage, help_about, help_section, help_usage, + parser::shortcut_value_parser::ShortcutValueParser, prompt_yes, show_error, show_warning, }; use crate::copydir::copy_directory; @@ -54,11 +49,11 @@ quick_error! { #[derive(Debug)] pub enum Error { /// Simple io::Error wrapper - IoErr(err: io::Error) { from() source(err) display("{}", err)} + IoErr(err: io::Error) { from() source(err) display("{err}")} /// Wrapper for io::Error with path context IoErrContext(err: io::Error, path: String) { - display("{}: {}", path, err) + display("{path}: {err}") context(path: &'a str, err: io::Error) -> (err, path.to_owned()) context(context: String, err: io::Error) -> (err, context) source(err) @@ -66,7 +61,7 @@ quick_error! { /// General copy error Error(err: String) { - display("{}", err) + display("{err}") from(err: String) -> (err) from(err: &'static str) -> (err.to_string()) } @@ -76,25 +71,27 @@ quick_error! { NotAllFilesCopied {} /// Simple walkdir::Error wrapper - WalkDirErr(err: walkdir::Error) { from() display("{}", err) source(err) } + WalkDirErr(err: walkdir::Error) { from() display("{err}") source(err) } /// Simple std::path::StripPrefixError wrapper StripPrefixError(err: StripPrefixError) { from() } /// Result of a skipped file - /// Currently happens when "no" is selected in interactive mode - Skipped { } + /// Currently happens when "no" is selected in interactive mode or when + /// `no-clobber` flag is set and destination is already present. + /// `exit with error` is used to determine which exit code should be returned. + Skipped(exit_with_error:bool) { } /// Result of a skipped file - InvalidArgument(description: String) { display("{}", description) } + InvalidArgument(description: String) { display("{description}") } /// All standard options are included as an an implementation /// path, but those that are not implemented yet should return /// a NotImplemented error. - NotImplemented(opt: String) { display("Option '{}' not yet implemented.", opt) } + NotImplemented(opt: String) { display("Option '{opt}' not yet implemented.") } /// Invalid arguments to backup - Backup(description: String) { display("{}\nTry '{} --help' for more information.", description, uucore::execution_phrase()) } + Backup(description: String) { display("{description}\nTry '{} --help' for more information.", uucore::execution_phrase()) } NotADirectory(path: PathBuf) { display("'{}' is not a directory", path.display()) } } @@ -109,15 +106,16 @@ impl UError for Error { pub type CopyResult = Result; /// Specifies how to overwrite files. -#[derive(Clone, Copy, Eq, PartialEq)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Default)] pub enum ClobberMode { Force, RemoveDestination, + #[default] Standard, } /// Specifies whether files should be overwritten. -#[derive(Clone, Copy, Eq, PartialEq)] +#[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum OverwriteMode { /// [Default] Always overwrite existing files Clobber(ClobberMode), @@ -127,32 +125,56 @@ pub enum OverwriteMode { NoClobber, } +impl Default for OverwriteMode { + fn default() -> Self { + Self::Clobber(ClobberMode::default()) + } +} + /// Possible arguments for `--reflink`. -#[derive(Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum ReflinkMode { Always, Auto, Never, } +impl Default for ReflinkMode { + #[allow(clippy::derivable_impls)] + fn default() -> Self { + #[cfg(any(target_os = "linux", target_os = "android", target_os = "macos"))] + { + ReflinkMode::Auto + } + #[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))] + { + ReflinkMode::Never + } + } +} + /// Possible arguments for `--sparse`. -#[derive(Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Default)] pub enum SparseMode { Always, + #[default] Auto, Never, } /// The expected file type of copy target +#[derive(Copy, Clone)] pub enum TargetType { Directory, File, } /// Copy action to perform +#[derive(Debug, Clone, Eq, PartialEq, Default)] pub enum CopyMode { Link, SymLink, + #[default] Copy, Update, AttrOnly, @@ -171,7 +193,7 @@ pub enum CopyMode { /// For full compatibility with GNU, these options should also combine. We /// currently only do a best effort imitation of that behavior, because it is /// difficult to achieve in clap, especially with `--no-preserve`. -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct Attributes { #[cfg(unix)] pub ownership: Preserve, @@ -182,9 +204,17 @@ pub struct Attributes { pub xattr: Preserve, } +impl Default for Attributes { + fn default() -> Self { + Self::NONE + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Preserve { - No, + // explicit means whether the --no-preserve flag is used or not to distinguish out the default value. + // e.g. --no-preserve=mode means mode = No { explicit = true } + No { explicit: bool }, Yes { required: bool }, } @@ -197,9 +227,9 @@ impl PartialOrd for Preserve { impl Ord for Preserve { fn cmp(&self, other: &Self) -> Ordering { match (self, other) { - (Self::No, Self::No) => Ordering::Equal, - (Self::Yes { .. }, Self::No) => Ordering::Greater, - (Self::No, Self::Yes { .. }) => Ordering::Less, + (Self::No { .. }, Self::No { .. }) => Ordering::Equal, + (Self::Yes { .. }, Self::No { .. }) => Ordering::Greater, + (Self::No { .. }, Self::Yes { .. }) => Ordering::Less, ( Self::Yes { required: req_self }, Self::Yes { @@ -219,6 +249,7 @@ impl Ord for Preserve { /// /// The fields are documented with the arguments that determine their value. #[allow(dead_code)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct Options { /// `--attributes-only` pub attributes_only: bool, @@ -280,6 +311,47 @@ pub struct Options { pub verbose: bool, /// `-g`, `--progress` pub progress_bar: bool, + /// -Z + pub set_selinux_context: bool, + // --context + pub context: Option, +} + +impl Default for Options { + fn default() -> Self { + Self { + attributes_only: false, + backup: BackupMode::default(), + copy_contents: false, + cli_dereference: false, + copy_mode: CopyMode::default(), + dereference: false, + no_target_dir: false, + one_file_system: false, + overwrite: OverwriteMode::default(), + parents: false, + sparse_mode: SparseMode::default(), + strip_trailing_slashes: false, + reflink_mode: ReflinkMode::default(), + attributes: Attributes::default(), + recursive: false, + backup_suffix: backup_control::DEFAULT_BACKUP_SUFFIX.to_owned(), + target_dir: None, + update: UpdateMode::default(), + debug: false, + verbose: false, + progress_bar: false, + set_selinux_context: false, + context: None, + } + } +} + +/// Enum representing if a file has been skipped. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PerformedAction { + Copied, + Skipped, } /// Enum representing various debug states of the offload and reflink actions. @@ -382,6 +454,7 @@ mod options { pub const RECURSIVE: &str = "recursive"; pub const REFLINK: &str = "reflink"; pub const REMOVE_DESTINATION: &str = "remove-destination"; + pub const SELINUX: &str = "Z"; pub const SPARSE: &str = "sparse"; pub const STRIP_TRAILING_SLASHES: &str = "strip-trailing-slashes"; pub const SYMBOLIC_LINK: &str = "symbolic-link"; @@ -396,22 +469,20 @@ static PRESERVABLE_ATTRIBUTES: &[&str] = &[ "ownership", "timestamps", "context", - "link", "links", "xattr", "all", ]; #[cfg(not(unix))] -static PRESERVABLE_ATTRIBUTES: &[&str] = &[ - "mode", - "timestamps", - "context", - "link", - "links", - "xattr", - "all", -]; +static PRESERVABLE_ATTRIBUTES: &[&str] = + &["mode", "timestamps", "context", "links", "xattr", "all"]; + +const PRESERVE_DEFAULT_VALUES: &str = if cfg!(unix) { + "mode,ownership,timestamp" +} else { + "mode,timestamp" +}; pub fn uu_app() -> Command { const MODE_ARGS: &[&str] = &[ @@ -422,7 +493,7 @@ pub fn uu_app() -> Command { options::COPY_CONTENTS, ]; Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .after_help(format!( @@ -475,8 +546,8 @@ pub fn uu_app() -> Command { ) .arg( Arg::new(options::RECURSIVE) - .short('r') - .visible_short_alias('R') + .short('R') + .visible_short_alias('r') .long(options::RECURSIVE) // --archive sets this option .help("copy directories recursively") @@ -543,7 +614,7 @@ pub fn uu_app() -> Command { .overrides_with_all(MODE_ARGS) .require_equals(true) .default_missing_value("always") - .value_parser(["auto", "always", "never"]) + .value_parser(ShortcutValueParser::new(["auto", "always", "never"])) .num_args(0..=1) .help("control clone/CoW copies. See below"), ) @@ -559,17 +630,11 @@ pub fn uu_app() -> Command { .long(options::PRESERVE) .action(ArgAction::Append) .use_value_delimiter(true) - .value_parser(clap::builder::PossibleValuesParser::new( - PRESERVABLE_ATTRIBUTES, - )) + .value_parser(ShortcutValueParser::new(PRESERVABLE_ATTRIBUTES)) .num_args(0..) .require_equals(true) .value_name("ATTR_LIST") - .overrides_with_all([ - options::ARCHIVE, - options::PRESERVE_DEFAULT_ATTRIBUTES, - options::NO_PRESERVE, - ]) + .default_missing_value(PRESERVE_DEFAULT_VALUES) // -d sets this option // --archive sets this option .help( @@ -581,19 +646,18 @@ pub fn uu_app() -> Command { Arg::new(options::PRESERVE_DEFAULT_ATTRIBUTES) .short('p') .long(options::PRESERVE_DEFAULT_ATTRIBUTES) - .overrides_with_all([options::PRESERVE, options::NO_PRESERVE, options::ARCHIVE]) .help("same as --preserve=mode,ownership(unix only),timestamps") .action(ArgAction::SetTrue), ) .arg( Arg::new(options::NO_PRESERVE) .long(options::NO_PRESERVE) + .action(ArgAction::Append) + .use_value_delimiter(true) + .value_parser(ShortcutValueParser::new(PRESERVABLE_ATTRIBUTES)) + .num_args(0..) + .require_equals(true) .value_name("ATTR_LIST") - .overrides_with_all([ - options::PRESERVE_DEFAULT_ATTRIBUTES, - options::PRESERVE, - options::ARCHIVE, - ]) .help("don't preserve the specified attributes"), ) .arg( @@ -630,11 +694,6 @@ pub fn uu_app() -> Command { Arg::new(options::ARCHIVE) .short('a') .long(options::ARCHIVE) - .overrides_with_all([ - options::PRESERVE_DEFAULT_ATTRIBUTES, - options::PRESERVE, - options::NO_PRESERVE, - ]) .help("Same as -dR --preserve=all") .action(ArgAction::SetTrue), ) @@ -655,44 +714,56 @@ pub fn uu_app() -> Command { Arg::new(options::SPARSE) .long(options::SPARSE) .value_name("WHEN") - .value_parser(["never", "auto", "always"]) + .value_parser(ShortcutValueParser::new(["never", "auto", "always"])) .help("control creation of sparse files. See below"), ) - // TODO: implement the following args .arg( - Arg::new(options::COPY_CONTENTS) - .long(options::COPY_CONTENTS) - .overrides_with(options::ATTRIBUTES_ONLY) - .help("NotImplemented: copy contents of special files when recursive") + Arg::new(options::SELINUX) + .short('Z') + .help("set SELinux security context of destination file to default type") .action(ArgAction::SetTrue), ) .arg( Arg::new(options::CONTEXT) .long(options::CONTEXT) .value_name("CTX") + .value_parser(value_parser!(String)) .help( - "NotImplemented: set SELinux security context of destination file to \ - default type", - ), + "like -Z, or if CTX is specified then set the SELinux or SMACK security \ + context to CTX", + ) + .num_args(0..=1) + .require_equals(true) + .default_missing_value(""), ) - // END TODO .arg( // The 'g' short flag is modeled after advcpmv // See this repo: https://github.com/jarun/advcpmv Arg::new(options::PROGRESS_BAR) .long(options::PROGRESS_BAR) .short('g') - .action(clap::ArgAction::SetTrue) + .action(ArgAction::SetTrue) .help( "Display a progress bar. \n\ Note: this feature is not supported by GNU coreutils.", ), ) + // TODO: implement the following args + .arg( + Arg::new(options::COPY_CONTENTS) + .long(options::COPY_CONTENTS) + .overrides_with(options::ATTRIBUTES_ONLY) + .help("NotImplemented: copy contents of special files when recursive") + .action(ArgAction::SetTrue), + ) + // END TODO .arg( Arg::new(options::PATHS) .action(ArgAction::Append) + .num_args(1..) + .required(true) .value_hint(clap::ValueHint::AnyPath) - .value_parser(ValueParser::path_buf()), + .value_parser(ValueParser::os_string()), ) } @@ -714,7 +785,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } else if let Ok(mut matches) = matches { let options = Options::from_matches(&matches)?; - if options.overwrite == OverwriteMode::NoClobber && options.backup != BackupMode::NoBackup { + if options.overwrite == OverwriteMode::NoClobber && options.backup != BackupMode::None { return Err(UUsageError::new( EXIT_ERR, "options --backup and --no-clobber are mutually exclusive", @@ -722,8 +793,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } let paths: Vec = matches - .remove_many::(options::PATHS) - .map(|v| v.collect()) + .remove_many::(options::PATHS) + .map(|v| v.map(PathBuf::from).collect()) .unwrap_or_default(); let (sources, target) = parse_path_args(paths, &options)?; @@ -734,7 +805,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { // code should still be EXIT_ERR as does GNU cp Error::NotAllFilesCopied => {} // Else we caught a fatal bubbled-up error, log it to stderr - _ => show_error!("{}", error), + _ => show_error!("{error}"), }; set_exit_code(EXIT_ERR); } @@ -780,7 +851,11 @@ impl CopyMode { { Self::Update } else if matches.get_flag(options::ATTRIBUTES_ONLY) { - Self::AttrOnly + if matches.get_flag(options::REMOVE_DESTINATION) { + Self::Copy + } else { + Self::AttrOnly + } } else { Self::Copy } @@ -800,7 +875,7 @@ impl Attributes { } #[cfg(not(feature = "feat_selinux"))] { - Preserve::No + Preserve::No { explicit: false } } }, links: Preserve::Yes { required: true }, @@ -809,12 +884,12 @@ impl Attributes { pub const NONE: Self = Self { #[cfg(unix)] - ownership: Preserve::No, - mode: Preserve::No, - timestamps: Preserve::No, - context: Preserve::No, - links: Preserve::No, - xattr: Preserve::No, + ownership: Preserve::No { explicit: false }, + mode: Preserve::No { explicit: false }, + timestamps: Preserve::No { explicit: false }, + context: Preserve::No { explicit: false }, + links: Preserve::No { explicit: false }, + xattr: Preserve::No { explicit: false }, }; // TODO: ownership is required if the user is root, for non-root users it's not required. @@ -823,6 +898,7 @@ impl Attributes { ownership: Preserve::Yes { required: true }, mode: Preserve::Yes { required: true }, timestamps: Preserve::Yes { required: true }, + xattr: Preserve::Yes { required: true }, ..Self::NONE }; @@ -843,6 +919,27 @@ impl Attributes { } } + /// Set the field to Preserve::NO { explicit: true } if the corresponding field + /// in other is set to Preserve::Yes { .. }. + pub fn diff(self, other: &Self) -> Self { + fn update_preserve_field(current: Preserve, other: Preserve) -> Preserve { + if matches!(other, Preserve::Yes { .. }) { + Preserve::No { explicit: true } + } else { + current + } + } + Self { + #[cfg(unix)] + ownership: update_preserve_field(self.ownership, other.ownership), + mode: update_preserve_field(self.mode, other.mode), + timestamps: update_preserve_field(self.timestamps, other.timestamps), + context: update_preserve_field(self.context, other.context), + links: update_preserve_field(self.links, other.links), + xattr: update_preserve_field(self.xattr, other.xattr), + } + } + pub fn parse_iter(values: impl Iterator) -> Result where T: AsRef, @@ -892,7 +989,6 @@ impl Options { let not_implemented_opts = vec![ #[cfg(not(any(windows, unix)))] options::ONE_FILE_SYSTEM, - options::CONTEXT, #[cfg(windows)] options::FORCE, ]; @@ -914,6 +1010,16 @@ impl Options { }; let update_mode = update_control::determine_update_mode(matches); + if backup_mode != BackupMode::None + && matches + .get_one::(update_control::arguments::OPT_UPDATE) + .is_some_and(|v| v == "none" || v == "none-fail") + { + return Err(Error::InvalidArgument( + "--backup is mutually exclusive with -n or --update=none-fail".to_string(), + )); + } + let backup_suffix = backup_control::determine_backup_suffix(matches); let overwrite = OverwriteMode::from_matches(matches); @@ -929,27 +1035,89 @@ impl Options { return Err(Error::NotADirectory(dir.clone())); } }; + // cp follows POSIX conventions for overriding options such as "-a", + // "-d", "--preserve", and "--no-preserve". We can use clap's + // override-all behavior to achieve this, but there's a challenge: when + // clap overrides an argument, it removes all traces of it from the + // match. This poses a problem because flags like "-a" expand to "-dR + // --preserve=all", and we only want to override the "--preserve=all" + // part. Additionally, we need to handle multiple occurrences of the + // same flags. To address this, we create an overriding order from the + // matches here. + let mut overriding_order: Vec<(usize, &str, Vec<&String>)> = vec![]; + // We iterate through each overriding option, adding each occurrence of + // the option along with its value and index as a tuple, and push it to + // `overriding_order`. + for option in [ + options::PRESERVE, + options::NO_PRESERVE, + options::ARCHIVE, + options::PRESERVE_DEFAULT_ATTRIBUTES, + options::NO_DEREFERENCE_PRESERVE_LINKS, + ] { + if let (Ok(Some(val)), Some(index)) = ( + matches.try_get_one::(option), + // even though it says in the doc that `index_of` would give us + // the first index of the argument, when it comes to flag it + // gives us the last index where the flag appeared (probably + // because it overrides itself). Since it is a flag and it would + // have same value across the occurrences we just need the last + // index. + matches.index_of(option), + ) { + if *val { + overriding_order.push((index, option, vec![])); + } + } else if let (Some(occurrences), Some(mut indices)) = ( + matches.get_occurrences::(option), + matches.indices_of(option), + ) { + occurrences.for_each(|val| { + if let Some(index) = indices.next() { + let val = val.collect::>(); + // As mentioned in the documentation of the indices_of + // function, it provides the indices of the individual + // values. Therefore, to get the index of the first + // value of the next occurrence in the next iteration, + // we need to advance the indices iterator by the length + // of the current occurrence's values. + for _ in 1..val.len() { + indices.next(); + } + overriding_order.push((index, option, val)); + } + }); + } + } + overriding_order.sort_by(|a, b| a.0.cmp(&b.0)); - // Parse attributes to preserve - let attributes = if let Some(attribute_strs) = matches.get_many::(options::PRESERVE) - { - if attribute_strs.len() == 0 { - Attributes::DEFAULT - } else { - Attributes::parse_iter(attribute_strs)? + let mut attributes = Attributes::NONE; + + // Iterate through the `overriding_order` and adjust the attributes accordingly. + for (_, option, val) in overriding_order { + match option { + options::ARCHIVE => { + attributes = Attributes::ALL; + } + options::PRESERVE_DEFAULT_ATTRIBUTES => { + attributes = attributes.union(&Attributes::DEFAULT); + } + options::NO_DEREFERENCE_PRESERVE_LINKS => { + attributes = attributes.union(&Attributes::LINKS); + } + options::PRESERVE => { + attributes = attributes.union(&Attributes::parse_iter(val.into_iter())?); + } + options::NO_PRESERVE => { + if !val.is_empty() { + attributes = attributes.diff(&Attributes::parse_iter(val.into_iter())?); + } + } + _ => (), } - } else if matches.get_flag(options::ARCHIVE) { - // --archive is used. Same as --preserve=all - Attributes::ALL - } else if matches.get_flag(options::NO_DEREFERENCE_PRESERVE_LINKS) { - Attributes::LINKS - } else if matches.get_flag(options::PRESERVE_DEFAULT_ATTRIBUTES) { - Attributes::DEFAULT - } else { - Attributes::NONE - }; + } - #[cfg(not(feature = "feat_selinux"))] + #[cfg(not(feature = "selinux"))] if let Preserve::Yes { required } = attributes.context { let selinux_disabled_error = Error::Error("SELinux was not enabled during the compile time!".to_string()); @@ -960,6 +1128,15 @@ impl Options { } } + // Extract the SELinux related flags and options + let set_selinux_context = matches.get_flag(options::SELINUX); + + let context = if matches.contains_id(options::CONTEXT) { + matches.get_one::(options::CONTEXT).cloned() + } else { + None + }; + let options = Self { attributes_only: matches.get_flag(options::ATTRIBUTES_ONLY), copy_contents: matches.get_flag(options::COPY_CONTENTS), @@ -969,7 +1146,9 @@ impl Options { dereference: !(matches.get_flag(options::NO_DEREFERENCE) || matches.get_flag(options::NO_DEREFERENCE_PRESERVE_LINKS) || matches.get_flag(options::ARCHIVE) - || recursive) + // cp normally follows the link only when not copying recursively or when + // --link (-l) is used + || (recursive && CopyMode::from_matches(matches)!= CopyMode::Link )) || matches.get_flag(options::DEREFERENCE), one_file_system: matches.get_flag(options::ONE_FILE_SYSTEM), parents: matches.get_flag(options::PARENTS), @@ -991,18 +1170,7 @@ impl Options { } } } else { - #[cfg(any(target_os = "linux", target_os = "android", target_os = "macos"))] - { - ReflinkMode::Auto - } - #[cfg(not(any( - target_os = "linux", - target_os = "android", - target_os = "macos" - )))] - { - ReflinkMode::Never - } + ReflinkMode::default() } }, sparse_mode: { @@ -1029,6 +1197,8 @@ impl Options { recursive, target_dir, progress_bar: matches.get_flag(options::PROGRESS_BAR), + set_selinux_context: set_selinux_context || context.is_some(), + context, }; Ok(options) @@ -1040,11 +1210,25 @@ impl Options { fn preserve_hard_links(&self) -> bool { match self.attributes.links { - Preserve::No => false, + Preserve::No { .. } => false, Preserve::Yes { .. } => true, } } + #[cfg(unix)] + fn preserve_mode(&self) -> (bool, bool) { + match self.attributes.mode { + Preserve::No { explicit } => { + if explicit { + (false, true) + } else { + (false, false) + } + } + Preserve::Yes { .. } => (true, false), + } + } + /// Whether to force overwriting the destination file. fn force(&self) -> bool { matches!(self.overwrite, OverwriteMode::Clobber(ClobberMode::Force)) @@ -1073,12 +1257,19 @@ fn parse_path_args( if paths.is_empty() { // No files specified return Err("missing file operand".into()); + } else if paths.len() == 1 && options.target_dir.is_none() { + // Only one file specified + return Err(format!( + "missing destination file operand after {}", + paths[0].display().to_string().quote() + ) + .into()); } // Return an error if the user requested to copy more than one // file source to a file target if options.no_target_dir && options.target_dir.is_none() && paths.len() > 2 { - return Err(format!("extra operand {:?}", paths[2]).into()); + return Err(format!("extra operand {:}", paths[2].display().to_string().quote()).into()); } let target = match options.target_dir { @@ -1095,6 +1286,9 @@ fn parse_path_args( }; if options.strip_trailing_slashes { + // clippy::assigning_clones added with Rust 1.78 + // Rust version = 1.76 on OpenBSD stable/7.5 + #[cfg_attr(not(target_os = "openbsd"), allow(clippy::assigning_clones))] for source in &mut paths { *source = source.components().as_path().to_owned(); } @@ -1103,67 +1297,6 @@ fn parse_path_args( Ok((paths, target)) } -/// Get the inode information for a file. -fn get_inode(file_info: &FileInformation) -> u64 { - #[cfg(unix)] - let result = file_info.inode(); - #[cfg(windows)] - let result = file_info.file_index(); - result -} - -#[cfg(target_os = "redox")] -fn preserve_hardlinks( - hard_links: &mut Vec<(String, u64)>, - source: &std::path::Path, - dest: &std::path::Path, - found_hard_link: &mut bool, -) -> CopyResult<()> { - // Redox does not currently support hard links - Ok(()) -} - -/// Hard link a pair of files if needed _and_ record if this pair is a new hard link. -#[cfg(not(target_os = "redox"))] -fn preserve_hardlinks( - hard_links: &mut Vec<(String, u64)>, - source: &std::path::Path, - dest: &std::path::Path, -) -> CopyResult { - let info = FileInformation::from_path(source, false) - .context(format!("cannot stat {}", source.quote()))?; - let inode = get_inode(&info); - let nlinks = info.number_of_links(); - let mut found_hard_link = false; - #[allow(clippy::explicit_iter_loop)] - for (link, link_inode) in hard_links.iter() { - if *link_inode == inode { - // Consider the following files: - // - // * `src/f` - a regular file - // * `src/link` - a hard link to `src/f` - // * `dest/src/f` - a different regular file - // - // In this scenario, if we do `cp -a src/ dest/`, it is - // possible that the order of traversal causes `src/link` - // to get copied first (to `dest/src/link`). In that case, - // in order to make sure `dest/src/link` is a hard link to - // `dest/src/f` and `dest/src/f` has the contents of - // `src/f`, we delete the existing file to allow the hard - // linking. - if file_or_link_exists(dest) && file_or_link_exists(Path::new(link)) { - std::fs::remove_file(dest)?; - } - std::fs::hard_link(link, dest).unwrap(); - found_hard_link = true; - } - } - if !found_hard_link && nlinks > 1 { - hard_links.push((dest.to_str().unwrap().to_string(), inode)); - } - Ok(found_hard_link) -} - /// When handling errors, we don't always want to show them to the user. This function handles that. fn show_error_if_needed(error: &Error) { match error { @@ -1172,12 +1305,12 @@ fn show_error_if_needed(error: &Error) { Error::NotAllFilesCopied => { // Need to return an error code } - Error::Skipped => { + Error::Skipped(_) => { // touch a b && echo "n"|cp -i a b && echo $? // should return an error from GNU 9.2 } _ => { - show_error!("{}", error); + show_error!("{error}"); } } } @@ -1192,14 +1325,22 @@ pub fn copy(sources: &[PathBuf], target: &Path, options: &Options) -> CopyResult let target_type = TargetType::determine(sources, target); verify_target_type(target, &target_type)?; - let preserve_hard_links = options.preserve_hard_links(); - - let mut hard_links: Vec<(String, u64)> = vec![]; - let mut non_fatal_errors = false; let mut seen_sources = HashSet::with_capacity(sources.len()); let mut symlinked_files = HashSet::new(); + // to remember the copied files for further usage. + // the FileInformation implemented the Hash trait by using + // 1. inode number + // 2. device number + // the combination of a file's inode number and device number is unique throughout all the file systems. + // + // key is the source file's information and the value is the destination filepath. + let mut copied_files: HashMap = HashMap::with_capacity(sources.len()); + // remember the copied destinations for further usage. + // we can't use copied_files as it is because the key is the source file's information. + let mut copied_destinations: HashSet = HashSet::with_capacity(sources.len()); + let progress_bar = if options.progress_bar { let pb = ProgressBar::new(disk_usage(sources, options.recursive)?) .with_style( @@ -1216,31 +1357,58 @@ pub fn copy(sources: &[PathBuf], target: &Path, options: &Options) -> CopyResult }; for source in sources { - if seen_sources.contains(source) { - // FIXME: compare sources by the actual file they point to, not their path. (e.g. dir/file == dir/../dir/file in most cases) - show_warning!("source {} specified more than once", source.quote()); - } else { - let found_hard_link = if preserve_hard_links && !source.is_dir() { - let dest = construct_dest_path(source, target, &target_type, options)?; - preserve_hardlinks(&mut hard_links, source, &dest)? + let normalized_source = normalize_path(source); + if options.backup == BackupMode::None && seen_sources.contains(&normalized_source) { + let file_type = if source.symlink_metadata()?.file_type().is_dir() { + "directory" } else { - false + "file" }; - if !found_hard_link { - if let Err(error) = copy_source( - &progress_bar, - source, - target, - &target_type, - options, - &mut symlinked_files, - ) { - show_error_if_needed(&error); + show_warning!( + "source {file_type} {} specified more than once", + source.quote() + ); + } else { + let dest = construct_dest_path(source, target, target_type, options) + .unwrap_or_else(|_| target.to_path_buf()); + + if fs::metadata(&dest).is_ok() + && !fs::symlink_metadata(&dest)?.file_type().is_symlink() + // if both `source` and `dest` are symlinks, it should be considered as an overwrite. + || fs::metadata(source).is_ok() + && fs::symlink_metadata(source)?.file_type().is_symlink() + || matches!(options.copy_mode, CopyMode::SymLink) + { + // There is already a file and it isn't a symlink (managed in a different place) + if copied_destinations.contains(&dest) && options.backup != BackupMode::Numbered { + // If the target file was already created in this cp call, do not overwrite + return Err(Error::Error(format!( + "will not overwrite just-created '{}' with '{}'", + dest.display(), + source.display() + ))); + } + } + + if let Err(error) = copy_source( + progress_bar.as_ref(), + source, + target, + target_type, + options, + &mut symlinked_files, + &copied_destinations, + &mut copied_files, + ) { + show_error_if_needed(&error); + if !matches!(error, Error::Skipped(false)) { non_fatal_errors = true; } + } else { + copied_destinations.insert(dest.clone()); } - seen_sources.insert(source); } + seen_sources.insert(normalized_source); } if let Some(pb) = progress_bar { @@ -1257,7 +1425,7 @@ pub fn copy(sources: &[PathBuf], target: &Path, options: &Options) -> CopyResult fn construct_dest_path( source_path: &Path, target: &Path, - target_type: &TargetType, + target_type: TargetType, options: &Options, ) -> CopyResult { if options.no_target_dir && target.is_dir() { @@ -1272,10 +1440,14 @@ fn construct_dest_path( return Err("with --parents, the destination must be a directory".into()); } - Ok(match *target_type { + Ok(match target_type { TargetType::Directory => { let root = if options.parents { - Path::new("") + if source_path.has_root() && cfg!(unix) { + Path::new("/") + } else { + Path::new("") + } } else { source_path.parent().unwrap_or(source_path) }; @@ -1284,19 +1456,30 @@ fn construct_dest_path( TargetType::File => target.to_path_buf(), }) } - +#[allow(clippy::too_many_arguments)] fn copy_source( - progress_bar: &Option, + progress_bar: Option<&ProgressBar>, source: &Path, target: &Path, - target_type: &TargetType, + target_type: TargetType, options: &Options, symlinked_files: &mut HashSet, + copied_destinations: &HashSet, + copied_files: &mut HashMap, ) -> CopyResult<()> { let source_path = Path::new(&source); if source_path.is_dir() { // Copy as directory - copy_directory(progress_bar, source, target, options, symlinked_files, true) + copy_directory( + progress_bar, + source, + target, + options, + symlinked_files, + copied_destinations, + copied_files, + true, + ) } else { // Copy as file let dest = construct_dest_path(source_path, target, target_type, options)?; @@ -1306,36 +1489,99 @@ fn copy_source( dest.as_path(), options, symlinked_files, + copied_destinations, + copied_files, true, ); if options.parents { for (x, y) in aligned_ancestors(source, dest.as_path()) { - copy_attributes(x, y, &options.attributes)?; + if let Ok(src) = canonicalize(x, MissingHandling::Normal, ResolveMode::Physical) { + copy_attributes(&src, y, &options.attributes)?; + } } } res } } +/// If `path` does not have `S_IWUSR` set, returns a tuple of the file's +/// mode in octal (index 0) and human-readable (index 1) formats. +/// +/// If the destination of a copy operation is a file that is not writeable to +/// the owner (bit `S_IWUSR`), extra information needs to be added to the +/// interactive mode prompt: the mode (permissions) of the file in octal and +/// human-readable format. +// TODO +// The destination metadata can be read multiple times in the course of a single execution of `cp`. +// This fix adds yet another metadata read. +// Should this metadata be read once and then reused throughout the execution? +// https://github.com/uutils/coreutils/issues/6658 +fn file_mode_for_interactive_overwrite( + #[cfg_attr(not(unix), allow(unused_variables))] path: &Path, +) -> Option<(String, String)> { + // Retain outer braces to ensure only one branch is included + { + #[cfg(unix)] + { + use libc::{S_IWUSR, mode_t}; + use std::os::unix::prelude::MetadataExt; + + match path.metadata() { + Ok(me) => { + // Cast is necessary on some platforms + let mode: mode_t = me.mode() as mode_t; + + // It looks like this extra information is added to the prompt iff the file's user write bit is 0 + // write permission, owner + if uucore::has!(mode, S_IWUSR) { + None + } else { + // Discard leading digits + let mode_without_leading_digits = mode & 0o7777; + + Some(( + format!("{mode_without_leading_digits:04o}"), + uucore::fs::display_permissions_unix(mode, false), + )) + } + } + // TODO: How should failure to read the metadata be handled? Ignoring for now. + Err(_) => None, + } + } + + #[cfg(not(unix))] + { + None + } + } +} + impl OverwriteMode { - fn verify(&self, path: &Path, verbose: bool) -> CopyResult<()> { + fn verify(&self, path: &Path, debug: bool) -> CopyResult<()> { match *self { Self::NoClobber => { - if verbose { + if debug { println!("skipped {}", path.quote()); - } else { - eprintln!("{}: not replacing {}", util_name(), path.quote()); } - Err(Error::NotAllFilesCopied) + Err(Error::Skipped(false)) } Self::Interactive(_) => { - if prompt_yes!("overwrite {}?", path.quote()) { + let prompt_yes_result = if let Some((octal, human_readable)) = + file_mode_for_interactive_overwrite(path) + { + prompt_yes!( + "replace {}, overriding mode {octal} ({human_readable})?", + path.quote() + ) + } else { + prompt_yes!("overwrite {}?", path.quote()) + }; + + if prompt_yes_result { Ok(()) } else { - if verbose { - println!("skipped {}", path.quote()); - } - Err(Error::Skipped) + Err(Error::Skipped(true)) } } Self::Clobber(_) => Ok(()), @@ -1348,7 +1594,7 @@ impl OverwriteMode { /// If it's required, then the error is thrown. fn handle_preserve CopyResult<()>>(p: &Preserve, f: F) -> CopyResult<()> { match p { - Preserve::No => {} + Preserve::No { .. } => {} Preserve::Yes { required } => { let result = f(); if *required { @@ -1361,6 +1607,42 @@ fn handle_preserve CopyResult<()>>(p: &Preserve, f: F) -> CopyResult< Ok(()) } +/// Copies extended attributes (xattrs) from `source` to `dest`, ensuring that `dest` is temporarily +/// user-writable if needed and restoring its original permissions afterward. This avoids “Operation +/// not permitted†errors on read-only files. Returns an error if permission or metadata operations fail, +/// or if xattr copying fails. +#[cfg(all(unix, not(target_os = "android")))] +fn copy_extended_attrs(source: &Path, dest: &Path) -> CopyResult<()> { + let metadata = fs::symlink_metadata(dest)?; + + // Check if the destination file is currently read-only for the user. + let mut perms = metadata.permissions(); + let was_readonly = perms.readonly(); + + // Temporarily grant user write if it was read-only. + if was_readonly { + #[allow(clippy::permissions_set_readonly_false)] + perms.set_readonly(false); + fs::set_permissions(dest, perms)?; + } + + // Perform the xattr copy and capture any potential error, + // so we can restore permissions before returning. + let copy_xattrs_result = copy_xattrs(source, dest); + + // Restore read-only if we changed it. + if was_readonly { + let mut revert_perms = fs::symlink_metadata(dest)?.permissions(); + revert_perms.set_readonly(true); + fs::set_permissions(dest, revert_perms)?; + } + + // If copying xattrs failed, propagate that error now. + copy_xattrs_result?; + + Ok(()) +} + /// Copy the specified attributes from one path to another. pub(crate) fn copy_attributes( source: &Path, @@ -1374,14 +1656,14 @@ pub(crate) fn copy_attributes( #[cfg(unix)] handle_preserve(&attributes.ownership, || -> CopyResult<()> { use std::os::unix::prelude::MetadataExt; - use uucore::perms::wrap_chown; use uucore::perms::Verbosity; use uucore::perms::VerbosityLevel; + use uucore::perms::wrap_chown; let dest_uid = source_metadata.uid(); let dest_gid = source_metadata.gid(); - - wrap_chown( + // gnu compatibility: cp doesn't report an error if it fails to set the ownership. + let _ = wrap_chown( dest, &dest.symlink_metadata().context(context)?, Some(dest_uid), @@ -1389,11 +1671,9 @@ pub(crate) fn copy_attributes( false, Verbosity { groups_only: false, - level: VerbosityLevel::Normal, + level: VerbosityLevel::Silent, }, - ) - .map_err(Error::Error)?; - + ); Ok(()) })?; @@ -1427,39 +1707,33 @@ pub(crate) fn copy_attributes( Ok(()) })?; - #[cfg(feature = "feat_selinux")] + #[cfg(feature = "selinux")] handle_preserve(&attributes.context, || -> CopyResult<()> { - let context = selinux::SecurityContext::of_path(source, false, false).map_err(|e| { - format!( - "failed to get security context of {}: {}", - source.display(), - e - ) - })?; - if let Some(context) = context { - context.set_for_path(dest, false, false).map_err(|e| { - format!( - "failed to set security context for {}: {}", - dest.display(), - e - ) - })?; + // Get the source context and apply it to the destination + if let Ok(context) = selinux::SecurityContext::of_path(source, false, false) { + if let Some(context) = context { + if let Err(e) = context.set_for_path(dest, false, false) { + return Err(Error::Error(format!( + "failed to set the security context of {}: {e}", + dest.display() + ))); + } + } + } else { + return Err(Error::Error(format!( + "failed to get security context of {}", + source.display() + ))); } - Ok(()) })?; handle_preserve(&attributes.xattr, || -> CopyResult<()> { - #[cfg(unix)] + #[cfg(all(unix, not(target_os = "android")))] { - let xattrs = xattr::list(source)?; - for attr in xattrs { - if let Some(attr_value) = xattr::get(source, attr.clone())? { - xattr::set(dest, attr, &attr_value[..])?; - } - } + copy_extended_attrs(source, dest)?; } - #[cfg(not(unix))] + #[cfg(not(all(unix, not(target_os = "android"))))] { // The documentation for GNU cp states: // @@ -1482,16 +1756,23 @@ pub(crate) fn copy_attributes( fn symlink_file( source: &Path, dest: &Path, - context: &str, symlinked_files: &mut HashSet, ) -> CopyResult<()> { #[cfg(not(windows))] { - std::os::unix::fs::symlink(source, dest).context(context)?; + std::os::unix::fs::symlink(source, dest).context(format!( + "cannot create symlink {} to {}", + get_filename(dest).unwrap_or("invalid file name").quote(), + get_filename(source).unwrap_or("invalid file name").quote() + ))?; } #[cfg(windows)] { - std::os::windows::fs::symlink_file(source, dest).context(context)?; + std::os::windows::fs::symlink_file(source, dest).context(format!( + "cannot create symlink {} to {}", + get_filename(dest).unwrap_or("invalid file name").quote(), + get_filename(source).unwrap_or("invalid file name").quote() + ))?; } if let Ok(file_info) = FileInformation::from_path(dest, false) { symlinked_files.insert(file_info); @@ -1503,10 +1784,15 @@ fn context_for(src: &Path, dest: &Path) -> String { format!("{} -> {}", src.quote(), dest.quote()) } -/// Implements a simple backup copy for the destination file. +/// Implements a simple backup copy for the destination file . +/// if is_dest_symlink flag is set to true dest will be renamed to backup_path /// TODO: for the backup, should this function be replaced by `copy_file(...)`? -fn backup_dest(dest: &Path, backup_path: &Path) -> CopyResult { - fs::copy(dest, backup_path)?; +fn backup_dest(dest: &Path, backup_path: &Path, is_dest_symlink: bool) -> CopyResult { + if is_dest_symlink { + fs::rename(dest, backup_path)?; + } else { + fs::copy(dest, backup_path)?; + } Ok(backup_path.into()) } @@ -1515,7 +1801,7 @@ fn backup_dest(dest: &Path, backup_path: &Path) -> CopyResult { /// /// Copying to the same file is only allowed if both `--backup` and /// `--force` are specified and the file is a regular file. -fn is_forbidden_copy_to_same_file( +fn is_forbidden_to_copy_to_same_file( source: &Path, dest: &Path, options: &Options, @@ -1523,10 +1809,44 @@ fn is_forbidden_copy_to_same_file( ) -> bool { // TODO To match the behavior of GNU cp, we also need to check // that the file is a regular file. + let source_is_symlink = source.is_symlink(); + let dest_is_symlink = dest.is_symlink(); + // only disable dereference if both source and dest is symlink and dereference flag is disabled let dereference_to_compare = - options.dereference(source_in_command_line) || !source.is_symlink(); - paths_refer_to_same_file(source, dest, dereference_to_compare) - && !(options.force() && options.backup != BackupMode::NoBackup) + options.dereference(source_in_command_line) || (!source_is_symlink || !dest_is_symlink); + if !paths_refer_to_same_file(source, dest, dereference_to_compare) { + return false; + } + if options.backup != BackupMode::None { + if options.force() && !source_is_symlink { + return false; + } + if source_is_symlink && !options.dereference { + return false; + } + if dest_is_symlink { + return false; + } + if !dest_is_symlink && !source_is_symlink && dest != source { + return false; + } + } + if options.copy_mode == CopyMode::Link { + return false; + } + if options.copy_mode == CopyMode::SymLink && dest_is_symlink { + return false; + } + // If source and dest are both the same symlink but with different names, then allow the copy. + // This can occur, for example, if source and dest are both hardlinks to the same symlink. + if dest_is_symlink + && source_is_symlink + && source.file_name() != dest.file_name() + && !options.dereference + { + return false; + } + true } /// Back up, remove, or leave intact the destination file, depending on the options. @@ -1535,15 +1855,26 @@ fn handle_existing_dest( dest: &Path, options: &Options, source_in_command_line: bool, + copied_files: &HashMap, ) -> CopyResult<()> { // Disallow copying a file to itself, unless `--force` and // `--backup` are both specified. - if is_forbidden_copy_to_same_file(source, dest, options, source_in_command_line) { + if is_forbidden_to_copy_to_same_file(source, dest, options, source_in_command_line) { return Err(format!("{} and {} are the same file", source.quote(), dest.quote()).into()); } - options.overwrite.verify(dest, options.verbose)?; + if options.update == UpdateMode::None { + if options.debug { + println!("skipped {}", dest.quote()); + } + return Err(Error::Skipped(false)); + } + + if options.update != UpdateMode::IfOlder { + options.overwrite.verify(dest, options.debug)?; + } + let mut is_dest_removed = false; let backup_path = backup_control::get_backup_path(options.backup, dest, &options.backup_suffix); if let Some(backup_path) = backup_path { if paths_refer_to_same_file(source, &backup_path, true) { @@ -1554,22 +1885,80 @@ fn handle_existing_dest( ) .into()); } else { - backup_dest(dest, &backup_path)?; + is_dest_removed = dest.is_symlink(); + backup_dest(dest, &backup_path, is_dest_removed)?; } } - match options.overwrite { - // FIXME: print that the file was removed if --verbose is enabled - OverwriteMode::Clobber(ClobberMode::Force) => { - if is_symlink_loop(dest) || fs::metadata(dest)?.permissions().readonly() { - fs::remove_file(dest)?; + if !is_dest_removed { + delete_dest_if_needed_and_allowed( + source, + dest, + options, + source_in_command_line, + copied_files, + )?; + } + + Ok(()) +} + +/// Checks if: +/// * `dest` needs to be deleted before the copy operation can proceed +/// * the provided options allow this deletion +/// +/// If so, deletes `dest`. +fn delete_dest_if_needed_and_allowed( + source: &Path, + dest: &Path, + options: &Options, + source_in_command_line: bool, + copied_files: &HashMap, +) -> CopyResult<()> { + let delete_dest = match options.overwrite { + OverwriteMode::Clobber(cl) | OverwriteMode::Interactive(cl) => { + match cl { + // FIXME: print that the file was removed if --verbose is enabled + ClobberMode::Force => { + // TODO + // Using `readonly` here to check if `dest` needs to be deleted is not correct: + // "On Unix-based platforms this checks if any of the owner, group or others write permission bits are set. It does not check if the current user is in the file's assigned group. It also does not check ACLs. Therefore the return value of this function cannot be relied upon to predict whether attempts to read or write the file will actually succeed." + // This results in some copy operations failing, because this necessary deletion is being skipped. + is_symlink_loop(dest) || fs::metadata(dest)?.permissions().readonly() + } + ClobberMode::RemoveDestination => true, + ClobberMode::Standard => { + // Consider the following files: + // + // * `src/f` - a regular file + // * `src/link` - a hard link to `src/f` + // * `dest/src/f` - a different regular file + // + // In this scenario, if we do `cp -a src/ dest/`, it is + // possible that the order of traversal causes `src/link` + // to get copied first (to `dest/src/link`). In that case, + // in order to make sure `dest/src/link` is a hard link to + // `dest/src/f` and `dest/src/f` has the contents of + // `src/f`, we delete the existing file to allow the hard + // linking. + options.preserve_hard_links() && + // only try to remove dest file only if the current source + // is hardlink to a file that is already copied + copied_files.contains_key( + &FileInformation::from_path( + source, + options.dereference(source_in_command_line) + ).context(format!("cannot stat {}", source.quote()))? + ) + } } } - OverwriteMode::Clobber(ClobberMode::RemoveDestination) => { - fs::remove_file(dest)?; - } - _ => (), + OverwriteMode::NoClobber => false, }; + if delete_dest { + fs::remove_file(dest)?; + } + Ok(()) } @@ -1609,7 +1998,7 @@ fn aligned_ancestors<'a>(source: &'a Path, dest: &'a Path) -> Vec<(&'a Path, &'a // Get the matching number of elements from the ancestors of the // destination path (for example, get "d/a" and "d/a/b"). let k = source_ancestors.len(); - let dest_ancestors = &dest_ancestors[1..1 + k]; + let dest_ancestors = &dest_ancestors[1..=k]; // Now we have two slices of the same length, so we zip them. let mut result = vec![]; @@ -1623,143 +2012,61 @@ fn aligned_ancestors<'a>(source: &'a Path, dest: &'a Path) -> Vec<(&'a Path, &'a result } -/// Copy the a file from `source` to `dest`. `source` will be dereferenced if -/// `options.dereference` is set to true. `dest` will be dereferenced only if -/// the source was not a symlink. -/// -/// Behavior when copying to existing files is contingent on the -/// `options.overwrite` mode. If a file is skipped, the return type -/// should be `Error:Skipped` -/// -/// The original permissions of `source` will be copied to `dest` -/// after a successful copy. -#[allow(clippy::cognitive_complexity)] -fn copy_file( - progress_bar: &Option, +fn print_verbose_output( + parents: bool, + progress_bar: Option<&ProgressBar>, source: &Path, dest: &Path, - options: &Options, - symlinked_files: &mut HashSet, - source_in_command_line: bool, -) -> CopyResult<()> { - if (options.update == UpdateMode::ReplaceIfOlder || options.update == UpdateMode::ReplaceNone) - && options.overwrite == OverwriteMode::Interactive(ClobberMode::Standard) - { - // `cp -i --update old new` when `new` exists doesn't copy anything - // and exit with 0 - return Ok(()); - } - - // Fail if dest is a dangling symlink or a symlink this program created previously - if dest.is_symlink() { - if FileInformation::from_path(dest, false) - .map(|info| symlinked_files.contains(&info)) - .unwrap_or(false) - { - return Err(Error::Error(format!( - "will not copy '{}' through just-created symlink '{}'", - source.display(), - dest.display() - ))); - } - let copy_contents = options.dereference(source_in_command_line) || !source.is_symlink(); - if copy_contents - && !dest.exists() - && !matches!( - options.overwrite, - OverwriteMode::Clobber(ClobberMode::RemoveDestination) - ) - && !is_symlink_loop(dest) - && std::env::var_os("POSIXLY_CORRECT").is_none() - { - return Err(Error::Error(format!( - "not writing through dangling symlink '{}'", - dest.display() - ))); - } - } - - if file_or_link_exists(dest) { - handle_existing_dest(source, dest, options, source_in_command_line)?; +) { + if let Some(pb) = progress_bar { + // Suspend (hide) the progress bar so the println won't overlap with the progress bar. + pb.suspend(|| { + print_paths(parents, source, dest); + }); + } else { + print_paths(parents, source, dest); } +} - if options.verbose { - if let Some(pb) = progress_bar { - // Suspend (hide) the progress bar so the println won't overlap with the progress bar. - pb.suspend(|| { - if options.parents { - // For example, if copying file `a/b/c` and its parents - // to directory `d/`, then print - // - // a -> d/a - // a/b -> d/a/b - // - for (x, y) in aligned_ancestors(source, dest) { - println!("{} -> {}", x.display(), y.display()); - } - } - - println!("{}", context_for(source, dest)); - }); - } else { - if options.parents { - // For example, if copying file `a/b/c` and its parents - // to directory `d/`, then print - // - // a -> d/a - // a/b -> d/a/b - // - for (x, y) in aligned_ancestors(source, dest) { - println!("{} -> {}", x.display(), y.display()); - } - } - - println!("{}", context_for(source, dest)); +fn print_paths(parents: bool, source: &Path, dest: &Path) { + if parents { + // For example, if copying file `a/b/c` and its parents + // to directory `d/`, then print + // + // a -> d/a + // a/b -> d/a/b + // + for (x, y) in aligned_ancestors(source, dest) { + println!("{} -> {}", x.display(), y.display()); } } - // Calculate the context upfront before canonicalizing the path - let context = context_for(source, dest); - let context = context.as_str(); - - let source_metadata = { - let result = if options.dereference(source_in_command_line) { - fs::metadata(source) - } else { - fs::symlink_metadata(source) - }; - result.context(context)? - }; - let source_file_type = source_metadata.file_type(); - let source_is_symlink = source_file_type.is_symlink(); - - #[cfg(unix)] - let source_is_fifo = source_file_type.is_fifo(); - #[cfg(not(unix))] - let source_is_fifo = false; - - let dest_permissions = if dest.exists() { - dest.symlink_metadata().context(context)?.permissions() - } else { - #[allow(unused_mut)] - let mut permissions = source_metadata.permissions(); - #[cfg(unix)] - { - use uucore::mode::get_umask; - - let mut mode = permissions.mode(); - - // remove sticky bit, suid and gid bit - const SPECIAL_PERMS_MASK: u32 = 0o7000; - mode &= !SPECIAL_PERMS_MASK; - - // apply umask - mode &= !get_umask(); + println!("{}", context_for(source, dest)); +} - permissions.set_mode(mode); - } - permissions - }; +/// Handles the copy mode for a file copy operation. +/// +/// This function determines how to copy a file based on the provided options. +/// It supports different copy modes, including hard linking, copying, symbolic linking, updating, and attribute-only copying. +/// It also handles file backups, overwriting, and dereferencing based on the provided options. +/// +/// # Returns +/// +/// * `Ok(())` - The file was copied successfully. +/// * `Err(CopyError)` - An error occurred while copying the file. +#[allow(clippy::too_many_arguments)] +fn handle_copy_mode( + source: &Path, + dest: &Path, + options: &Options, + context: &str, + source_metadata: &Metadata, + symlinked_files: &mut HashSet, + source_in_command_line: bool, + source_is_fifo: bool, + #[cfg(unix)] source_is_stream: bool, +) -> CopyResult { + let source_is_symlink = source_metadata.is_symlink(); match options.copy_mode { CopyMode::Link => { @@ -1767,7 +2074,7 @@ fn copy_file( let backup_path = backup_control::get_backup_path(options.backup, dest, &options.backup_suffix); if let Some(backup_path) = backup_path { - backup_dest(dest, &backup_path)?; + backup_dest(dest, &backup_path, dest.is_symlink())?; fs::remove_file(dest)?; } if options.overwrite == OverwriteMode::Clobber(ClobberMode::Force) { @@ -1781,7 +2088,11 @@ fn copy_file( } else { fs::hard_link(source, dest) } - .context(context)?; + .context(format!( + "cannot create hard link {} to {}", + get_filename(dest).unwrap_or("invalid file name").quote(), + get_filename(source).unwrap_or("invalid file name").quote() + ))?; } CopyMode::Copy => { copy_helper( @@ -1792,18 +2103,20 @@ fn copy_file( source_is_symlink, source_is_fifo, symlinked_files, + #[cfg(unix)] + source_is_stream, )?; } CopyMode::SymLink => { if dest.exists() && options.overwrite == OverwriteMode::Clobber(ClobberMode::Force) { fs::remove_file(dest)?; } - symlink_file(source, dest, context, symlinked_files)?; + symlink_file(source, dest, symlinked_files)?; } CopyMode::Update => { if dest.exists() { match options.update { - update_control::UpdateMode::ReplaceAll => { + UpdateMode::All => { copy_helper( source, dest, @@ -1812,17 +2125,30 @@ fn copy_file( source_is_symlink, source_is_fifo, symlinked_files, + #[cfg(unix)] + source_is_stream, )?; } - update_control::UpdateMode::ReplaceNone => return Ok(()), - update_control::UpdateMode::ReplaceIfOlder => { + UpdateMode::None => { + if options.debug { + println!("skipped {}", dest.quote()); + } + + return Ok(PerformedAction::Skipped); + } + UpdateMode::NoneFail => { + return Err(Error::Error(format!("not replacing '{}'", dest.display()))); + } + UpdateMode::IfOlder => { let dest_metadata = fs::symlink_metadata(dest)?; let src_time = source_metadata.modified()?; let dest_time = dest_metadata.modified()?; if src_time <= dest_time { - return Ok(()); + return Ok(PerformedAction::Skipped); } else { + options.overwrite.verify(dest, options.debug)?; + copy_helper( source, dest, @@ -1831,6 +2157,8 @@ fn copy_file( source_is_symlink, source_is_fifo, symlinked_files, + #[cfg(unix)] + source_is_stream, )?; } } @@ -1844,6 +2172,8 @@ fn copy_file( source_is_symlink, source_is_fifo, symlinked_files, + #[cfg(unix)] + source_is_stream, )?; } } @@ -1857,8 +2187,251 @@ fn copy_file( } }; + Ok(PerformedAction::Copied) +} + +/// Calculates the permissions for the destination file in a copy operation. +/// +/// If the destination file already exists, its current permissions are returned. +/// If the destination file does not exist, the source file's permissions are used, +/// with the `no-preserve` option and the umask taken into account on Unix platforms. +/// # Returns +/// +/// * `Ok(Permissions)` - The calculated permissions for the destination file. +/// * `Err(CopyError)` - An error occurred while getting the metadata of the destination file. +/// +// Allow unused variables for Windows (on options) +#[allow(unused_variables)] +fn calculate_dest_permissions( + dest: &Path, + source_metadata: &Metadata, + options: &Options, + context: &str, +) -> CopyResult { + if dest.exists() { + Ok(dest.symlink_metadata().context(context)?.permissions()) + } else { + #[cfg(unix)] + { + let mut permissions = source_metadata.permissions(); + let mode = handle_no_preserve_mode(options, permissions.mode()); + + // Apply umask + use uucore::mode::get_umask; + let mode = mode & !get_umask(); + permissions.set_mode(mode); + Ok(permissions) + } + #[cfg(not(unix))] + { + let permissions = source_metadata.permissions(); + Ok(permissions) + } + } +} + +/// Copy the a file from `source` to `dest`. `source` will be dereferenced if +/// `options.dereference` is set to true. `dest` will be dereferenced only if +/// the source was not a symlink. +/// +/// Behavior when copying to existing files is contingent on the +/// `options.overwrite` mode. If a file is skipped, the return type +/// should be `Error:Skipped` +/// +/// The original permissions of `source` will be copied to `dest` +/// after a successful copy. +#[allow(clippy::cognitive_complexity, clippy::too_many_arguments)] +fn copy_file( + progress_bar: Option<&ProgressBar>, + source: &Path, + dest: &Path, + options: &Options, + symlinked_files: &mut HashSet, + copied_destinations: &HashSet, + copied_files: &mut HashMap, + source_in_command_line: bool, +) -> CopyResult<()> { + let source_is_symlink = source.is_symlink(); + let dest_is_symlink = dest.is_symlink(); + // Fail if dest is a dangling symlink or a symlink this program created previously + if dest_is_symlink { + if FileInformation::from_path(dest, false) + .map(|info| symlinked_files.contains(&info)) + .unwrap_or(false) + { + return Err(Error::Error(format!( + "will not copy '{}' through just-created symlink '{}'", + source.display(), + dest.display() + ))); + } + // Fail if cp tries to copy two sources of the same name into a single symlink + // Example: "cp file1 dir1/file1 tmp" where "tmp" is a directory containing a symlink "file1" pointing to a file named "foo". + // foo will contain the contents of "file1" and "dir1/file1" will not be copied over to "tmp/file1" + if copied_destinations.contains(dest) { + return Err(Error::Error(format!( + "will not copy '{}' through just-created symlink '{}'", + source.display(), + dest.display() + ))); + } + + let copy_contents = options.dereference(source_in_command_line) || !source_is_symlink; + if copy_contents + && !dest.exists() + && !matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + ) + && !is_symlink_loop(dest) + && std::env::var_os("POSIXLY_CORRECT").is_none() + { + return Err(Error::Error(format!( + "not writing through dangling symlink '{}'", + dest.display() + ))); + } + if paths_refer_to_same_file(source, dest, true) + && matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + ) + && options.backup == BackupMode::None + { + fs::remove_file(dest)?; + } + } + + if are_hardlinks_to_same_file(source, dest) + && source != dest + && matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + ) + { + fs::remove_file(dest)?; + } + + if file_or_link_exists(dest) + && (!options.attributes_only + || matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + )) + { + if paths_refer_to_same_file(source, dest, true) && options.copy_mode == CopyMode::Link { + if source_is_symlink { + if !dest_is_symlink { + return Ok(()); + } + if !options.dereference { + return Ok(()); + } + } else if options.backup != BackupMode::None && !dest_is_symlink { + if source == dest { + if !options.force() { + return Ok(()); + } + } else { + return Ok(()); + } + } + } + handle_existing_dest(source, dest, options, source_in_command_line, copied_files)?; + if are_hardlinks_to_same_file(source, dest) { + if options.copy_mode == CopyMode::Copy { + return Ok(()); + } + if options.copy_mode == CopyMode::Link && (!source_is_symlink || !dest_is_symlink) { + return Ok(()); + } + } + } + + if options.attributes_only + && source_is_symlink + && !matches!( + options.overwrite, + OverwriteMode::Clobber(ClobberMode::RemoveDestination) + ) + { + return Err(format!( + "cannot change attribute {}: Source file is a non regular file", + dest.quote() + ) + .into()); + } + + if options.preserve_hard_links() { + // if we encounter a matching device/inode pair in the source tree + // we can arrange to create a hard link between the corresponding names + // in the destination tree. + if let Some(new_source) = copied_files.get( + &FileInformation::from_path(source, options.dereference(source_in_command_line)) + .context(format!("cannot stat {}", source.quote()))?, + ) { + fs::hard_link(new_source, dest)?; + + if options.verbose { + print_verbose_output(options.parents, progress_bar, source, dest); + } + + return Ok(()); + }; + } + + // Calculate the context upfront before canonicalizing the path + let context = context_for(source, dest); + let context = context.as_str(); + + let source_metadata = { + let result = if options.dereference(source_in_command_line) { + fs::metadata(source) + } else { + fs::symlink_metadata(source) + }; + // this is just for gnu tests compatibility + result.map_err(|err| { + if err.to_string().contains("No such file or directory") { + return format!("cannot stat {}: No such file or directory", source.quote()); + } + err.to_string() + })? + }; + + let dest_permissions = calculate_dest_permissions(dest, &source_metadata, options, context)?; + + #[cfg(unix)] + let source_is_fifo = source_metadata.file_type().is_fifo(); + #[cfg(not(unix))] + let source_is_fifo = false; + + #[cfg(unix)] + let source_is_stream = source_is_fifo + || source_metadata.file_type().is_char_device() + || source_metadata.file_type().is_block_device(); + #[cfg(not(unix))] + let source_is_stream = false; + + let performed_action = handle_copy_mode( + source, + dest, + options, + context, + &source_metadata, + symlinked_files, + source_in_command_line, + source_is_fifo, + #[cfg(unix)] + source_is_stream, + )?; + + if options.verbose && performed_action != PerformedAction::Skipped { + print_verbose_output(options.parents, progress_bar, source, dest); + } + // TODO: implement something similar to gnu's lchown - if !dest.is_symlink() { + if !dest_is_symlink { // Here, to match GNU semantics, we quietly ignore an error // if a user does not have the correct ownership to modify // the permissions of a file. @@ -1868,7 +2441,35 @@ fn copy_file( fs::set_permissions(dest, dest_permissions).ok(); } - copy_attributes(source, dest, &options.attributes)?; + if options.dereference(source_in_command_line) { + if let Ok(src) = canonicalize(source, MissingHandling::Normal, ResolveMode::Physical) { + if src.exists() { + copy_attributes(&src, dest, &options.attributes)?; + } + } + } else if source_is_stream && source.exists() { + // Some stream files may not exist after we have copied it, + // like anonymous pipes. Thus, we can't really copy its + // attributes. However, this is already handled in the stream + // copy function (see `copy_stream` under platform/linux.rs). + } else { + copy_attributes(source, dest, &options.attributes)?; + } + + #[cfg(feature = "selinux")] + if options.set_selinux_context && uucore::selinux::is_selinux_enabled() { + // Set the given selinux permissions on the copied file. + if let Err(e) = + uucore::selinux::set_selinux_security_context(dest, options.context.as_ref()) + { + return Err(Error::Error(format!("SELinux error: {}", e))); + } + } + + copied_files.insert( + FileInformation::from_path(source, options.dereference(source_in_command_line))?, + dest.to_path_buf(), + ); if let Some(progress_bar) = progress_bar { progress_bar.inc(fs::metadata(source)?.len()); @@ -1877,8 +2478,54 @@ fn copy_file( Ok(()) } +#[cfg(unix)] +fn handle_no_preserve_mode(options: &Options, org_mode: u32) -> u32 { + let (is_preserve_mode, is_explicit_no_preserve_mode) = options.preserve_mode(); + if !is_preserve_mode { + use libc::{ + S_IRGRP, S_IROTH, S_IRUSR, S_IRWXG, S_IRWXO, S_IRWXU, S_IWGRP, S_IWOTH, S_IWUSR, + }; + + #[cfg(not(any( + target_os = "android", + target_os = "macos", + target_os = "freebsd", + target_os = "redox", + )))] + { + const MODE_RW_UGO: u32 = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; + const S_IRWXUGO: u32 = S_IRWXU | S_IRWXG | S_IRWXO; + return if is_explicit_no_preserve_mode { + MODE_RW_UGO + } else { + org_mode & S_IRWXUGO + }; + } + + #[cfg(any( + target_os = "android", + target_os = "macos", + target_os = "freebsd", + target_os = "redox", + ))] + { + const MODE_RW_UGO: u32 = + (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH) as u32; + const S_IRWXUGO: u32 = (S_IRWXU | S_IRWXG | S_IRWXO) as u32; + if is_explicit_no_preserve_mode { + return MODE_RW_UGO; + } else { + return org_mode & S_IRWXUGO; + }; + } + } + + org_mode +} + /// Copy the file from `source` to `dest` either using the normal `fs::copy` or a /// copy-on-write scheme if --reflink is specified and the filesystem supports it. +#[allow(clippy::too_many_arguments)] fn copy_helper( source: &Path, dest: &Path, @@ -1887,20 +2534,20 @@ fn copy_helper( source_is_symlink: bool, source_is_fifo: bool, symlinked_files: &mut HashSet, + #[cfg(unix)] source_is_stream: bool, ) -> CopyResult<()> { if options.parents { let parent = dest.parent().unwrap_or(dest); fs::create_dir_all(parent)?; } - if source.as_os_str() == "/dev/null" { - /* workaround a limitation of fs::copy - * https://github.com/rust-lang/rust/issues/79390 - */ - File::create(dest).context(dest.display().to_string())?; - } else if source_is_fifo && options.recursive && !options.copy_contents { + if path_ends_with_terminator(dest) && !dest.is_dir() { + return Err(Error::NotADirectory(dest.to_path_buf())); + } + + if source_is_fifo && options.recursive && !options.copy_contents { #[cfg(unix)] - copy_fifo(dest, options.overwrite, options.verbose)?; + copy_fifo(dest, options.overwrite, options.debug)?; } else if source_is_symlink { copy_link(source, dest, symlinked_files)?; } else { @@ -1910,8 +2557,10 @@ fn copy_helper( options.reflink_mode, options.sparse_mode, context, - #[cfg(any(target_os = "linux", target_os = "android", target_os = "macos"))] + #[cfg(unix)] source_is_fifo, + #[cfg(unix)] + source_is_stream, )?; if !options.attributes_only && options.debug { @@ -1925,18 +2574,13 @@ fn copy_helper( // "Copies" a FIFO by creating a new one. This workaround is because Rust's // built-in fs::copy does not handle FIFOs (see rust-lang/rust/issues/79390). #[cfg(unix)] -fn copy_fifo(dest: &Path, overwrite: OverwriteMode, verbose: bool) -> CopyResult<()> { +fn copy_fifo(dest: &Path, overwrite: OverwriteMode, debug: bool) -> CopyResult<()> { if dest.exists() { - overwrite.verify(dest, verbose)?; + overwrite.verify(dest, debug)?; fs::remove_file(dest)?; } - let name = CString::new(dest.as_os_str().as_bytes()).unwrap(); - let err = unsafe { mkfifo(name.as_ptr(), 0o666) }; - if err == -1 { - return Err(format!("cannot create fifo {}: File exists", dest.quote()).into()); - } - Ok(()) + make_fifo(dest).map_err(|_| format!("cannot create fifo {}: File exists", dest.quote()).into()) } fn copy_link( @@ -1946,24 +2590,12 @@ fn copy_link( ) -> CopyResult<()> { // Here, we will copy the symlink itself (actually, just recreate it) let link = fs::read_link(source)?; - let dest: Cow<'_, Path> = if dest.is_dir() { - match source.file_name() { - Some(name) => dest.join(name).into(), - None => crash!( - EXIT_ERR, - "cannot stat {}: No such file or directory", - source.quote() - ), - } - } else { - // we always need to remove the file to be able to create a symlink, - // even if it is writeable. - if dest.is_symlink() || dest.is_file() { - fs::remove_file(dest)?; - } - dest.into() - }; - symlink_file(&link, &dest, &context_for(&link, &dest), symlinked_files) + // we always need to remove the file to be able to create a symlink, + // even if it is writeable. + if dest.is_symlink() || dest.is_file() { + fs::remove_file(dest)?; + } + symlink_file(&link, dest, symlinked_files) } /// Generate an error message if `target` is not the correct `target_type` @@ -2036,7 +2668,7 @@ fn disk_usage_directory(p: &Path) -> io::Result { #[cfg(test)] mod tests { - use crate::{aligned_ancestors, localize_to_target}; + use crate::{Attributes, Preserve, aligned_ancestors, localize_to_target}; use std::path::Path; #[test] @@ -2058,4 +2690,52 @@ mod tests { ]; assert_eq!(actual, expected); } + #[test] + fn test_diff_attrs() { + assert_eq!( + Attributes::ALL.diff(&Attributes { + context: Preserve::Yes { required: true }, + xattr: Preserve::Yes { required: true }, + ..Attributes::ALL + }), + Attributes { + #[cfg(unix)] + ownership: Preserve::No { explicit: true }, + mode: Preserve::No { explicit: true }, + timestamps: Preserve::No { explicit: true }, + context: Preserve::No { explicit: true }, + links: Preserve::No { explicit: true }, + xattr: Preserve::No { explicit: true } + } + ); + assert_eq!( + Attributes { + context: Preserve::Yes { required: true }, + xattr: Preserve::Yes { required: true }, + ..Attributes::ALL + } + .diff(&Attributes::NONE), + Attributes { + context: Preserve::Yes { required: true }, + xattr: Preserve::Yes { required: true }, + ..Attributes::ALL + } + ); + assert_eq!( + Attributes::NONE.diff(&Attributes { + context: Preserve::Yes { required: true }, + xattr: Preserve::Yes { required: true }, + ..Attributes::ALL + }), + Attributes { + #[cfg(unix)] + ownership: Preserve::No { explicit: true }, + mode: Preserve::No { explicit: true }, + timestamps: Preserve::No { explicit: true }, + context: Preserve::No { explicit: true }, + links: Preserve::No { explicit: true }, + xattr: Preserve::No { explicit: true } + } + ); + } } diff --git a/src/uu/cp/src/platform/linux.rs b/src/uu/cp/src/platform/linux.rs index 674e66ea575..9bf257f8276 100644 --- a/src/uu/cp/src/platform/linux.rs +++ b/src/uu/cp/src/platform/linux.rs @@ -2,12 +2,17 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore ficlone reflink ftruncate pwrite fiemap +// spell-checker:ignore ficlone reflink ftruncate pwrite fiemap lseek + +use libc::{SEEK_DATA, SEEK_HOLE}; use std::fs::{File, OpenOptions}; use std::io::Read; -use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::fs::FileExt; +use std::os::unix::fs::MetadataExt; +use std::os::unix::fs::{FileTypeExt, OpenOptionsExt}; use std::os::unix::io::AsRawFd; use std::path::Path; +use uucore::buf_copy; use quick_error::ResultExt; @@ -15,15 +20,6 @@ use uucore::mode::get_umask; use crate::{CopyDebug, CopyResult, OffloadReflinkDebug, ReflinkMode, SparseDebug, SparseMode}; -// From /usr/include/linux/fs.h: -// #define FICLONE _IOW(0x94, 9, int) -// Use a macro as libc::ioctl expects u32 or u64 depending on the arch -macro_rules! FICLONE { - () => { - 0x40049409 - }; -} - /// The fallback behavior for [`clone`] on failed system call. #[derive(Clone, Copy)] enum CloneFallback { @@ -32,6 +28,25 @@ enum CloneFallback { /// Use [`std::fs::copy`]. FSCopy, + + /// Use sparse_copy + SparseCopy, + + /// Use sparse_copy_without_hole + SparseCopyWithoutHole, +} + +/// Type of method used for copying files +#[derive(Clone, Copy)] +enum CopyMethod { + /// Do a sparse copy + SparseCopy, + /// Use [`std::fs::copy`]. + FSCopy, + /// Default (can either be sparse_copy or FSCopy) + Default, + /// Use sparse_copy_without_hole + SparseCopyWithoutHole, } /// Use the Linux `ioctl_ficlone` API to do a copy-on-write clone. @@ -46,24 +61,126 @@ where let dst_file = File::create(&dest)?; let src_fd = src_file.as_raw_fd(); let dst_fd = dst_file.as_raw_fd(); - let result = unsafe { libc::ioctl(dst_fd, FICLONE!(), src_fd) }; + // Using .try_into().unwrap() is required as glibc, musl & android all have different type for ioctl() + #[allow(clippy::unnecessary_fallible_conversions)] + let result = unsafe { + libc::ioctl( + dst_fd, + linux_raw_sys::ioctl::FICLONE.try_into().unwrap(), + src_fd, + ) + }; if result == 0 { return Ok(()); } match fallback { CloneFallback::Error => Err(std::io::Error::last_os_error()), CloneFallback::FSCopy => std::fs::copy(source, dest).map(|_| ()), + CloneFallback::SparseCopy => sparse_copy(source, dest), + CloneFallback::SparseCopyWithoutHole => sparse_copy_without_hole(source, dest), } } +/// Checks whether a file contains any non null bytes i.e. any byte != 0x0 +/// This function returns a tuple of (bool, u64, u64) signifying a tuple of (whether a file has +/// data, its size, no of blocks it has allocated in disk) +#[cfg(any(target_os = "linux", target_os = "android"))] +fn check_for_data(source: &Path) -> Result<(bool, u64, u64), std::io::Error> { + let mut src_file = File::open(source)?; + let metadata = src_file.metadata()?; + + let size = metadata.size(); + let blocks = metadata.blocks(); + // checks edge case of virtual files in /proc which have a size of zero but contains data + if size == 0 { + let mut buf: Vec = vec![0; metadata.blksize() as usize]; // Directly use metadata.blksize() + let _ = src_file.read(&mut buf)?; + return Ok((buf.iter().any(|&x| x != 0x0), size, 0)); + } + + let src_fd = src_file.as_raw_fd(); + + let result = unsafe { libc::lseek(src_fd, 0, SEEK_DATA) }; + + match result { + -1 => Ok((false, size, blocks)), // No data found or end of file + _ if result >= 0 => Ok((true, size, blocks)), // Data found + _ => Err(std::io::Error::last_os_error()), + } +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +/// Checks whether a file is sparse i.e. it contains holes, uses the crude heuristic blocks < size / 512 +/// Reference:`` +fn check_sparse_detection(source: &Path) -> Result { + let src_file = File::open(source)?; + let metadata = src_file.metadata()?; + let size = metadata.size(); + let blocks = metadata.blocks(); + + if blocks < size / 512 { + return Ok(true); + } + Ok(false) +} + +/// Optimized sparse_copy, doesn't create holes for large sequences of zeros in non sparse_files +/// Used when --sparse=auto +#[cfg(any(target_os = "linux", target_os = "android"))] +fn sparse_copy_without_hole

(source: P, dest: P) -> std::io::Result<()> +where + P: AsRef, +{ + let src_file = File::open(source)?; + let dst_file = File::create(dest)?; + let dst_fd = dst_file.as_raw_fd(); + + let size = src_file.metadata()?.size(); + if unsafe { libc::ftruncate(dst_fd, size.try_into().unwrap()) } < 0 { + return Err(std::io::Error::last_os_error()); + } + let src_fd = src_file.as_raw_fd(); + let mut current_offset: isize = 0; + // Maximize the data read at once to 16 MiB to avoid memory hogging with large files + // 16 MiB chunks should saturate an SSD + let step = std::cmp::min(size, 16 * 1024 * 1024) as usize; + let mut buf: Vec = vec![0x0; step]; + loop { + let result = unsafe { libc::lseek(src_fd, current_offset.try_into().unwrap(), SEEK_DATA) } + .try_into() + .unwrap(); + + current_offset = result; + let hole: isize = + unsafe { libc::lseek(src_fd, current_offset.try_into().unwrap(), SEEK_HOLE) } + .try_into() + .unwrap(); + if result == -1 || hole == -1 { + break; + } + if result <= -2 || hole <= -2 { + return Err(std::io::Error::last_os_error()); + } + let len: isize = hole - current_offset; + // Read and write data in chunks of `step` while reusing the same buffer + for i in (0..len).step_by(step) { + // Ensure we don't read past the end of the file or the start of the next hole + let read_len = std::cmp::min((len - i) as usize, step); + let buf = &mut buf[..read_len]; + src_file.read_exact_at(buf, (current_offset + i) as u64)?; + dst_file.write_all_at(buf, (current_offset + i) as u64)?; + } + current_offset = hole; + } + Ok(()) +} /// Perform a sparse copy from one file to another. +/// Creates a holes for large sequences of zeros in non_sparse_files, used for --sparse=always #[cfg(any(target_os = "linux", target_os = "android"))] fn sparse_copy

(source: P, dest: P) -> std::io::Result<()> where P: AsRef, { - use std::os::unix::prelude::MetadataExt; - let mut src_file = File::open(source)?; let dst_file = File::create(dest)?; let dst_fd = dst_file.as_raw_fd(); @@ -82,23 +199,30 @@ where // https://www.kernel.org/doc/html/latest/filesystems/fiemap.html while current_offset < size { let this_read = src_file.read(&mut buf)?; + let buf = &buf[..this_read]; if buf.iter().any(|&x| x != 0) { - unsafe { - libc::pwrite( - dst_fd, - buf.as_ptr() as *const libc::c_void, - this_read, - current_offset.try_into().unwrap(), - ) - }; + dst_file.write_all_at(buf, current_offset.try_into().unwrap())?; } current_offset += this_read; } Ok(()) } -/// Copy the contents of the given source FIFO to the given file. -fn copy_fifo_contents

(source: P, dest: P) -> std::io::Result +#[cfg(any(target_os = "linux", target_os = "android"))] +/// Checks whether an existing destination is a fifo +fn check_dest_is_fifo(dest: &Path) -> bool { + // If our destination file exists and its a fifo , we do a standard copy . + let file_type = std::fs::metadata(dest); + match file_type { + Ok(f) => f.file_type().is_fifo(), + + _ => false, + } +} + +/// Copy the contents of a stream from `source` to `dest`. The `if_fifo` argument is used to +/// determine if we need to modify the file's attributes before and after copying. +fn copy_stream

(source: P, dest: P, is_fifo: bool) -> std::io::Result where P: AsRef, { @@ -127,8 +251,14 @@ where .write(true) .mode(mode) .open(&dest)?; - let num_bytes_copied = std::io::copy(&mut src_file, &mut dst_file)?; - dst_file.set_permissions(src_file.metadata()?.permissions())?; + + let num_bytes_copied = buf_copy::copy_stream(&mut src_file, &mut dst_file) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::Other))?; + + if is_fifo { + dst_file.set_permissions(src_file.metadata()?.permissions())?; + } + Ok(num_bytes_copied) } @@ -145,41 +275,125 @@ pub(crate) fn copy_on_write( sparse_mode: SparseMode, context: &str, source_is_fifo: bool, + source_is_stream: bool, ) -> CopyResult { let mut copy_debug = CopyDebug { offload: OffloadReflinkDebug::Unknown, reflink: OffloadReflinkDebug::Unsupported, sparse_detection: SparseDebug::No, }; - let result = match (reflink_mode, sparse_mode) { (ReflinkMode::Never, SparseMode::Always) => { copy_debug.sparse_detection = SparseDebug::Zeros; - copy_debug.offload = OffloadReflinkDebug::Avoided; + // Default SparseDebug val for SparseMode::Always copy_debug.reflink = OffloadReflinkDebug::No; - sparse_copy(source, dest) + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_never_sparse_always(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; + } + + match copy_method { + CopyMethod::FSCopy => std::fs::copy(source, dest).map(|_| ()), + _ => sparse_copy(source, dest), + } + } } - (ReflinkMode::Never, _) => { - copy_debug.sparse_detection = SparseDebug::No; + (ReflinkMode::Never, SparseMode::Never) => { copy_debug.reflink = OffloadReflinkDebug::No; - std::fs::copy(source, dest).map(|_| ()) + + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let result = handle_reflink_never_sparse_never(source); + if let Ok(debug) = result { + copy_debug = debug; + } + std::fs::copy(source, dest).map(|_| ()) + } + } + (ReflinkMode::Never, SparseMode::Auto) => { + copy_debug.reflink = OffloadReflinkDebug::No; + + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_never_sparse_auto(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; + } + + match copy_method { + CopyMethod::SparseCopyWithoutHole => sparse_copy_without_hole(source, dest), + _ => std::fs::copy(source, dest).map(|_| ()), + } + } } (ReflinkMode::Auto, SparseMode::Always) => { - copy_debug.offload = OffloadReflinkDebug::Avoided; - copy_debug.sparse_detection = SparseDebug::Zeros; - copy_debug.reflink = OffloadReflinkDebug::Unsupported; - sparse_copy(source, dest) + copy_debug.sparse_detection = SparseDebug::Zeros; // Default SparseDebug val for + // SparseMode::Always + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_auto_sparse_always(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; + } + + match copy_method { + CopyMethod::FSCopy => clone(source, dest, CloneFallback::FSCopy), + _ => clone(source, dest, CloneFallback::SparseCopy), + } + } } - (ReflinkMode::Auto, _) => { - copy_debug.sparse_detection = SparseDebug::No; - copy_debug.reflink = OffloadReflinkDebug::Unsupported; - if source_is_fifo { - copy_fifo_contents(source, dest).map(|_| ()) + (ReflinkMode::Auto, SparseMode::Never) => { + copy_debug.reflink = OffloadReflinkDebug::No; + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Avoided; + copy_stream(source, dest, source_is_fifo).map(|_| ()) } else { + let result = handle_reflink_auto_sparse_never(source); + if let Ok(debug) = result { + copy_debug = debug; + } + clone(source, dest, CloneFallback::FSCopy) } } + (ReflinkMode::Auto, SparseMode::Auto) => { + if source_is_stream { + copy_debug.offload = OffloadReflinkDebug::Unsupported; + copy_stream(source, dest, source_is_fifo).map(|_| ()) + } else { + let mut copy_method = CopyMethod::Default; + let result = handle_reflink_auto_sparse_auto(source, dest); + if let Ok((debug, method)) = result { + copy_debug = debug; + copy_method = method; + } + + match copy_method { + CopyMethod::SparseCopyWithoutHole => { + clone(source, dest, CloneFallback::SparseCopyWithoutHole) + } + _ => clone(source, dest, CloneFallback::FSCopy), + } + } + } + (ReflinkMode::Always, SparseMode::Auto) => { copy_debug.sparse_detection = SparseDebug::No; copy_debug.reflink = OffloadReflinkDebug::Yes; @@ -187,9 +401,217 @@ pub(crate) fn copy_on_write( clone(source, dest, CloneFallback::Error) } (ReflinkMode::Always, _) => { - return Err("`--reflink=always` can be used only with --sparse=auto".into()) + return Err("`--reflink=always` can be used only with --sparse=auto".into()); } }; result.context(context)?; Ok(copy_debug) } + +/// Handles debug results when flags are "--reflink=auto" and "--sparse=always" and specifies what +/// type of copy should be used +fn handle_reflink_auto_sparse_always( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::Unsupported, + sparse_detection: SparseDebug::Zeros, + }; + let mut copy_method = CopyMethod::Default; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + match (sparse_flag, data_flag, blocks) { + (true, true, 0) => { + // Handling funny files with 0 block allocation but has data + // in it + copy_method = CopyMethod::FSCopy; + copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; + } + (false, true, 0) => copy_method = CopyMethod::FSCopy, + + (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, + (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, + + (true, false, _) => copy_debug.sparse_detection = SparseDebug::SeekHole, + + (_, _, _) => (), + } + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + Ok((copy_debug, copy_method)) +} + +/// Handles debug results when flags are "--reflink=auto" and "--sparse=auto" and specifies what +/// type of copy should be used +fn handle_reflink_never_sparse_never(source: &Path) -> Result { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::No, + }; + let (data_flag, size, _blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if sparse_flag { + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + Ok(copy_debug) +} + +/// Handles debug results when flags are "--reflink=auto" and "--sparse=never", files will be copied +/// through cloning them with fallback switching to std::fs::copy +fn handle_reflink_auto_sparse_never(source: &Path) -> Result { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::No, + }; + + let (data_flag, size, _blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if sparse_flag { + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + Ok(copy_debug) +} + +/// Handles debug results when flags are "--reflink=auto" and "--sparse=auto" and specifies what +/// type of copy should be used +fn handle_reflink_auto_sparse_auto( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::Unsupported, + sparse_detection: SparseDebug::No, + }; + + let mut copy_method = CopyMethod::Default; + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if (data_flag && size != 0) || (size > 0 && size < 512) { + copy_debug.offload = OffloadReflinkDebug::Yes; + } + + if data_flag && size == 0 { + // Handling /proc/ files + copy_debug.offload = OffloadReflinkDebug::Unsupported; + } + if sparse_flag { + if blocks == 0 && data_flag { + // Handling other "virtual" files + copy_debug.offload = OffloadReflinkDebug::Unsupported; + + copy_method = CopyMethod::FSCopy; // Doing a standard copy for the virtual files + } else { + copy_method = CopyMethod::SparseCopyWithoutHole; + } // Since sparse_flag is true, sparse_detection shall be SeekHole for any non virtual + // regular sparse file and the file will be sparsely copied + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + Ok((copy_debug, copy_method)) +} + +/// Handles debug results when flags are "--reflink=never" and "--sparse=auto" and specifies what +/// type of copy should be used +fn handle_reflink_never_sparse_auto( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::No, + }; + + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + let mut copy_method = CopyMethod::Default; + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + + if sparse_flag { + if blocks == 0 && data_flag { + copy_method = CopyMethod::FSCopy; // Handles virtual files which have size > 0 but no + // disk allocation + } else { + copy_method = CopyMethod::SparseCopyWithoutHole; // Handles regular sparse-files + } + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + Ok((copy_debug, copy_method)) +} + +/// Handles debug results when flags are "--reflink=never" and "--sparse=always" and specifies what +/// type of copy should be used +fn handle_reflink_never_sparse_always( + source: &Path, + dest: &Path, +) -> Result<(CopyDebug, CopyMethod), std::io::Error> { + let mut copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unknown, + reflink: OffloadReflinkDebug::No, + sparse_detection: SparseDebug::Zeros, + }; + let mut copy_method = CopyMethod::SparseCopy; + + let (data_flag, size, blocks) = check_for_data(source)?; + let sparse_flag = check_sparse_detection(source)?; + + if data_flag || size < 512 { + copy_debug.offload = OffloadReflinkDebug::Avoided; + } + match (sparse_flag, data_flag, blocks) { + (true, true, 0) => { + // Handling funny files with 0 block allocation but has data + // in it, e.g. files in /sys and other virtual files + copy_method = CopyMethod::FSCopy; + copy_debug.sparse_detection = SparseDebug::SeekHoleZeros; + } + (false, true, 0) => copy_method = CopyMethod::FSCopy, // Handling data containing zero sized + // files in /proc + (true, false, 0) => copy_debug.sparse_detection = SparseDebug::SeekHole, // Handles files + // with 0 blocks allocated in disk and + (true, true, _) => copy_debug.sparse_detection = SparseDebug::SeekHoleZeros, // Any + // sparse_files with data in it will display SeekHoleZeros + (true, false, _) => { + copy_debug.offload = OffloadReflinkDebug::Unknown; + copy_debug.sparse_detection = SparseDebug::SeekHole; + } + + (_, _, _) => (), + } + if check_dest_is_fifo(dest) { + copy_method = CopyMethod::FSCopy; + } + + Ok((copy_debug, copy_method)) +} diff --git a/src/uu/cp/src/platform/macos.rs b/src/uu/cp/src/platform/macos.rs index 8c62c78d95d..35879c29df7 100644 --- a/src/uu/cp/src/platform/macos.rs +++ b/src/uu/cp/src/platform/macos.rs @@ -4,12 +4,14 @@ // file that was distributed with this source code. // spell-checker:ignore reflink use std::ffi::CString; -use std::fs::{self, File}; -use std::io; +use std::fs::{self, File, OpenOptions}; use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use quick_error::ResultExt; +use uucore::buf_copy; +use uucore::mode::get_umask; use crate::{CopyDebug, CopyResult, OffloadReflinkDebug, ReflinkMode, SparseDebug, SparseMode}; @@ -24,6 +26,7 @@ pub(crate) fn copy_on_write( sparse_mode: SparseMode, context: &str, source_is_fifo: bool, + source_is_stream: bool, ) -> CopyResult { if sparse_mode != SparseMode::Auto { return Err("--sparse is only supported on linux".to_string().into()); @@ -63,8 +66,15 @@ pub(crate) fn copy_on_write( { // clonefile(2) fails if the destination exists. Remove it and try again. Do not // bother to check if removal worked because we're going to try to clone again. - let _ = fs::remove_file(dest); - error = pfn(src.as_ptr(), dst.as_ptr(), 0); + // first lets make sure the dest file is not read only + if fs::metadata(dest).is_ok_and(|md| !md.permissions().readonly()) { + // remove and copy again + // TODO: rewrite this to better match linux behavior + // linux first opens the source file and destination file then uses the file + // descriptors to do the clone. + let _ = fs::remove_file(dest); + error = pfn(src.as_ptr(), dst.as_ptr(), 0); + } } } } @@ -74,14 +84,32 @@ pub(crate) fn copy_on_write( // support COW). match reflink_mode { ReflinkMode::Always => { - return Err(format!("failed to clone {source:?} from {dest:?}: {error}").into()) + return Err(format!( + "failed to clone {} from {}: {error}", + source.display(), + dest.display() + ) + .into()); } _ => { copy_debug.reflink = OffloadReflinkDebug::Yes; - if source_is_fifo { + if source_is_stream { let mut src_file = File::open(source)?; - let mut dst_file = File::create(dest)?; - io::copy(&mut src_file, &mut dst_file).context(context)? + let mode = 0o622 & !get_umask(); + let mut dst_file = OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest)?; + + let context = buf_copy::copy_stream(&mut src_file, &mut dst_file) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::Other)) + .context(context)?; + + if source_is_fifo { + dst_file.set_permissions(src_file.metadata()?.permissions())?; + } + context } else { fs::copy(source, dest).context(context)? } diff --git a/src/uu/cp/src/platform/mod.rs b/src/uu/cp/src/platform/mod.rs index c7942706868..2071e928f41 100644 --- a/src/uu/cp/src/platform/mod.rs +++ b/src/uu/cp/src/platform/mod.rs @@ -2,6 +2,18 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. + +#[cfg(all( + unix, + not(any(target_os = "macos", target_os = "linux", target_os = "android")) +))] +mod other_unix; +#[cfg(all( + unix, + not(any(target_os = "macos", target_os = "linux", target_os = "android")) +))] +pub(crate) use self::other_unix::copy_on_write; + #[cfg(target_os = "macos")] mod macos; #[cfg(target_os = "macos")] @@ -12,7 +24,13 @@ mod linux; #[cfg(any(target_os = "linux", target_os = "android"))] pub(crate) use self::linux::copy_on_write; -#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))] +#[cfg(not(any( + unix, + any(target_os = "macos", target_os = "linux", target_os = "android") +)))] mod other; -#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))] +#[cfg(not(any( + unix, + any(target_os = "macos", target_os = "linux", target_os = "android") +)))] pub(crate) use self::other::copy_on_write; diff --git a/src/uu/cp/src/platform/other_unix.rs b/src/uu/cp/src/platform/other_unix.rs new file mode 100644 index 00000000000..aa8fed3fab1 --- /dev/null +++ b/src/uu/cp/src/platform/other_unix.rs @@ -0,0 +1,62 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore reflink +use std::fs::{self, File, OpenOptions}; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; + +use quick_error::ResultExt; +use uucore::buf_copy; +use uucore::mode::get_umask; + +use crate::{CopyDebug, CopyResult, OffloadReflinkDebug, ReflinkMode, SparseDebug, SparseMode}; + +/// Copies `source` to `dest` for systems without copy-on-write +pub(crate) fn copy_on_write( + source: &Path, + dest: &Path, + reflink_mode: ReflinkMode, + sparse_mode: SparseMode, + context: &str, + source_is_fifo: bool, + source_is_stream: bool, +) -> CopyResult { + if reflink_mode != ReflinkMode::Never { + return Err("--reflink is only supported on linux and macOS" + .to_string() + .into()); + } + if sparse_mode != SparseMode::Auto { + return Err("--sparse is only supported on linux".to_string().into()); + } + let copy_debug = CopyDebug { + offload: OffloadReflinkDebug::Unsupported, + reflink: OffloadReflinkDebug::Unsupported, + sparse_detection: SparseDebug::Unsupported, + }; + + if source_is_stream { + let mut src_file = File::open(source)?; + let mode = 0o622 & !get_umask(); + let mut dst_file = OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest)?; + + buf_copy::copy_stream(&mut src_file, &mut dst_file) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::Other)) + .context(context)?; + + if source_is_fifo { + dst_file.set_permissions(src_file.metadata()?.permissions())?; + } + return Ok(copy_debug); + } + + fs::copy(source, dest).context(context)?; + + Ok(copy_debug) +} diff --git a/src/uu/csplit/Cargo.toml b/src/uu/csplit/Cargo.toml index ba051ff8ab4..508656f681d 100644 --- a/src/uu/csplit/Cargo.toml +++ b/src/uu/csplit/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_csplit" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "csplit ~ (uutils) Output pieces of FILE separated by PATTERN(s) to files 'xx00', 'xx01', ..., and output byte counts of each piece to standard output" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/ls" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/csplit.rs" @@ -18,7 +21,7 @@ path = "src/csplit.rs" clap = { workspace = true } thiserror = { workspace = true } regex = { workspace = true } -uucore = { workspace = true, features = ["entries", "fs"] } +uucore = { workspace = true, features = ["entries", "fs", "format"] } [[bin]] name = "csplit" diff --git a/src/uu/csplit/src/csplit.rs b/src/uu/csplit/src/csplit.rs index 6c9a776c388..621823aebba 100644 --- a/src/uu/csplit/src/csplit.rs +++ b/src/uu/csplit/src/csplit.rs @@ -2,22 +2,21 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -#![crate_name = "uu_csplit"] // spell-checker:ignore rustdoc #![allow(rustdoc::private_intra_doc_links)] use std::cmp::Ordering; -use std::io::{self, BufReader}; +use std::io::{self, BufReader, ErrorKind}; use std::{ - fs::{remove_file, File}, + fs::{File, remove_file}, io::{BufRead, BufWriter, Write}, }; -use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command}; use regex::Regex; use uucore::display::Quotable; use uucore::error::{FromIo, UResult}; -use uucore::{crash_if_err, format_usage, help_about, help_section, help_usage}; +use uucore::{format_usage, help_about, help_section, help_usage}; mod csplit_error; mod patterns; @@ -44,7 +43,7 @@ mod options { /// Command line options for csplit. pub struct CsplitOptions { - split_name: crate::SplitName, + split_name: SplitName, keep_files: bool, quiet: bool, elide_empty_files: bool, @@ -52,31 +51,51 @@ pub struct CsplitOptions { } impl CsplitOptions { - fn new(matches: &ArgMatches) -> Self { + fn new(matches: &ArgMatches) -> Result { let keep_files = matches.get_flag(options::KEEP_FILES); let quiet = matches.get_flag(options::QUIET); let elide_empty_files = matches.get_flag(options::ELIDE_EMPTY_FILES); let suppress_matched = matches.get_flag(options::SUPPRESS_MATCHED); - Self { - split_name: crash_if_err!( - 1, - SplitName::new( - matches - .get_one::(options::PREFIX) - .map(|s| s.to_owned()), - matches - .get_one::(options::SUFFIX_FORMAT) - .map(|s| s.to_owned()), - matches - .get_one::(options::DIGITS) - .map(|s| s.to_owned()) - ) - ), + Ok(Self { + split_name: SplitName::new( + matches.get_one::(options::PREFIX).cloned(), + matches.get_one::(options::SUFFIX_FORMAT).cloned(), + matches.get_one::(options::DIGITS).cloned(), + )?, keep_files, quiet, elide_empty_files, suppress_matched, + }) + } +} + +pub struct LinesWithNewlines { + inner: T, +} + +impl LinesWithNewlines { + fn new(s: T) -> Self { + Self { inner: s } + } +} + +impl Iterator for LinesWithNewlines { + type Item = io::Result; + + fn next(&mut self) -> Option { + fn ret(v: Vec) -> io::Result { + String::from_utf8(v).map_err(|_| { + io::Error::new(ErrorKind::InvalidData, "stream did not contain valid UTF-8") + }) + } + + let mut v = Vec::new(); + match self.inner.read_until(b'\n', &mut v) { + Ok(0) => None, + Ok(_) => Some(ret(v)), + Err(e) => Some(Err(e)), } } } @@ -93,27 +112,29 @@ impl CsplitOptions { /// - [`CsplitError::MatchNotFound`] if no line matched a regular expression. /// - [`CsplitError::MatchNotFoundOnRepetition`], like previous but after applying the pattern /// more than once. -pub fn csplit( - options: &CsplitOptions, - patterns: Vec, - input: T, -) -> Result<(), CsplitError> +pub fn csplit(options: &CsplitOptions, patterns: &[String], input: T) -> Result<(), CsplitError> where T: BufRead, { - let mut input_iter = InputSplitter::new(input.lines().enumerate()); + let enumerated_input_lines = LinesWithNewlines::new(input) + .map(|line| line.map_err_context(|| "read error".to_string())) + .enumerate(); + let mut input_iter = InputSplitter::new(enumerated_input_lines); let mut split_writer = SplitWriter::new(options); + let patterns: Vec = patterns::get_patterns(patterns)?; let ret = do_csplit(&mut split_writer, patterns, &mut input_iter); - // consume the rest - input_iter.rewind_buffer(); - if let Some((_, line)) = input_iter.next() { - split_writer.new_writer()?; - split_writer.writeln(&line?)?; - for (_, line) in input_iter { + // consume the rest, unless there was an error + if ret.is_ok() { + input_iter.rewind_buffer(); + if let Some((_, line)) = input_iter.next() { + split_writer.new_writer()?; split_writer.writeln(&line?)?; + for (_, line) in input_iter { + split_writer.writeln(&line?)?; + } + split_writer.finish_split(); } - split_writer.finish_split(); } // delete files on error by default if ret.is_err() && !options.keep_files { @@ -128,7 +149,7 @@ fn do_csplit( input_iter: &mut InputSplitter, ) -> Result<(), CsplitError> where - I: Iterator)>, + I: Iterator)>, { // split the file based on patterns for pattern in patterns { @@ -205,16 +226,21 @@ struct SplitWriter<'a> { dev_null: bool, } -impl<'a> Drop for SplitWriter<'a> { +impl Drop for SplitWriter<'_> { fn drop(&mut self) { if self.options.elide_empty_files && self.size == 0 { let file_name = self.options.split_name.get(self.counter); - remove_file(file_name).expect("Failed to elide split"); + // In the case of `echo a | csplit -z - %a%1`, the file + // `xx00` does not exist because the positive offset + // advanced past the end of the input. Since there is no + // file to remove in that case, `remove_file` would return + // an error, so we just ignore it. + let _ = remove_file(file_name); } } } -impl<'a> SplitWriter<'a> { +impl SplitWriter<'_> { fn new(options: &CsplitOptions) -> SplitWriter { SplitWriter { options, @@ -245,7 +271,7 @@ impl<'a> SplitWriter<'a> { self.dev_null = true; } - /// Writes the line to the current split, appending a newline character. + /// Writes the line to the current split. /// If [`self.dev_null`] is true, then the line is discarded. /// /// # Errors @@ -257,8 +283,7 @@ impl<'a> SplitWriter<'a> { Some(ref mut current_writer) => { let bytes = line.as_bytes(); current_writer.write_all(bytes)?; - current_writer.write_all(b"\n")?; - self.size += bytes.len() + 1; + self.size += bytes.len(); } None => panic!("trying to write to a split that was not created"), } @@ -316,18 +341,18 @@ impl<'a> SplitWriter<'a> { input_iter: &mut InputSplitter, ) -> Result<(), CsplitError> where - I: Iterator)>, + I: Iterator)>, { input_iter.rewind_buffer(); input_iter.set_size_of_buffer(1); let mut ret = Err(CsplitError::LineOutOfRange(pattern_as_str.to_string())); while let Some((ln, line)) = input_iter.next() { - let l = line?; + let line = line?; match n.cmp(&(&ln + 1)) { Ordering::Less => { assert!( - input_iter.add_line_to_buffer(ln, l).is_none(), + input_iter.add_line_to_buffer(ln, line).is_none(), "the buffer is big enough to contain 1 line" ); ret = Ok(()); @@ -336,7 +361,7 @@ impl<'a> SplitWriter<'a> { Ordering::Equal => { assert!( self.options.suppress_matched - || input_iter.add_line_to_buffer(ln, l).is_none(), + || input_iter.add_line_to_buffer(ln, line).is_none(), "the buffer is big enough to contain 1 line" ); ret = Ok(()); @@ -344,7 +369,7 @@ impl<'a> SplitWriter<'a> { } Ordering::Greater => (), } - self.writeln(&l)?; + self.writeln(&line)?; } self.finish_split(); ret @@ -359,7 +384,7 @@ impl<'a> SplitWriter<'a> { /// In addition to errors reading/writing from/to a file, the following errors may be returned: /// - if no line matched, an [`CsplitError::MatchNotFound`]. /// - if there are not enough lines to accommodate the offset, an - /// [`CsplitError::LineOutOfRange`]. + /// [`CsplitError::LineOutOfRange`]. #[allow(clippy::cognitive_complexity)] fn do_to_match( &mut self, @@ -369,7 +394,7 @@ impl<'a> SplitWriter<'a> { input_iter: &mut InputSplitter, ) -> Result<(), CsplitError> where - I: Iterator)>, + I: Iterator)>, { if offset >= 0 { // The offset is zero or positive, no need for a buffer on the lines read. @@ -381,18 +406,27 @@ impl<'a> SplitWriter<'a> { input_iter.set_size_of_buffer(1); while let Some((ln, line)) = input_iter.next() { - let l = line?; - if regex.is_match(&l) { + let line = line?; + let l = line + .strip_suffix("\r\n") + .unwrap_or_else(|| line.strip_suffix('\n').unwrap_or(&line)); + if regex.is_match(l) { + let mut next_line_suppress_matched = false; match (self.options.suppress_matched, offset) { // no offset, add the line to the next split (false, 0) => { assert!( - input_iter.add_line_to_buffer(ln, l).is_none(), + input_iter.add_line_to_buffer(ln, line).is_none(), "the buffer is big enough to contain 1 line" ); } // a positive offset, some more lines need to be added to the current split - (false, _) => self.writeln(&l)?, + (false, _) => self.writeln(&line)?, + // suppress matched option true, but there is a positive offset, so the line is printed + (true, 1..) => { + next_line_suppress_matched = true; + self.writeln(&line)?; + } _ => (), }; offset -= 1; @@ -413,9 +447,14 @@ impl<'a> SplitWriter<'a> { offset -= 1; } self.finish_split(); + + // if we have to suppress one line after we take the next and do nothing + if next_line_suppress_matched { + input_iter.next(); + } return Ok(()); } - self.writeln(&l)?; + self.writeln(&line)?; } } else { // With a negative offset we use a buffer to keep the lines within the offset. @@ -426,26 +465,35 @@ impl<'a> SplitWriter<'a> { let offset_usize = -offset as usize; input_iter.set_size_of_buffer(offset_usize); while let Some((ln, line)) = input_iter.next() { - let l = line?; - if regex.is_match(&l) { + let line = line?; + let l = line + .strip_suffix("\r\n") + .unwrap_or_else(|| line.strip_suffix('\n').unwrap_or(&line)); + if regex.is_match(l) { for line in input_iter.shrink_buffer_to_size() { self.writeln(&line)?; } - if !self.options.suppress_matched { + if self.options.suppress_matched { + // since offset_usize is for sure greater than 0 + // the first element of the buffer should be removed and this + // line inserted to be coherent with GNU implementation + input_iter.add_line_to_buffer(ln, line); + } else { // add 1 to the buffer size to make place for the matched line input_iter.set_size_of_buffer(offset_usize + 1); assert!( - input_iter.add_line_to_buffer(ln, l).is_none(), + input_iter.add_line_to_buffer(ln, line).is_none(), "should be big enough to hold every lines" ); } + self.finish_split(); if input_iter.buffer_len() < offset_usize { return Err(CsplitError::LineOutOfRange(pattern_as_str.to_string())); } return Ok(()); } - if let Some(line) = input_iter.add_line_to_buffer(ln, l) { + if let Some(line) = input_iter.add_line_to_buffer(ln, line) { self.writeln(&line)?; } } @@ -464,7 +512,7 @@ impl<'a> SplitWriter<'a> { /// This is used to pass matching lines to the next split and to support patterns with a negative offset. struct InputSplitter where - I: Iterator)>, + I: Iterator)>, { iter: I, buffer: Vec<::Item>, @@ -477,7 +525,7 @@ where impl InputSplitter where - I: Iterator)>, + I: Iterator)>, { fn new(iter: I) -> Self { Self { @@ -541,7 +589,7 @@ where impl Iterator for InputSplitter where - I: Iterator)>, + I: Iterator)>, { type Item = ::Item; @@ -558,8 +606,6 @@ where #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_ignore(); - let matches = uu_app().try_get_matches_from(args)?; // get the file to split @@ -571,29 +617,23 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { .unwrap() .map(|s| s.to_string()) .collect(); - let patterns = patterns::get_patterns(&patterns[..])?; - let options = CsplitOptions::new(&matches); + let options = CsplitOptions::new(&matches)?; if file_name == "-" { let stdin = io::stdin(); - Ok(csplit(&options, patterns, stdin.lock())?) + Ok(csplit(&options, &patterns, stdin.lock())?) } else { let file = File::open(file_name) - .map_err_context(|| format!("cannot access {}", file_name.quote()))?; - let file_metadata = file - .metadata() - .map_err_context(|| format!("cannot access {}", file_name.quote()))?; - if !file_metadata.is_file() { - return Err(CsplitError::NotRegularFile(file_name.to_string()).into()); - } - Ok(csplit(&options, patterns, BufReader::new(file))?) + .map_err_context(|| format!("cannot open {} for reading", file_name.quote()))?; + Ok(csplit(&options, &patterns, BufReader::new(file))?) } } pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) + .args_override_self(true) .infer_long_args(true) .arg( Arg::new(options::SUFFIX_FORMAT) @@ -631,8 +671,9 @@ pub fn uu_app() -> Command { ) .arg( Arg::new(options::QUIET) - .short('s') + .short('q') .long(options::QUIET) + .visible_short_alias('s') .visible_alias("silent") .help("do not print counts of output file sizes") .action(ArgAction::SetTrue), @@ -653,7 +694,7 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::PATTERN) .hide(true) - .action(clap::ArgAction::Append) + .action(ArgAction::Append) .required(true), ) .after_help(AFTER_HELP) @@ -664,6 +705,7 @@ mod tests { use super::*; #[test] + #[allow(clippy::cognitive_complexity)] fn input_splitter() { let input = vec![ Ok(String::from("aaa")), @@ -736,6 +778,7 @@ mod tests { } #[test] + #[allow(clippy::cognitive_complexity)] fn input_splitter_interrupt_rewind() { let input = vec![ Ok(String::from("aaa")), diff --git a/src/uu/csplit/src/csplit_error.rs b/src/uu/csplit/src/csplit_error.rs index 1559a29f8bc..a8c0fd1af08 100644 --- a/src/uu/csplit/src/csplit_error.rs +++ b/src/uu/csplit/src/csplit_error.rs @@ -12,7 +12,7 @@ use uucore::error::UError; #[derive(Debug, Error)] pub enum CsplitError { #[error("IO error: {}", _0)] - IoError(io::Error), + IoError(#[from] io::Error), #[error("{}: line number out of range", ._0.quote())] LineOutOfRange(String), #[error("{}: line number out of range on repetition {}", ._0.quote(), _1)] @@ -21,7 +21,7 @@ pub enum CsplitError { MatchNotFound(String), #[error("{}: match not found on repetition {}", ._0.quote(), _1)] MatchNotFoundOnRepetition(String, usize), - #[error("line number must be greater than zero")] + #[error("0: line number must be greater than zero")] LineNumberIsZero, #[error("line number '{}' is smaller than preceding line number, {}", _0, _1)] LineNumberSmallerThanPrevious(usize, usize), @@ -35,16 +35,21 @@ pub enum CsplitError { SuffixFormatTooManyPercents, #[error("{} is not a regular file", ._0.quote())] NotRegularFile(String), + #[error("{}", _0)] + UError(Box), } -impl From for CsplitError { - fn from(error: io::Error) -> Self { - Self::IoError(error) +impl From> for CsplitError { + fn from(error: Box) -> Self { + Self::UError(error) } } impl UError for CsplitError { fn code(&self) -> i32 { - 1 + match self { + Self::UError(e) => e.code(), + _ => 1, + } } } diff --git a/src/uu/csplit/src/patterns.rs b/src/uu/csplit/src/patterns.rs index fd96fd9fb9e..9326ea3795e 100644 --- a/src/uu/csplit/src/patterns.rs +++ b/src/uu/csplit/src/patterns.rs @@ -25,14 +25,14 @@ pub enum Pattern { SkipToMatch(Regex, i32, ExecutePattern), } -impl ToString for Pattern { - fn to_string(&self) -> String { +impl std::fmt::Display for Pattern { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::UpToLine(n, _) => n.to_string(), - Self::UpToMatch(regex, 0, _) => format!("/{}/", regex.as_str()), - Self::UpToMatch(regex, offset, _) => format!("/{}/{:+}", regex.as_str(), offset), - Self::SkipToMatch(regex, 0, _) => format!("%{}%", regex.as_str()), - Self::SkipToMatch(regex, offset, _) => format!("%{}%{:+}", regex.as_str(), offset), + Self::UpToLine(n, _) => write!(f, "{n}"), + Self::UpToMatch(regex, 0, _) => write!(f, "/{}/", regex.as_str()), + Self::UpToMatch(regex, offset, _) => write!(f, "/{}/{offset:+}", regex.as_str()), + Self::SkipToMatch(regex, 0, _) => write!(f, "%{}%", regex.as_str()), + Self::SkipToMatch(regex, offset, _) => write!(f, "%{}%{offset:+}", regex.as_str()), } } } @@ -106,8 +106,8 @@ pub fn get_patterns(args: &[String]) -> Result, CsplitError> { fn extract_patterns(args: &[String]) -> Result, CsplitError> { let mut patterns = Vec::with_capacity(args.len()); let to_match_reg = - Regex::new(r"^(/(?P.+)/|%(?P.+)%)(?P[\+-]\d+)?$").unwrap(); - let execute_ntimes_reg = Regex::new(r"^\{(?P\d+)|\*\}$").unwrap(); + Regex::new(r"^(/(?P.+)/|%(?P.+)%)(?P[\+-]?[0-9]+)?$").unwrap(); + let execute_ntimes_reg = Regex::new(r"^\{(?P[0-9]+)|\*\}$").unwrap(); let mut iter = args.iter().peekable(); while let Some(arg) = iter.next() { @@ -168,7 +168,7 @@ fn validate_line_numbers(patterns: &[Pattern]) -> Result<(), CsplitError> { (_, 0) => Err(CsplitError::LineNumberIsZero), // two consecutive numbers should not be equal (n, m) if n == m => { - show_warning!("line number '{}' is the same as preceding line number", n); + show_warning!("line number '{n}' is the same as preceding line number"); Ok(n) } // a number cannot be greater than the one that follows @@ -196,7 +196,7 @@ mod tests { .collect(); let patterns = get_patterns(input.as_slice()).unwrap(); assert_eq!(patterns.len(), 3); - match patterns.get(0) { + match patterns.first() { Some(Pattern::UpToLine(24, ExecutePattern::Times(1))) => (), _ => panic!("expected UpToLine pattern"), }; @@ -211,6 +211,7 @@ mod tests { } #[test] + #[allow(clippy::cognitive_complexity)] fn up_to_match_pattern() { let input: Vec = vec![ "/test1.*end$/", @@ -218,15 +219,16 @@ mod tests { "{*}", "/test3.*end$/", "{4}", - "/test4.*end$/+3", - "/test5.*end$/-3", + "/test4.*end$/3", + "/test5.*end$/+3", + "/test6.*end$/-3", ] .into_iter() .map(|v| v.to_string()) .collect(); let patterns = get_patterns(input.as_slice()).unwrap(); - assert_eq!(patterns.len(), 5); - match patterns.get(0) { + assert_eq!(patterns.len(), 6); + match patterns.first() { Some(Pattern::UpToMatch(reg, 0, ExecutePattern::Times(1))) => { let parsed_reg = format!("{reg}"); assert_eq!(parsed_reg, "test1.*end$"); @@ -255,15 +257,23 @@ mod tests { _ => panic!("expected UpToMatch pattern"), }; match patterns.get(4) { - Some(Pattern::UpToMatch(reg, -3, ExecutePattern::Times(1))) => { + Some(Pattern::UpToMatch(reg, 3, ExecutePattern::Times(1))) => { let parsed_reg = format!("{reg}"); assert_eq!(parsed_reg, "test5.*end$"); } _ => panic!("expected UpToMatch pattern"), }; + match patterns.get(5) { + Some(Pattern::UpToMatch(reg, -3, ExecutePattern::Times(1))) => { + let parsed_reg = format!("{reg}"); + assert_eq!(parsed_reg, "test6.*end$"); + } + _ => panic!("expected UpToMatch pattern"), + }; } #[test] + #[allow(clippy::cognitive_complexity)] fn skip_to_match_pattern() { let input: Vec = vec![ "%test1.*end$%", @@ -271,15 +281,16 @@ mod tests { "{*}", "%test3.*end$%", "{4}", - "%test4.*end$%+3", - "%test5.*end$%-3", + "%test4.*end$%3", + "%test5.*end$%+3", + "%test6.*end$%-3", ] .into_iter() .map(|v| v.to_string()) .collect(); let patterns = get_patterns(input.as_slice()).unwrap(); - assert_eq!(patterns.len(), 5); - match patterns.get(0) { + assert_eq!(patterns.len(), 6); + match patterns.first() { Some(Pattern::SkipToMatch(reg, 0, ExecutePattern::Times(1))) => { let parsed_reg = format!("{reg}"); assert_eq!(parsed_reg, "test1.*end$"); @@ -308,12 +319,19 @@ mod tests { _ => panic!("expected SkipToMatch pattern"), }; match patterns.get(4) { - Some(Pattern::SkipToMatch(reg, -3, ExecutePattern::Times(1))) => { + Some(Pattern::SkipToMatch(reg, 3, ExecutePattern::Times(1))) => { let parsed_reg = format!("{reg}"); assert_eq!(parsed_reg, "test5.*end$"); } _ => panic!("expected SkipToMatch pattern"), }; + match patterns.get(5) { + Some(Pattern::SkipToMatch(reg, -3, ExecutePattern::Times(1))) => { + let parsed_reg = format!("{reg}"); + assert_eq!(parsed_reg, "test6.*end$"); + } + _ => panic!("expected SkipToMatch pattern"), + }; } #[test] diff --git a/src/uu/csplit/src/split_name.rs b/src/uu/csplit/src/split_name.rs index 4d94b56a923..925ded4cc7b 100644 --- a/src/uu/csplit/src/split_name.rs +++ b/src/uu/csplit/src/split_name.rs @@ -4,21 +4,22 @@ // file that was distributed with this source code. // spell-checker:ignore (regex) diuox -use regex::Regex; +use uucore::format::{Format, FormatError, num_format::UnsignedInt}; use crate::csplit_error::CsplitError; /// Computes the filename of a split, taking into consideration a possible user-defined suffix /// format. pub struct SplitName { - fn_split_name: Box String>, + prefix: Vec, + format: Format, } impl SplitName { /// Creates a new SplitName with the given user-defined options: /// - `prefix_opt` specifies a prefix for all splits. /// - `format_opt` specifies a custom format for the suffix part of the filename, using the - /// `sprintf` format notation. + /// `sprintf` format notation. /// - `n_digits_opt` defines the width of the split number. /// /// # Caveats @@ -36,6 +37,7 @@ impl SplitName { ) -> Result { // get the prefix let prefix = prefix_opt.unwrap_or_else(|| "xx".to_string()); + // the width for the split offset let n_digits = n_digits_opt .map(|opt| { @@ -44,120 +46,26 @@ impl SplitName { }) .transpose()? .unwrap_or(2); - // translate the custom format into a function - let fn_split_name: Box String> = match format_opt { - None => Box::new(move |n: usize| -> String { format!("{prefix}{n:0n_digits$}") }), - Some(custom) => { - let spec = - Regex::new(r"(?P%((?P[0#-])(?P\d+)?)?(?P[diuoxX]))") - .unwrap(); - let mut captures_iter = spec.captures_iter(&custom); - let custom_fn: Box String> = match captures_iter.next() { - Some(captures) => { - let all = captures.name("ALL").unwrap(); - let before = custom[0..all.start()].to_owned(); - let after = custom[all.end()..].to_owned(); - let width = match captures.name("WIDTH") { - None => 0, - Some(m) => m.as_str().parse::().unwrap(), - }; - match (captures.name("FLAG"), captures.name("TYPE")) { - (None, Some(ref t)) => match t.as_str() { - "d" | "i" | "u" => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n}{after}") - }), - "o" => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:o}{after}") - }), - "x" => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:x}{after}") - }), - "X" => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:X}{after}") - }), - _ => return Err(CsplitError::SuffixFormatIncorrect), - }, - (Some(ref f), Some(ref t)) => { - match (f.as_str(), t.as_str()) { - /* - * zero padding - */ - // decimal - ("0", "d" | "i" | "u") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:0width$}{after}") - }), - // octal - ("0", "o") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:0width$o}{after}") - }), - // lower hexadecimal - ("0", "x") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:0width$x}{after}") - }), - // upper hexadecimal - ("0", "X") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:0width$X}{after}") - }), - - /* - * Alternate form - */ - // octal - ("#", "o") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:>#width$o}{after}") - }), - // lower hexadecimal - ("#", "x") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:>#width$x}{after}") - }), - // upper hexadecimal - ("#", "X") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:>#width$X}{after}") - }), - - /* - * Left adjusted - */ - // decimal - ("-", "d" | "i" | "u") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:<#width$}{after}") - }), - // octal - ("-", "o") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:<#width$o}{after}") - }), - // lower hexadecimal - ("-", "x") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:<#width$x}{after}") - }), - // upper hexadecimal - ("-", "X") => Box::new(move |n: usize| -> String { - format!("{prefix}{before}{n:<#width$X}{after}") - }), - - _ => return Err(CsplitError::SuffixFormatIncorrect), - } - } - _ => return Err(CsplitError::SuffixFormatIncorrect), - } - } - None => return Err(CsplitError::SuffixFormatIncorrect), - }; - - // there cannot be more than one format pattern - if captures_iter.next().is_some() { - return Err(CsplitError::SuffixFormatTooManyPercents); - } - custom_fn - } - }; - Ok(Self { fn_split_name }) + let format_string = format_opt.unwrap_or_else(|| format!("%0{n_digits}u")); + + let format = match Format::::parse(format_string) { + Ok(format) => Ok(format), + Err(FormatError::TooManySpecs(_)) => Err(CsplitError::SuffixFormatTooManyPercents), + Err(_) => Err(CsplitError::SuffixFormatIncorrect), + }?; + + Ok(Self { + prefix: prefix.as_bytes().to_owned(), + format, + }) } /// Returns the filename of the i-th split. pub fn get(&self, n: usize) -> String { - (self.fn_split_name)(n) + let mut v = self.prefix.clone(); + self.format.fmt(&mut v, n as u64).unwrap(); + String::from_utf8_lossy(&v).to_string() } } @@ -279,7 +187,7 @@ mod tests { #[test] fn alternate_form_octal() { let split_name = SplitName::new(None, Some(String::from("cst-%#10o-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst- 0o52-"); + assert_eq!(split_name.get(42), "xxcst- 052-"); } #[test] @@ -291,7 +199,7 @@ mod tests { #[test] fn alternate_form_upper_hex() { let split_name = SplitName::new(None, Some(String::from("cst-%#10X-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst- 0x2A-"); + assert_eq!(split_name.get(42), "xxcst- 0X2A-"); } #[test] @@ -315,19 +223,19 @@ mod tests { #[test] fn left_adjusted_octal() { let split_name = SplitName::new(None, Some(String::from("cst-%-10o-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst-0o52 -"); + assert_eq!(split_name.get(42), "xxcst-52 -"); } #[test] fn left_adjusted_lower_hex() { let split_name = SplitName::new(None, Some(String::from("cst-%-10x-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst-0x2a -"); + assert_eq!(split_name.get(42), "xxcst-2a -"); } #[test] fn left_adjusted_upper_hex() { let split_name = SplitName::new(None, Some(String::from("cst-%-10X-")), None).unwrap(); - assert_eq!(split_name.get(42), "xxcst-0x2A -"); + assert_eq!(split_name.get(42), "xxcst-2A -"); } #[test] diff --git a/src/uu/cut/Cargo.toml b/src/uu/cut/Cargo.toml index affeb38058f..84fe09f23cf 100644 --- a/src/uu/cut/Cargo.toml +++ b/src/uu/cut/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_cut" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "cut ~ (uutils) display byte/field columns of input lines" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/cut" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/cut.rs" @@ -19,7 +22,6 @@ clap = { workspace = true } uucore = { workspace = true, features = ["ranges"] } memchr = { workspace = true } bstr = { workspace = true } -is-terminal = { workspace = true } [[bin]] name = "cut" diff --git a/src/uu/cut/cut.md b/src/uu/cut/cut.md index 972fcb6322e..5c21d23dcf9 100644 --- a/src/uu/cut/cut.md +++ b/src/uu/cut/cut.md @@ -3,7 +3,7 @@ ``` -cut [-d|-w] [-s] [-z] [--output-delimiter] ((-f|-b|-c) {{sequence}}) {{sourcefile}}+ +cut OPTION... [FILE]... ``` Prints specified byte or field columns from each line of stdin or the input files diff --git a/src/uu/cut/src/cut.rs b/src/uu/cut/src/cut.rs index 89128175354..49f5445f36f 100644 --- a/src/uu/cut/src/cut.rs +++ b/src/uu/cut/src/cut.rs @@ -6,19 +6,20 @@ // spell-checker:ignore (ToDO) delim sourcefiles use bstr::io::BufReadExt; -use clap::{crate_version, Arg, ArgAction, Command}; -use is_terminal::IsTerminal; +use clap::{Arg, ArgAction, ArgMatches, Command, builder::ValueParser}; +use std::ffi::OsString; use std::fs::File; -use std::io::{stdin, stdout, BufReader, BufWriter, Read, Write}; +use std::io::{BufRead, BufReader, BufWriter, IsTerminal, Read, Write, stdin, stdout}; use std::path::Path; use uucore::display::Quotable; -use uucore::error::{FromIo, UResult, USimpleError}; +use uucore::error::{FromIo, UResult, USimpleError, set_exit_code}; use uucore::line_ending::LineEnding; +use uucore::os_str_as_bytes; use self::searcher::Searcher; use matcher::{ExactMatcher, Matcher, WhitespaceMatcher}; use uucore::ranges::Range; -use uucore::{format_usage, help_about, help_section, help_usage, show, show_error, show_if_err}; +use uucore::{format_usage, help_about, help_section, help_usage, show_error, show_if_err}; mod matcher; mod searcher; @@ -27,34 +28,37 @@ const USAGE: &str = help_usage!("cut.md"); const ABOUT: &str = help_about!("cut.md"); const AFTER_HELP: &str = help_section!("after help", "cut.md"); -struct Options { - out_delim: Option, +struct Options<'a> { + out_delimiter: Option<&'a [u8]>, line_ending: LineEnding, + field_opts: Option>, } -enum Delimiter { +enum Delimiter<'a> { Whitespace, - String(String), // FIXME: use char? + Slice(&'a [u8]), } -struct FieldOptions { - delimiter: Delimiter, - out_delimiter: Option, +struct FieldOptions<'a> { + delimiter: Delimiter<'a>, only_delimited: bool, - line_ending: LineEnding, } -enum Mode { - Bytes(Vec, Options), - Characters(Vec, Options), - Fields(Vec, FieldOptions), +enum Mode<'a> { + Bytes(Vec, Options<'a>), + Characters(Vec, Options<'a>), + Fields(Vec, Options<'a>), } -fn stdout_writer() -> Box { - if std::io::stdout().is_terminal() { - Box::new(stdout()) - } else { - Box::new(BufWriter::new(stdout())) as Box +impl Default for Delimiter<'_> { + fn default() -> Self { + Self::Slice(b"\t") + } +} + +impl<'a> From<&'a OsString> for Delimiter<'a> { + fn from(s: &'a OsString) -> Self { + Self::Slice(os_str_as_bytes(s).unwrap()) } } @@ -66,15 +70,15 @@ fn list_to_ranges(list: &str, complement: bool) -> Result, String> { } } -fn cut_bytes(reader: R, ranges: &[Range], opts: &Options) -> UResult<()> { +fn cut_bytes( + reader: R, + out: &mut W, + ranges: &[Range], + opts: &Options, +) -> UResult<()> { let newline_char = opts.line_ending.into(); let mut buf_in = BufReader::new(reader); - let mut out = stdout_writer(); - let delim = opts - .out_delim - .as_ref() - .map_or("", String::as_str) - .as_bytes(); + let out_delim = opts.out_delimiter.unwrap_or(b"\t"); let result = buf_in.for_byte_record(newline_char, |line| { let mut print_delim = false; @@ -83,8 +87,8 @@ fn cut_bytes(reader: R, ranges: &[Range], opts: &Options) -> UResult<() break; } if print_delim { - out.write_all(delim)?; - } else if opts.out_delim.is_some() { + out.write_all(out_delim)?; + } else if opts.out_delimiter.is_some() { print_delim = true; } // change `low` from 1-indexed value to 0-index value @@ -104,16 +108,16 @@ fn cut_bytes(reader: R, ranges: &[Range], opts: &Options) -> UResult<() } // Output delimiter is explicitly specified -fn cut_fields_explicit_out_delim( +fn cut_fields_explicit_out_delim( reader: R, + out: &mut W, matcher: &M, ranges: &[Range], only_delimited: bool, newline_char: u8, - out_delim: &str, + out_delim: &[u8], ) -> UResult<()> { let mut buf_in = BufReader::new(reader); - let mut out = stdout_writer(); let result = buf_in.for_byte_record_with_terminator(newline_char, |line| { let mut fields_pos = 1; @@ -123,8 +127,9 @@ fn cut_fields_explicit_out_delim( if delim_search.peek().is_none() { if !only_delimited { + // Always write the entire line, even if it doesn't end with `newline_char` out.write_all(line)?; - if line[line.len() - 1] != newline_char { + if line.is_empty() || line[line.len() - 1] != newline_char { out.write_all(&[newline_char])?; } } @@ -146,7 +151,7 @@ fn cut_fields_explicit_out_delim( for _ in 0..=high - low { // skip printing delimiter if this is the first matching field for this line if print_delim { - out.write_all(out_delim.as_bytes())?; + out.write_all(out_delim)?; } else { print_delim = true; } @@ -188,15 +193,15 @@ fn cut_fields_explicit_out_delim( } // Output delimiter is the same as input delimiter -fn cut_fields_implicit_out_delim( +fn cut_fields_implicit_out_delim( reader: R, + out: &mut W, matcher: &M, ranges: &[Range], only_delimited: bool, newline_char: u8, ) -> UResult<()> { let mut buf_in = BufReader::new(reader); - let mut out = stdout_writer(); let result = buf_in.for_byte_record_with_terminator(newline_char, |line| { let mut fields_pos = 1; @@ -206,8 +211,9 @@ fn cut_fields_implicit_out_delim( if delim_search.peek().is_none() { if !only_delimited { + // Always write the entire line, even if it doesn't end with `newline_char` out.write_all(line)?; - if line[line.len() - 1] != newline_char { + if line.is_empty() || line[line.len() - 1] != newline_char { out.write_all(&[newline_char])?; } } @@ -257,39 +263,83 @@ fn cut_fields_implicit_out_delim( Ok(()) } -fn cut_fields(reader: R, ranges: &[Range], opts: &FieldOptions) -> UResult<()> { +// The input delimiter is identical to `newline_char` +fn cut_fields_newline_char_delim( + reader: R, + out: &mut W, + ranges: &[Range], + newline_char: u8, + out_delim: &[u8], +) -> UResult<()> { + let buf_in = BufReader::new(reader); + + let segments: Vec<_> = buf_in.split(newline_char).filter_map(|x| x.ok()).collect(); + let mut print_delim = false; + + for &Range { low, high } in ranges { + for i in low..=high { + // "- 1" is necessary because fields start from 1 whereas a Vec starts from 0 + if let Some(segment) = segments.get(i - 1) { + if print_delim { + out.write_all(out_delim)?; + } else { + print_delim = true; + } + out.write_all(segment.as_slice())?; + } else { + break; + } + } + } + out.write_all(&[newline_char])?; + Ok(()) +} + +fn cut_fields( + reader: R, + out: &mut W, + ranges: &[Range], + opts: &Options, +) -> UResult<()> { let newline_char = opts.line_ending.into(); - match opts.delimiter { - Delimiter::String(ref delim) => { - let matcher = ExactMatcher::new(delim.as_bytes()); + let field_opts = opts.field_opts.as_ref().unwrap(); // it is safe to unwrap() here - field_opts will always be Some() for cut_fields() call + match field_opts.delimiter { + Delimiter::Slice(delim) if delim == [newline_char] => { + let out_delim = opts.out_delimiter.unwrap_or(delim); + cut_fields_newline_char_delim(reader, out, ranges, newline_char, out_delim) + } + Delimiter::Slice(delim) => { + let matcher = ExactMatcher::new(delim); match opts.out_delimiter { - Some(ref out_delim) => cut_fields_explicit_out_delim( + Some(out_delim) => cut_fields_explicit_out_delim( reader, + out, &matcher, ranges, - opts.only_delimited, + field_opts.only_delimited, newline_char, out_delim, ), None => cut_fields_implicit_out_delim( reader, + out, &matcher, ranges, - opts.only_delimited, + field_opts.only_delimited, newline_char, ), } } Delimiter::Whitespace => { let matcher = WhitespaceMatcher {}; - let out_delim = opts.out_delimiter.as_deref().unwrap_or("\t"); cut_fields_explicit_out_delim( reader, + out, &matcher, ranges, - opts.only_delimited, + field_opts.only_delimited, newline_char, - out_delim, + opts.out_delimiter.unwrap_or(b"\t"), ) } } @@ -302,6 +352,12 @@ fn cut_files(mut filenames: Vec, mode: &Mode) { filenames.push("-".to_owned()); } + let mut out: Box = if stdout().is_terminal() { + Box::new(stdout()) + } else { + Box::new(BufWriter::new(stdout())) as Box + }; + for filename in &filenames { if filename == "-" { if stdin_read { @@ -309,9 +365,9 @@ fn cut_files(mut filenames: Vec, mode: &Mode) { } show_if_err!(match mode { - Mode::Bytes(ref ranges, ref opts) => cut_bytes(stdin(), ranges, opts), - Mode::Characters(ref ranges, ref opts) => cut_bytes(stdin(), ranges, opts), - Mode::Fields(ref ranges, ref opts) => cut_fields(stdin(), ranges, opts), + Mode::Bytes(ranges, opts) => cut_bytes(stdin(), &mut out, ranges, opts), + Mode::Characters(ranges, opts) => cut_bytes(stdin(), &mut out, ranges, opts), + Mode::Fields(ranges, opts) => cut_fields(stdin(), &mut out, ranges, opts), }); stdin_read = true; @@ -320,21 +376,78 @@ fn cut_files(mut filenames: Vec, mode: &Mode) { if path.is_dir() { show_error!("{}: Is a directory", filename.maybe_quote()); + set_exit_code(1); continue; } - show_if_err!(File::open(path) - .map_err_context(|| filename.maybe_quote().to_string()) - .and_then(|file| { - match &mode { - Mode::Bytes(ranges, opts) | Mode::Characters(ranges, opts) => { - cut_bytes(file, ranges, opts) + show_if_err!( + File::open(path) + .map_err_context(|| filename.maybe_quote().to_string()) + .and_then(|file| { + match &mode { + Mode::Bytes(ranges, opts) | Mode::Characters(ranges, opts) => { + cut_bytes(file, &mut out, ranges, opts) + } + Mode::Fields(ranges, opts) => cut_fields(file, &mut out, ranges, opts), } - Mode::Fields(ranges, opts) => cut_fields(file, ranges, opts), - } - })); + }) + ); } } + + show_if_err!(out.flush().map_err_context(|| "write error".into())); +} + +// Get delimiter and output delimiter from `-d`/`--delimiter` and `--output-delimiter` options respectively +// Allow either delimiter to have a value that is neither UTF-8 nor ASCII to align with GNU behavior +fn get_delimiters(matches: &ArgMatches) -> UResult<(Delimiter, Option<&[u8]>)> { + let whitespace_delimited = matches.get_flag(options::WHITESPACE_DELIMITED); + let delim_opt = matches.get_one::(options::DELIMITER); + let delim = match delim_opt { + Some(_) if whitespace_delimited => { + return Err(USimpleError::new( + 1, + "invalid input: Only one of --delimiter (-d) or -w option can be specified", + )); + } + Some(os_string) => { + if os_string == "''" || os_string.is_empty() { + // treat `''` as empty delimiter + Delimiter::Slice(b"\0") + } else { + // For delimiter `-d` option value - allow both UTF-8 (possibly multi-byte) characters + // and Non UTF-8 (and not ASCII) single byte "characters", like `b"\xAD"` to align with GNU behavior + let bytes = os_str_as_bytes(os_string)?; + if os_string.to_str().is_some_and(|s| s.chars().count() > 1) + || os_string.to_str().is_none() && bytes.len() > 1 + { + return Err(USimpleError::new( + 1, + "the delimiter must be a single character", + )); + } else { + Delimiter::from(os_string) + } + } + } + None => { + if whitespace_delimited { + Delimiter::Whitespace + } else { + Delimiter::default() + } + } + }; + let out_delim = matches + .get_one::(options::OUTPUT_DELIMITER) + .map(|os_string| { + if os_string.is_empty() || os_string == "''" { + b"\0" + } else { + os_str_as_bytes(os_string).unwrap() + } + }); + Ok((delim, out_delim)) } mod options { @@ -352,116 +465,76 @@ mod options { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_ignore(); + // GNU's `cut` supports `-d=` to set the delimiter to `=`. + // Clap parsing is limited in this situation, see: + // https://github.com/uutils/coreutils/issues/2424#issuecomment-863825242 + let args: Vec = args + .into_iter() + .map(|x| { + if x == "-d=" { + "--delimiter==".into() + } else { + x + } + }) + .collect(); - let delimiter_is_equal = args.contains(&"-d=".to_string()); // special case let matches = uu_app().try_get_matches_from(args)?; let complement = matches.get_flag(options::COMPLEMENT); + let only_delimited = matches.get_flag(options::ONLY_DELIMITED); + + let (delimiter, out_delimiter) = get_delimiters(&matches)?; + let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO_TERMINATED)); + + // Only one, and only one of cutting mode arguments, i.e. `-b`, `-c`, `-f`, + // is expected. The number of those arguments is used for parsing a cutting + // mode and handling the error cases. + let mode_args_count = [ + matches.indices_of(options::BYTES), + matches.indices_of(options::CHARACTERS), + matches.indices_of(options::FIELDS), + ] + .into_iter() + .map(|indices| indices.unwrap_or_default().count()) + .sum(); let mode_parse = match ( + mode_args_count, matches.get_one::(options::BYTES), matches.get_one::(options::CHARACTERS), matches.get_one::(options::FIELDS), ) { - (Some(byte_ranges), None, None) => list_to_ranges(byte_ranges, complement).map(|ranges| { + (1, Some(byte_ranges), None, None) => list_to_ranges(byte_ranges, complement).map(|ranges| { Mode::Bytes( ranges, Options { - out_delim: Some( - matches - .get_one::(options::OUTPUT_DELIMITER) - .map(|s| s.as_str()) - .unwrap_or_default() - .to_owned(), - ), - line_ending: LineEnding::from_zero_flag(matches.get_flag(options::ZERO_TERMINATED)), + out_delimiter, + line_ending, + field_opts: None, }, ) }), - (None, Some(char_ranges), None) => list_to_ranges(char_ranges, complement).map(|ranges| { + (1, None, Some(char_ranges), None) => list_to_ranges(char_ranges, complement).map(|ranges| { Mode::Characters( ranges, Options { - out_delim: Some( - matches - .get_one::(options::OUTPUT_DELIMITER) - .map(|s| s.as_str()) - .unwrap_or_default() - .to_owned(), - ), - line_ending: LineEnding::from_zero_flag(matches.get_flag(options::ZERO_TERMINATED)), + out_delimiter, + line_ending, + field_opts: None, }, ) }), - (None, None, Some(field_ranges)) => { - list_to_ranges(field_ranges, complement).and_then(|ranges| { - let out_delim = match matches.get_one::(options::OUTPUT_DELIMITER) { - Some(s) => { - if s.is_empty() { - Some("\0".to_owned()) - } else { - Some(s.to_owned()) - } - } - None => None, - }; - - let only_delimited = matches.get_flag(options::ONLY_DELIMITED); - let whitespace_delimited = matches.get_flag(options::WHITESPACE_DELIMITED); - let zero_terminated = matches.get_flag(options::ZERO_TERMINATED); - let line_ending = LineEnding::from_zero_flag(zero_terminated); - - match matches.get_one::(options::DELIMITER).map(|s| s.as_str()) { - Some(_) if whitespace_delimited => { - Err("invalid input: Only one of --delimiter (-d) or -w option can be specified".into()) - } - Some(mut delim) => { - // GNU's `cut` supports `-d=` to set the delimiter to `=`. - // Clap parsing is limited in this situation, see: - // https://github.com/uutils/coreutils/issues/2424#issuecomment-863825242 - if delimiter_is_equal { - delim = "="; - } else if delim == "''" { - // treat `''` as empty delimiter - delim = ""; - } - if delim.chars().count() > 1 { - Err("invalid input: The '--delimiter' ('-d') option expects empty or 1 character long, but was provided a value 2 characters or longer".into()) - } else { - let delim = if delim.is_empty() { - "\0".to_owned() - } else { - delim.to_owned() - }; - - Ok(Mode::Fields( - ranges, - FieldOptions { - delimiter: Delimiter::String(delim), - out_delimiter: out_delim, - only_delimited, - line_ending, - }, - )) - } - } - None => Ok(Mode::Fields( - ranges, - FieldOptions { - delimiter: match whitespace_delimited { - true => Delimiter::Whitespace, - false => Delimiter::String("\t".to_owned()), - }, - out_delimiter: out_delim, - only_delimited, - line_ending, - }, - )), - } - }) - } - (ref b, ref c, ref f) if b.is_some() || c.is_some() || f.is_some() => Err( + (1, None, None, Some(field_ranges)) => list_to_ranges(field_ranges, complement).map(|ranges| { + Mode::Fields( + ranges, + Options { + out_delimiter, + line_ending, + field_opts: Some(FieldOptions { delimiter, only_delimited })}, + ) + }), + (2.., _, _, _) => Err( "invalid usage: expects no more than one of --fields (-f), --chars (-c) or --bytes (-b)".into() ), _ => Err("invalid usage: expects one of --fields (-f), --chars (-c) or --bytes (-b)".into()), @@ -492,7 +565,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let files: Vec = matches .get_many::(options::FILE) .unwrap_or_default() - .map(|s| s.to_owned()) + .cloned() .collect(); match mode_parse { @@ -506,18 +579,27 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .override_usage(format_usage(USAGE)) .about(ABOUT) .after_help(AFTER_HELP) .infer_long_args(true) + // While `args_override_self(true)` for some arguments, such as `-d` + // and `--output-delimiter`, is consistent to the behavior of GNU cut, + // arguments related to cutting mode, i.e. `-b`, `-c`, `-f`, should + // cause an error when there is more than one of them, as described in + // the manual of GNU cut: "Use one, and only one of -b, -c or -f". + // `ArgAction::Append` is used on `-b`, `-c`, `-f` arguments, so that + // the occurrences of those could be counted and be handled accordingly. + .args_override_self(true) .arg( Arg::new(options::BYTES) .short('b') .long(options::BYTES) .help("filter byte columns from the input source") .allow_hyphen_values(true) - .value_name("LIST"), + .value_name("LIST") + .action(ArgAction::Append), ) .arg( Arg::new(options::CHARACTERS) @@ -525,12 +607,14 @@ pub fn uu_app() -> Command { .long(options::CHARACTERS) .help("alias for character mode") .allow_hyphen_values(true) - .value_name("LIST"), + .value_name("LIST") + .action(ArgAction::Append), ) .arg( Arg::new(options::DELIMITER) .short('d') .long(options::DELIMITER) + .value_parser(ValueParser::os_string()) .help("specify the delimiter character that separates fields in the input source. Defaults to Tab.") .value_name("DELIM"), ) @@ -547,7 +631,8 @@ pub fn uu_app() -> Command { .long(options::FIELDS) .help("filter field columns from the input source") .allow_hyphen_values(true) - .value_name("LIST"), + .value_name("LIST") + .action(ArgAction::Append), ) .arg( Arg::new(options::COMPLEMENT) @@ -572,6 +657,7 @@ pub fn uu_app() -> Command { .arg( Arg::new(options::OUTPUT_DELIMITER) .long(options::OUTPUT_DELIMITER) + .value_parser(ValueParser::os_string()) .help("in field mode, replace the delimiter in output lines with this option's argument") .value_name("NEW_DELIM"), ) diff --git a/src/uu/cut/src/matcher.rs b/src/uu/cut/src/matcher.rs index 953e083b139..bb0c44d5bb4 100644 --- a/src/uu/cut/src/matcher.rs +++ b/src/uu/cut/src/matcher.rs @@ -23,7 +23,7 @@ impl<'a> ExactMatcher<'a> { } } -impl<'a> Matcher for ExactMatcher<'a> { +impl Matcher for ExactMatcher<'_> { fn next_match(&self, haystack: &[u8]) -> Option<(usize, usize)> { let mut pos = 0usize; loop { diff --git a/src/uu/cut/src/searcher.rs b/src/uu/cut/src/searcher.rs index 21424790eea..dc252d804f7 100644 --- a/src/uu/cut/src/searcher.rs +++ b/src/uu/cut/src/searcher.rs @@ -27,7 +27,7 @@ impl<'a, 'b, M: Matcher> Searcher<'a, 'b, M> { // Iterate over field delimiters // Returns (first, last) positions of each sequence, where `haystack[first..last]` // corresponds to the delimiter. -impl<'a, 'b, M: Matcher> Iterator for Searcher<'a, 'b, M> { +impl Iterator for Searcher<'_, '_, M> { type Item = (usize, usize); fn next(&mut self) -> Option { @@ -61,7 +61,7 @@ mod exact_searcher_tests { let matcher = ExactMatcher::new("a".as_bytes()); let iter = Searcher::new(&matcher, "".as_bytes()); let items: Vec<(usize, usize)> = iter.collect(); - assert_eq!(vec![] as Vec<(usize, usize)>, items); + assert!(items.is_empty()); } fn test_multibyte(line: &[u8], expected: &[(usize, usize)]) { @@ -140,7 +140,7 @@ mod whitespace_searcher_tests { let matcher = WhitespaceMatcher {}; let iter = Searcher::new(&matcher, "".as_bytes()); let items: Vec<(usize, usize)> = iter.collect(); - assert_eq!(vec![] as Vec<(usize, usize)>, items); + assert!(items.is_empty()); } fn test_multispace(line: &[u8], expected: &[(usize, usize)]) { diff --git a/src/uu/date/Cargo.toml b/src/uu/date/Cargo.toml index 7590b0b746c..087d4befc7e 100644 --- a/src/uu/date/Cargo.toml +++ b/src/uu/date/Cargo.toml @@ -1,16 +1,19 @@ # spell-checker:ignore datetime [package] name = "uu_date" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "date ~ (uutils) display or set the current time" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/date" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/date.rs" @@ -18,7 +21,7 @@ path = "src/date.rs" [dependencies] chrono = { workspace = true } clap = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["custom-tz-fmt", "parser"] } parse_datetime = { workspace = true } [target.'cfg(unix)'.dependencies] diff --git a/src/uu/date/date-usage.md b/src/uu/date/date-usage.md index bf2dc469d3a..109bfd3988b 100644 --- a/src/uu/date/date-usage.md +++ b/src/uu/date/date-usage.md @@ -79,9 +79,3 @@ Show the time on the west coast of the US (use tzselect(1) to find TZ) ``` TZ='America/Los_Angeles' date ``` - -Show the local time for 9AM next Friday on the west coast of the US - -``` -date --date='TZ="America/Los_Angeles" 09:00 next Fri' -``` diff --git a/src/uu/date/src/date.rs b/src/uu/date/src/date.rs index 745fd54239c..f4c9313cb62 100644 --- a/src/uu/date/src/date.rs +++ b/src/uu/date/src/date.rs @@ -6,24 +6,24 @@ // spell-checker:ignore (chrono) Datelike Timelike ; (format) DATEFILE MMDDhhmm ; (vars) datetime datetimes use chrono::format::{Item, StrftimeItems}; -use chrono::{DateTime, Duration, FixedOffset, Local, Offset, Utc}; +use chrono::{DateTime, FixedOffset, Local, Offset, TimeDelta, Utc}; #[cfg(windows)] use chrono::{Datelike, Timelike}; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; #[cfg(all(unix, not(target_os = "macos"), not(target_os = "redox")))] -use libc::{clock_settime, timespec, CLOCK_REALTIME}; +use libc::{CLOCK_REALTIME, clock_settime, timespec}; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::PathBuf; +use uucore::custom_tz_fmt::custom_time_format; use uucore::display::Quotable; -#[cfg(not(any(target_os = "redox")))] use uucore::error::FromIo; use uucore::error::{UResult, USimpleError}; use uucore::{format_usage, help_about, help_usage, show}; #[cfg(windows)] use windows_sys::Win32::{Foundation::SYSTEMTIME, System::SystemInformation::SetSystemTime}; -use uucore::shortcut_value_parser::ShortcutValueParser; +use uucore::parser::shortcut_value_parser::ShortcutValueParser; // Options const DATE: &str = "date"; @@ -92,7 +92,8 @@ enum DateSource { Now, Custom(String), File(PathBuf), - Human(Duration), + Stdin, + Human(TimeDelta), } enum Iso8601Format { @@ -103,7 +104,7 @@ enum Iso8601Format { Ns, } -impl<'a> From<&'a str> for Iso8601Format { +impl From<&str> for Iso8601Format { fn from(s: &str) -> Self { match s { HOURS => Self::Hours, @@ -123,7 +124,7 @@ enum Rfc3339Format { Ns, } -impl<'a> From<&'a str> for Rfc3339Format { +impl From<&str> for Rfc3339Format { fn from(s: &str) -> Self { match s { DATE => Self::Date, @@ -166,13 +167,18 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { }; let date_source = if let Some(date) = matches.get_one::(OPT_DATE) { - if let Ok(duration) = parse_datetime::from_str(date.as_str()) { + let ref_time = Local::now(); + if let Ok(new_time) = parse_datetime::parse_datetime_at_date(ref_time, date.as_str()) { + let duration = new_time.signed_duration_since(ref_time); DateSource::Human(duration) } else { DateSource::Custom(date.into()) } } else if let Some(file) = matches.get_one::(OPT_FILE) { - DateSource::File(file.into()) + match file.as_ref() { + "-" => DateSource::Stdin, + _ => DateSource::File(file.into()), + } } else { DateSource::Now }; @@ -222,11 +228,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { Box::new(iter) } DateSource::Human(relative_time) => { - // Get the current DateTime for things like "1 year ago" - let current_time = DateTime::::from(Local::now()); - // double check the result is overflow or not of the current_time + relative_time + // Double check the result is overflow or not of the current_time + relative_time // it may cause a panic of chrono::datetime::DateTime add - match current_time.checked_add_signed(relative_time) { + match now.checked_add_signed(relative_time) { Some(date) => { let iter = std::iter::once(Ok(date)); Box::new(iter) @@ -234,11 +238,16 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { None => { return Err(USimpleError::new( 1, - format!("invalid date {}", relative_time), + format!("invalid date {relative_time}"), )); } } } + DateSource::Stdin => { + let lines = BufReader::new(std::io::stdin()).lines(); + let iter = lines.map_while(Result::ok).map(parse_date); + Box::new(iter) + } DateSource::File(ref path) => { if path.is_dir() { return Err(USimpleError::new( @@ -264,18 +273,10 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { for date in dates { match date { Ok(date) => { - // GNU `date` uses `%N` for nano seconds, however crate::chrono uses `%f` - let format_string = &format_string.replace("%N", "%f"); - // Refuse to pass this string to chrono as it is crashing in this crate - if format_string.contains("%#z") { - return Err(USimpleError::new( - 1, - format!("invalid format {}", format_string.replace("%f", "%N")), - )); - } + let format_string = custom_time_format(format_string); // Hack to work around panic in chrono, // TODO - remove when a fix for https://github.com/chronotope/chrono/issues/623 is released - let format_items = StrftimeItems::new(format_string); + let format_items = StrftimeItems::new(format_string.as_str()); if format_items.clone().any(|i| i == Item::Error) { return Err(USimpleError::new( 1, @@ -301,7 +302,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .infer_long_args(true) @@ -310,6 +311,7 @@ pub fn uu_app() -> Command { .short('d') .long(OPT_DATE) .value_name("STRING") + .allow_hyphen_values(true) .help("display time described by STRING, not 'now'"), ) .arg( @@ -395,7 +397,7 @@ fn make_format_string(settings: &Settings) -> &str { Rfc3339Format::Ns => "%F %T.%f%:z", }, Format::Custom(ref fmt) => fmt, - Format::Default => "%c", + Format::Default => "%a %b %e %X %Z %Y", } } @@ -403,9 +405,8 @@ fn make_format_string(settings: &Settings) -> &str { /// If it fails, return a tuple of the `String` along with its `ParseError`. fn parse_date + Clone>( s: S, -) -> Result, (String, chrono::format::ParseError)> { - // TODO: The GNU date command can parse a wide variety of inputs. - s.as_ref().parse().map_err(|e| (s.as_ref().into(), e)) +) -> Result, (String, parse_datetime::ParseDateTimeError)> { + parse_datetime::parse_datetime(s.as_ref()).map_err(|e| (s.as_ref().into(), e)) } #[cfg(not(any(unix, windows)))] diff --git a/src/uu/dd/Cargo.toml b/src/uu/dd/Cargo.toml index 482128e1d08..04f05179926 100644 --- a/src/uu/dd/Cargo.toml +++ b/src/uu/dd/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_dd" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "dd ~ (uutils) copy and convert files" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/dd" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/dd.rs" @@ -18,13 +21,17 @@ path = "src/dd.rs" clap = { workspace = true } gcd = { workspace = true } libc = { workspace = true } -uucore = { workspace = true, features = ["memo"] } - -[target.'cfg(any(target_os = "linux"))'.dependencies] -nix = { workspace = true, features = ["fs"] } +uucore = { workspace = true, features = [ + "format", + "parser", + "quoting-style", + "fs", +] } +thiserror = { workspace = true } [target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies] signal-hook = { workspace = true } +nix = { workspace = true, features = ["fs"] } [[bin]] name = "dd" diff --git a/src/uu/dd/src/blocks.rs b/src/uu/dd/src/blocks.rs index 8e5557a2c9b..b7449c98be7 100644 --- a/src/uu/dd/src/blocks.rs +++ b/src/uu/dd/src/blocks.rs @@ -133,7 +133,7 @@ mod tests { let buf = [0u8, 1u8, 2u8, 3u8]; let res = block(&buf, 4, false, &mut rs); - assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8],]); + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8]]); } #[test] @@ -144,7 +144,7 @@ mod tests { assert_eq!( res, - vec![vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE],] + vec![vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE]] ); } @@ -155,7 +155,7 @@ mod tests { let res = block(&buf, 4, false, &mut rs); // Commented section(s) should be truncated and appear for reference only. - assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8 /*, 4u8*/],]); + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8 /*, 4u8*/]]); assert_eq!(rs.records_truncated, 1); } @@ -238,7 +238,7 @@ mod tests { let buf = [0u8, 1u8, 2u8, 3u8, NEWLINE]; let res = block(&buf, 4, false, &mut rs); - assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8],]); + assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8]]); } #[test] @@ -258,7 +258,7 @@ mod tests { assert_eq!( res, - vec![vec![0u8, 1u8, 2u8, SPACE], vec![SPACE, SPACE, SPACE, SPACE],] + vec![vec![0u8, 1u8, 2u8, SPACE], vec![SPACE, SPACE, SPACE, SPACE]] ); } @@ -270,7 +270,7 @@ mod tests { assert_eq!( res, - vec![vec![SPACE, SPACE, SPACE, SPACE], vec![0u8, 1u8, 2u8, 3u8],] + vec![vec![SPACE, SPACE, SPACE, SPACE], vec![0u8, 1u8, 2u8, 3u8]] ); } @@ -315,7 +315,7 @@ mod tests { let buf = [0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8]; let res = unblock(&buf, 8); - assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, NEWLINE],); + assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, NEWLINE]); } #[test] @@ -323,7 +323,7 @@ mod tests { let buf = [SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE]; let res = unblock(&buf, 8); - assert_eq!(res, vec![NEWLINE],); + assert_eq!(res, vec![NEWLINE]); } #[test] @@ -342,7 +342,7 @@ mod tests { let buf = [0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE]; let res = unblock(&buf, 8); - assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, NEWLINE],); + assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, NEWLINE]); } #[test] diff --git a/src/uu/dd/src/bufferedoutput.rs b/src/uu/dd/src/bufferedoutput.rs new file mode 100644 index 00000000000..6ac3b430046 --- /dev/null +++ b/src/uu/dd/src/bufferedoutput.rs @@ -0,0 +1,206 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// +// spell-checker:ignore wstat towrite cdefg bufferedoutput +//! Buffer partial output blocks until they are completed. +//! +//! Use the [`BufferedOutput`] struct to create a buffered form of the +//! [`Output`] writer. +use crate::{Output, WriteStat}; + +/// Buffer partial output blocks until they are completed. +/// +/// Complete blocks are written immediately to the inner [`Output`], +/// but partial blocks are stored in an internal buffer until they are +/// completed. +pub(crate) struct BufferedOutput<'a> { + /// The unbuffered inner block writer. + inner: Output<'a>, + + /// The internal buffer that stores a partial block. + /// + /// The size of this buffer is always less than the output block + /// size (that is, the value of the `obs` command-line option). + buf: Vec, +} + +impl<'a> BufferedOutput<'a> { + /// Add partial block buffering to the given block writer. + /// + /// The internal buffer size is at most the value of `obs` as + /// defined in `inner`. + pub(crate) fn new(inner: Output<'a>) -> Self { + let obs = inner.settings.obs; + Self { + inner, + buf: Vec::with_capacity(obs), + } + } + + pub(crate) fn discard_cache(&self, offset: libc::off_t, len: libc::off_t) { + self.inner.discard_cache(offset, len); + } + + /// Flush the partial block stored in the internal buffer. + pub(crate) fn flush(&mut self) -> std::io::Result { + let wstat = self.inner.write_blocks(&self.buf)?; + let n = wstat.bytes_total.try_into().unwrap(); + self.buf.drain(0..n); + Ok(wstat) + } + + /// Synchronize the inner block writer. + pub(crate) fn sync(&mut self) -> std::io::Result<()> { + self.inner.sync() + } + + /// Truncate the underlying file to the current stream position, if possible. + pub(crate) fn truncate(&mut self) -> std::io::Result<()> { + self.inner.dst.truncate() + } + + /// Write the given bytes one block at a time. + /// + /// Only complete blocks will be written. Partial blocks will be + /// buffered until enough bytes have been provided to complete a + /// block. The returned [`WriteStat`] object will include the + /// number of blocks written during execution of this function. + pub(crate) fn write_blocks(&mut self, buf: &[u8]) -> std::io::Result { + // Split the incoming buffer into two parts: the bytes to write + // and the bytes to buffer for next time. + // + // If `buf` does not include enough bytes to form a full block, + // just buffer the whole thing and write zero blocks. + let n = self.buf.len() + buf.len(); + let rem = n % self.inner.settings.obs; + let i = buf.len().saturating_sub(rem); + let (to_write, to_buffer) = buf.split_at(i); + + // Concatenate the old partial block with the new bytes to form + // some number of complete blocks. + self.buf.extend_from_slice(to_write); + + // Write all complete blocks to the inner block writer. + // + // For example, if the output block size were 3, the buffered + // partial block were `b"ab"` and the new incoming bytes were + // `b"cdefg"`, then we would write blocks `b"abc"` and + // b`"def"` to the inner block writer. + let wstat = self.inner.write_blocks(&self.buf)?; + + // Buffer any remaining bytes as a partial block. + // + // Continuing the example above, the last byte `b"g"` would be + // buffered as a partial block until the next call to + // `write_blocks()`. + self.buf.clear(); + self.buf.extend_from_slice(to_buffer); + + Ok(wstat) + } +} + +#[cfg(unix)] +#[cfg(test)] +mod tests { + use crate::bufferedoutput::BufferedOutput; + use crate::{Dest, Output, Settings}; + + #[test] + fn test_buffered_output_write_blocks_empty() { + let settings = Settings { + obs: 3, + ..Default::default() + }; + let inner = Output { + dst: Dest::Sink, + settings: &settings, + }; + let mut output = BufferedOutput::new(inner); + let wstat = output.write_blocks(&[]).unwrap(); + assert_eq!(wstat.writes_complete, 0); + assert_eq!(wstat.writes_partial, 0); + assert_eq!(wstat.bytes_total, 0); + assert_eq!(output.buf, vec![]); + } + + #[test] + fn test_buffered_output_write_blocks_partial() { + let settings = Settings { + obs: 3, + ..Default::default() + }; + let inner = Output { + dst: Dest::Sink, + settings: &settings, + }; + let mut output = BufferedOutput::new(inner); + let wstat = output.write_blocks(b"ab").unwrap(); + assert_eq!(wstat.writes_complete, 0); + assert_eq!(wstat.writes_partial, 0); + assert_eq!(wstat.bytes_total, 0); + assert_eq!(output.buf, b"ab"); + } + + #[test] + fn test_buffered_output_write_blocks_complete() { + let settings = Settings { + obs: 3, + ..Default::default() + }; + let inner = Output { + dst: Dest::Sink, + settings: &settings, + }; + let mut output = BufferedOutput::new(inner); + let wstat = output.write_blocks(b"abcd").unwrap(); + assert_eq!(wstat.writes_complete, 1); + assert_eq!(wstat.writes_partial, 0); + assert_eq!(wstat.bytes_total, 3); + assert_eq!(output.buf, b"d"); + } + + #[test] + fn test_buffered_output_write_blocks_append() { + let settings = Settings { + obs: 3, + ..Default::default() + }; + let inner = Output { + dst: Dest::Sink, + settings: &settings, + }; + let mut output = BufferedOutput { + inner, + buf: b"ab".to_vec(), + }; + let wstat = output.write_blocks(b"cdefg").unwrap(); + assert_eq!(wstat.writes_complete, 2); + assert_eq!(wstat.writes_partial, 0); + assert_eq!(wstat.bytes_total, 6); + assert_eq!(output.buf, b"g"); + } + + #[test] + fn test_buffered_output_flush() { + let settings = Settings { + obs: 10, + ..Default::default() + }; + let inner = Output { + dst: Dest::Sink, + settings: &settings, + }; + let mut output = BufferedOutput { + inner, + buf: b"abc".to_vec(), + }; + let wstat = output.flush().unwrap(); + assert_eq!(wstat.writes_complete, 0); + assert_eq!(wstat.writes_partial, 1); + assert_eq!(wstat.bytes_total, 3); + assert_eq!(output.buf, vec![]); + } +} diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index 4ac2aa78006..4de05246f43 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -3,23 +3,27 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore fname, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, iseek, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, oseek, outfile, parseargs, rlen, rmax, rremain, rsofar, rstat, sigusr, wlen, wstat seekable oconv canonicalized fadvise Fadvise FADV DONTNEED ESPIPE +// spell-checker:ignore fname, ftype, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, iseek, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, oseek, outfile, parseargs, rlen, rmax, rremain, rsofar, rstat, sigusr, wlen, wstat seekable oconv canonicalized fadvise Fadvise FADV DONTNEED ESPIPE bufferedoutput, SETFL +mod blocks; +mod bufferedoutput; +mod conversion_tables; mod datastructures; -use datastructures::*; - +mod numbers; mod parseargs; -use parseargs::Parser; - -mod conversion_tables; - mod progress; -use progress::{gen_prog_updater, ProgUpdate, ReadStat, StatusLevel, WriteStat}; -mod blocks; +use crate::bufferedoutput::BufferedOutput; use blocks::conv_block_unblock_helper; - -mod numbers; +use datastructures::*; +#[cfg(any(target_os = "linux", target_os = "android"))] +use nix::fcntl::FcntlArg::F_SETFL; +#[cfg(any(target_os = "linux", target_os = "android"))] +use nix::fcntl::OFlag; +use parseargs::Parser; +use progress::ProgUpdateType; +use progress::{ProgUpdate, ReadStat, StatusLevel, WriteStat, gen_prog_updater}; +use uucore::io::OwnedFileDescriptorOrHandle; use std::cmp; use std::env; @@ -27,32 +31,36 @@ use std::ffi::OsString; use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Stdout, Write}; #[cfg(any(target_os = "linux", target_os = "android"))] +use std::os::fd::AsFd; +#[cfg(any(target_os = "linux", target_os = "android"))] use std::os::unix::fs::OpenOptionsExt; #[cfg(unix)] use std::os::unix::{ fs::FileTypeExt, io::{AsRawFd, FromRawFd}, }; +#[cfg(windows)] +use std::os::windows::{fs::MetadataExt, io::AsHandle}; use std::path::Path; -use std::sync::{ - atomic::{AtomicBool, Ordering::Relaxed}, - mpsc, Arc, -}; +use std::sync::atomic::AtomicU8; +use std::sync::{Arc, atomic::Ordering::Relaxed, mpsc}; use std::thread; use std::time::{Duration, Instant}; -use clap::{crate_version, Arg, Command}; +use clap::{Arg, Command}; use gcd::Gcd; #[cfg(target_os = "linux")] use nix::{ errno::Errno, - fcntl::{posix_fadvise, PosixFadviseAdvice}, + fcntl::{PosixFadviseAdvice, posix_fadvise}, }; use uucore::display::Quotable; use uucore::error::{FromIo, UResult}; -use uucore::{format_usage, help_about, help_section, help_usage, show_error}; +#[cfg(unix)] +use uucore::error::{USimpleError, set_exit_code}; #[cfg(target_os = "linux")] -use uucore::{show, show_if_err}; +use uucore::show_if_err; +use uucore::{format_usage, help_about, help_section, help_usage, show_error}; const ABOUT: &str = help_about!("dd.md"); const AFTER_HELP: &str = help_section!("after help", "dd.md"); @@ -74,42 +82,71 @@ struct Settings { oconv: OConvFlags, oflags: OFlags, status: Option, + /// Whether the output writer should buffer partial blocks until complete. + buffered: bool, } /// A timer which triggers on a given interval /// -/// After being constructed with [`Alarm::with_interval`], [`Alarm::is_triggered`] -/// will return true once per the given [`Duration`]. +/// After being constructed with [`Alarm::with_interval`], [`Alarm::get_trigger`] +/// will return [`ALARM_TRIGGER_TIMER`] once per the given [`Duration`]. +/// Alarm can be manually triggered with closure returned by [`Alarm::manual_trigger_fn`]. +/// [`Alarm::get_trigger`] will return [`ALARM_TRIGGER_SIGNAL`] in this case. /// /// Can be cloned, but the trigger status is shared across all instances so only /// the first caller each interval will yield true. /// /// When all instances are dropped the background thread will exit on the next interval. -#[derive(Debug, Clone)] pub struct Alarm { interval: Duration, - trigger: Arc, + trigger: Arc, } +pub const ALARM_TRIGGER_NONE: u8 = 0; +pub const ALARM_TRIGGER_TIMER: u8 = 1; +pub const ALARM_TRIGGER_SIGNAL: u8 = 2; + impl Alarm { + /// use to construct alarm timer with duration pub fn with_interval(interval: Duration) -> Self { - let trigger = Arc::new(AtomicBool::default()); + let trigger = Arc::new(AtomicU8::default()); let weak_trigger = Arc::downgrade(&trigger); thread::spawn(move || { while let Some(trigger) = weak_trigger.upgrade() { thread::sleep(interval); - trigger.store(true, Relaxed); + trigger.store(ALARM_TRIGGER_TIMER, Relaxed); } }); Self { interval, trigger } } - pub fn is_triggered(&self) -> bool { - self.trigger.swap(false, Relaxed) + /// Returns a closure that allows to manually trigger the alarm + /// + /// This is useful for cases where more than one alarm even source exists + /// In case of `dd` there is the SIGUSR1/SIGINFO case where we want to + /// trigger an manual progress report. + pub fn manual_trigger_fn(&self) -> Box { + let weak_trigger = Arc::downgrade(&self.trigger); + Box::new(move || { + if let Some(trigger) = weak_trigger.upgrade() { + trigger.store(ALARM_TRIGGER_SIGNAL, Relaxed); + } + }) + } + + /// Use this function to poll for any pending alarm event + /// + /// Returns `ALARM_TRIGGER_NONE` for no pending event. + /// Returns `ALARM_TRIGGER_TIMER` if the event was triggered by timer + /// Returns `ALARM_TRIGGER_SIGNAL` if the event was triggered manually + /// by the closure returned from `manual_trigger_fn` + pub fn get_trigger(&self) -> u8 { + self.trigger.swap(ALARM_TRIGGER_NONE, Relaxed) } + // Getter function for the configured interval duration pub fn get_interval(&self) -> Duration { self.interval } @@ -126,6 +163,12 @@ enum Num { Bytes(u64), } +impl Default for Num { + fn default() -> Self { + Self::Blocks(0) + } +} + impl Num { fn force_bytes_if(self, force: bool) -> Self { match self { @@ -181,7 +224,8 @@ impl Source { /// The length of the data source in number of bytes. /// /// If it cannot be determined, then this function returns 0. - fn len(&self) -> std::io::Result { + fn len(&self) -> io::Result { + #[allow(clippy::match_wildcard_for_single_variants)] match self { Self::File(f) => Ok(f.metadata()?.len().try_into().unwrap_or(i64::MAX)), _ => Ok(0), @@ -200,15 +244,26 @@ impl Source { Err(e) => Err(e), }, #[cfg(unix)] - Self::StdinFile(f) => match io::copy(&mut f.take(n), &mut io::sink()) { - Ok(m) if m < n => { - show_error!("'standard input': cannot skip to specified offset"); - Ok(m) + Self::StdinFile(f) => { + if let Ok(Some(len)) = try_get_len_of_block_device(f) { + if len < n { + // GNU compatibility: + // this case prints the stats but sets the exit code to 1 + show_error!("'standard input': cannot skip: Invalid argument"); + set_exit_code(1); + return Ok(len); + } } - Ok(m) => Ok(m), - Err(e) => Err(e), - }, - Self::File(f) => f.seek(io::SeekFrom::Start(n)), + match io::copy(&mut f.take(n), &mut io::sink()) { + Ok(m) if m < n => { + show_error!("'standard input': cannot skip to specified offset"); + Ok(m) + } + Ok(m) => Ok(m), + Err(e) => Err(e), + } + } + Self::File(f) => f.seek(SeekFrom::Current(n.try_into().unwrap())), #[cfg(unix)] Self::Fifo(f) => io::copy(&mut f.take(n), &mut io::sink()), } @@ -222,10 +277,11 @@ impl Source { /// then this function returns an error. #[cfg(target_os = "linux")] fn discard_cache(&self, offset: libc::off_t, len: libc::off_t) -> nix::Result<()> { + #[allow(clippy::match_wildcard_for_single_variants)] match self { Self::File(f) => { let advice = PosixFadviseAdvice::POSIX_FADV_DONTNEED; - posix_fadvise(f.as_raw_fd(), offset, len, advice) + posix_fadvise(f.as_fd(), offset, len, advice) } _ => Err(Errno::ESPIPE), // "Illegal seek" } @@ -264,9 +320,35 @@ impl<'a> Input<'a> { /// Instantiate this struct with stdin as a source. fn new_stdin(settings: &'a Settings) -> UResult { #[cfg(not(unix))] - let mut src = Source::Stdin(io::stdin()); + let mut src = { + let f = File::from(io::stdin().as_handle().try_clone_to_owned()?); + let is_file = if let Ok(metadata) = f.metadata() { + // this hack is needed as there is no other way on windows + // to differentiate between the case where `seek` works + // on a file handle or not. i.e. when the handle is no real + // file but a pipe, `seek` is still successful, but following + // `read`s are not affected by the seek. + metadata.creation_time() != 0 + } else { + false + }; + if is_file { + Source::File(f) + } else { + Source::Stdin(io::stdin()) + } + }; #[cfg(unix)] let mut src = Source::stdin_as_file(); + #[cfg(unix)] + if let Source::StdinFile(f) = &src { + if settings.iflags.directory && !f.metadata()?.is_dir() { + return Err(USimpleError::new( + 1, + "setting flags for 'standard input': Not a directory", + )); + } + }; if settings.skip > 0 { src.skip(settings.skip)?; } @@ -339,14 +421,10 @@ fn make_linux_iflags(iflags: &IFlags) -> Option { flag |= libc::O_SYNC; } - if flag == 0 { - None - } else { - Some(flag) - } + if flag == 0 { None } else { Some(flag) } } -impl<'a> Read for Input<'a> { +impl Read for Input<'_> { fn read(&mut self, buf: &mut [u8]) -> io::Result { let mut base_idx = 0; let target_len = buf.len(); @@ -369,7 +447,7 @@ impl<'a> Read for Input<'a> { } } -impl<'a> Input<'a> { +impl Input<'_> { /// Discard the system file cache for the given portion of the input. /// /// `offset` and `len` specify a contiguous portion of the input. @@ -377,7 +455,7 @@ impl<'a> Input<'a> { /// the input file is no longer needed. If not possible, then this /// function prints an error message to stderr and sets the exit /// status code to 1. - #[allow(unused_variables)] + #[cfg_attr(not(target_os = "linux"), allow(clippy::unused_self, unused_variables))] fn discard_cache(&self, offset: libc::off_t, len: libc::off_t) { #[cfg(target_os = "linux")] { @@ -396,7 +474,7 @@ impl<'a> Input<'a> { /// Fills a given buffer. /// Reads in increments of 'self.ibs'. /// The start of each ibs-sized read follows the previous one. - fn fill_consecutive(&mut self, buf: &mut Vec) -> std::io::Result { + fn fill_consecutive(&mut self, buf: &mut Vec) -> io::Result { let mut reads_complete = 0; let mut reads_partial = 0; let mut bytes_total = 0; @@ -427,7 +505,7 @@ impl<'a> Input<'a> { /// Fills a given buffer. /// Reads in increments of 'self.ibs'. /// The start of each ibs-sized read is aligned to multiples of ibs; remaining space is filled with the 'pad' byte. - fn fill_blocks(&mut self, buf: &mut Vec, pad: u8) -> std::io::Result { + fn fill_blocks(&mut self, buf: &mut Vec, pad: u8) -> io::Result { let mut reads_complete = 0; let mut reads_partial = 0; let mut base_idx = 0; @@ -527,7 +605,19 @@ impl Dest { fn seek(&mut self, n: u64) -> io::Result { match self { Self::Stdout(stdout) => io::copy(&mut io::repeat(0).take(n), stdout), - Self::File(f, _) => f.seek(io::SeekFrom::Start(n)), + Self::File(f, _) => { + #[cfg(unix)] + if let Ok(Some(len)) = try_get_len_of_block_device(f) { + if len < n { + // GNU compatibility: + // this case prints the stats but sets the exit code to 1 + show_error!("'standard output': cannot seek: Invalid argument"); + set_exit_code(1); + return Ok(len); + } + } + f.seek(SeekFrom::Current(n.try_into().unwrap())) + } #[cfg(unix)] Self::Fifo(f) => { // Seeking in a named pipe means *reading* from the pipe. @@ -540,6 +630,7 @@ impl Dest { /// Truncate the underlying file to the current stream position, if possible. fn truncate(&mut self) -> io::Result<()> { + #[allow(clippy::match_wildcard_for_single_variants)] match self { Self::File(f, _) => { let pos = f.stream_position()?; @@ -560,7 +651,7 @@ impl Dest { match self { Self::File(f, _) => { let advice = PosixFadviseAdvice::POSIX_FADV_DONTNEED; - posix_fadvise(f.as_raw_fd(), offset, len, advice) + posix_fadvise(f.as_fd(), offset, len, advice) } _ => Err(Errno::ESPIPE), // "Illegal seek" } @@ -569,7 +660,8 @@ impl Dest { /// The length of the data destination in number of bytes. /// /// If it cannot be determined, then this function returns 0. - fn len(&self) -> std::io::Result { + fn len(&self) -> io::Result { + #[allow(clippy::match_wildcard_for_single_variants)] match self { Self::File(f, _) => Ok(f.metadata()?.len().try_into().unwrap_or(i64::MAX)), _ => Ok(0), @@ -590,7 +682,7 @@ impl Write for Dest { .len() .try_into() .expect("Internal dd Error: Seek amount greater than signed 64-bit integer"); - f.seek(io::SeekFrom::Current(seek_amt))?; + f.seek(SeekFrom::Current(seek_amt))?; Ok(buf.len()) } Self::File(f, _) => f.write(buf), @@ -668,6 +760,11 @@ impl<'a> Output<'a> { if !settings.oconv.notrunc { dst.set_len(settings.seek).ok(); } + + Self::prepare_file(dst, settings) + } + + fn prepare_file(dst: File, settings: &'a Settings) -> UResult { let density = if settings.oconv.sparse { Density::Sparse } else { @@ -679,6 +776,24 @@ impl<'a> Output<'a> { Ok(Self { dst, settings }) } + /// Instantiate this struct with file descriptor as a destination. + /// + /// This is useful e.g. for the case when the file descriptor was + /// already opened by the system (stdout) and has a state + /// (current position) that shall be used. + fn new_file_from_stdout(settings: &'a Settings) -> UResult { + let fx = OwnedFileDescriptorOrHandle::from(io::stdout())?; + #[cfg(any(target_os = "linux", target_os = "android"))] + if let Some(libc_flags) = make_linux_oflags(&settings.oflags) { + nix::fcntl::fcntl( + fx.as_raw().as_fd(), + F_SETFL(OFlag::from_bits_retain(libc_flags)), + )?; + } + + Self::prepare_file(fx.into_file(), settings) + } + /// Instantiate this struct with the given named pipe as a destination. #[cfg(unix)] fn new_fifo(filename: &Path, settings: &'a Settings) -> UResult { @@ -715,22 +830,45 @@ impl<'a> Output<'a> { /// the output file is no longer needed. If not possible, then /// this function prints an error message to stderr and sets the /// exit status code to 1. - #[allow(unused_variables)] + #[cfg_attr(not(target_os = "linux"), allow(clippy::unused_self, unused_variables))] fn discard_cache(&self, offset: libc::off_t, len: libc::off_t) { #[cfg(target_os = "linux")] { - show_if_err!(self - .dst - .discard_cache(offset, len) - .map_err_context(|| "failed to discard cache for: 'standard output'".to_string())); + show_if_err!(self.dst.discard_cache(offset, len).map_err_context(|| { + "failed to discard cache for: 'standard output'".to_string() + })); } - #[cfg(target_os = "linux")] + #[cfg(not(target_os = "linux"))] { // TODO Is there a way to discard filesystem cache on // these other operating systems? } } + /// writes a block of data. optionally retries when first try didn't complete + /// + /// this is needed by gnu-test: tests/dd/stats.s + /// the write can be interrupted by a system signal. + /// e.g. SIGUSR1 which is send to report status + /// without retry, the data might not be fully written to destination. + fn write_block(&mut self, chunk: &[u8]) -> io::Result { + let full_len = chunk.len(); + let mut base_idx = 0; + loop { + match self.dst.write(&chunk[base_idx..]) { + Ok(wlen) => { + base_idx += wlen; + // take iflags.fullblock as oflags shall not have this option + if (base_idx >= full_len) || !self.settings.iflags.fullblock { + return Ok(base_idx); + } + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + } + } + /// Write the given bytes one block at a time. /// /// This may write partial blocks (for example, if the underlying @@ -744,7 +882,7 @@ impl<'a> Output<'a> { let mut bytes_total = 0; for chunk in buf.chunks(self.settings.obs) { - let wlen = self.dst.write(chunk)?; + let wlen = self.write_block(chunk)?; if wlen < self.settings.obs { writes_partial += 1; } else { @@ -761,7 +899,7 @@ impl<'a> Output<'a> { } /// Flush the output to disk, if configured to do so. - fn sync(&mut self) -> std::io::Result<()> { + fn sync(&mut self) -> io::Result<()> { if self.settings.oconv.fsync { self.dst.fsync() } else if self.settings.oconv.fdatasync { @@ -771,6 +909,91 @@ impl<'a> Output<'a> { Ok(()) } } + + /// Truncate the underlying file to the current stream position, if possible. + fn truncate(&mut self) -> io::Result<()> { + self.dst.truncate() + } +} + +/// The block writer either with or without partial block buffering. +enum BlockWriter<'a> { + /// Block writer with partial block buffering. + /// + /// Partial blocks are buffered until completed. + Buffered(BufferedOutput<'a>), + + /// Block writer without partial block buffering. + /// + /// Partial blocks are written immediately. + Unbuffered(Output<'a>), +} + +impl BlockWriter<'_> { + fn discard_cache(&self, offset: libc::off_t, len: libc::off_t) { + match self { + Self::Unbuffered(o) => o.discard_cache(offset, len), + Self::Buffered(o) => o.discard_cache(offset, len), + } + } + + fn flush(&mut self) -> io::Result { + match self { + Self::Unbuffered(_) => Ok(WriteStat::default()), + Self::Buffered(o) => o.flush(), + } + } + + fn sync(&mut self) -> io::Result<()> { + match self { + Self::Unbuffered(o) => o.sync(), + Self::Buffered(o) => o.sync(), + } + } + + /// Truncate the file to the final cursor location. + fn truncate(&mut self) { + // Calling `set_len()` may result in an error (for example, + // when calling it on `/dev/null`), but we don't want to + // terminate the process when that happens. Instead, we + // suppress the error by calling `Result::ok()`. This matches + // the behavior of GNU `dd` when given the command-line + // argument `of=/dev/null`. + match self { + Self::Unbuffered(o) => o.truncate().ok(), + Self::Buffered(o) => o.truncate().ok(), + }; + } + + fn write_blocks(&mut self, buf: &[u8]) -> io::Result { + match self { + Self::Unbuffered(o) => o.write_blocks(buf), + Self::Buffered(o) => o.write_blocks(buf), + } + } +} + +/// depending on the command line arguments, this function +/// informs the OS to flush/discard the caches for input and/or output file. +fn flush_caches_full_length(i: &Input, o: &Output) -> io::Result<()> { + // TODO Better error handling for overflowing `len`. + if i.settings.iflags.nocache { + let offset = 0; + #[allow(clippy::useless_conversion)] + let len = i.src.len()?.try_into().unwrap(); + i.discard_cache(offset, len); + } + // Similarly, discard the system cache for the output file. + // + // TODO Better error handling for overflowing `len`. + if i.settings.oflags.nocache { + let offset = 0; + #[allow(clippy::useless_conversion)] + let len = o.dst.len()?.try_into().unwrap(); + o.discard_cache(offset, len); + } + + Ok(()) } /// Copy the given input data to this output, consuming both. @@ -784,7 +1007,7 @@ impl<'a> Output<'a> { /// /// If there is a problem reading from the input or writing to /// this output. -fn dd_copy(mut i: Input, mut o: Output) -> std::io::Result<()> { +fn dd_copy(mut i: Input, o: Output) -> io::Result<()> { // The read and write statistics. // // These objects are counters, initialized to zero. After each @@ -821,6 +1044,9 @@ fn dd_copy(mut i: Input, mut o: Output) -> std::io::Result<()> { let (prog_tx, rx) = mpsc::channel(); let output_thread = thread::spawn(gen_prog_updater(rx, i.settings.status)); + // Whether to truncate the output file after all blocks have been written. + let truncate = !o.settings.oconv.notrunc; + // Optimization: if no blocks are to be written, then don't // bother allocating any buffers. if let Some(Num::Blocks(0) | Num::Bytes(0)) = i.settings.count { @@ -829,23 +1055,16 @@ fn dd_copy(mut i: Input, mut o: Output) -> std::io::Result<()> { // requests that we inform the system that we no longer // need the contents of the input file in a system cache. // - // TODO Better error handling for overflowing `len`. - if i.settings.iflags.nocache { - let offset = 0; - #[allow(clippy::useless_conversion)] - let len = i.src.len()?.try_into().unwrap(); - i.discard_cache(offset, len); - } - // Similarly, discard the system cache for the output file. - // - // TODO Better error handling for overflowing `len`. - if i.settings.oflags.nocache { - let offset = 0; - #[allow(clippy::useless_conversion)] - let len = o.dst.len()?.try_into().unwrap(); - o.discard_cache(offset, len); - } - return finalize(&mut o, rstat, wstat, start, &prog_tx, output_thread); + flush_caches_full_length(&i, &o)?; + return finalize( + BlockWriter::Unbuffered(o), + rstat, + wstat, + start, + &prog_tx, + output_thread, + truncate, + ); }; // Create a common buffer with a capacity of the block size. @@ -858,6 +1077,18 @@ fn dd_copy(mut i: Input, mut o: Output) -> std::io::Result<()> { // This avoids the need to query the OS monotonic clock for every block. let alarm = Alarm::with_interval(Duration::from_secs(1)); + // The signal handler spawns an own thread that waits for signals. + // When the signal is received, it calls a handler function. + // We inject a handler function that manually triggers the alarm. + #[cfg(target_os = "linux")] + let signal_handler = progress::SignalHandler::install_signal_handler(alarm.manual_trigger_fn()); + #[cfg(target_os = "linux")] + if let Err(e) = &signal_handler { + if Some(StatusLevel::None) != i.settings.status { + eprintln!("Internal dd Warning: Unable to register signal handler \n\t{e}"); + } + } + // Index in the input file where we are reading bytes and in // the output file where we are writing bytes. // @@ -865,19 +1096,29 @@ fn dd_copy(mut i: Input, mut o: Output) -> std::io::Result<()> { let mut read_offset = 0; let mut write_offset = 0; + let input_nocache = i.settings.iflags.nocache; + let output_nocache = o.settings.oflags.nocache; + + // Add partial block buffering, if needed. + let mut o = if o.settings.buffered { + BlockWriter::Buffered(BufferedOutput::new(o)) + } else { + BlockWriter::Unbuffered(o) + }; + // The main read/write loop. // // Each iteration reads blocks from the input and writes // blocks to this output. Read/write statistics are updated on // each iteration and cumulative statistics are reported to // the progress reporting thread. - while below_count_limit(&i.settings.count, &rstat, &wstat) { + while below_count_limit(i.settings.count, &rstat) { // Read a block from the input then write the block to the output. // // As an optimization, make an educated guess about the // best buffer size for reading based on the number of // blocks already read and the number of blocks remaining. - let loop_bsize = calc_loop_bsize(&i.settings.count, &rstat, &wstat, i.settings.ibs, bsize); + let loop_bsize = calc_loop_bsize(i.settings.count, &rstat, &wstat, i.settings.ibs, bsize); let rstat_update = read_helper(&mut i, &mut buf, loop_bsize)?; if rstat_update.is_empty() { break; @@ -889,7 +1130,7 @@ fn dd_copy(mut i: Input, mut o: Output) -> std::io::Result<()> { // // TODO Better error handling for overflowing `offset` and `len`. let read_len = rstat_update.bytes_total; - if i.settings.iflags.nocache { + if input_nocache { let offset = read_offset.try_into().unwrap(); let len = read_len.try_into().unwrap(); i.discard_cache(offset, len); @@ -901,7 +1142,7 @@ fn dd_copy(mut i: Input, mut o: Output) -> std::io::Result<()> { // // TODO Better error handling for overflowing `offset` and `len`. let write_len = wstat_update.bytes_total; - if o.settings.oflags.nocache { + if output_nocache { let offset = write_offset.try_into().unwrap(); let len = write_len.try_into().unwrap(); o.discard_cache(offset, len); @@ -916,45 +1157,54 @@ fn dd_copy(mut i: Input, mut o: Output) -> std::io::Result<()> { // error. rstat += rstat_update; wstat += wstat_update; - if alarm.is_triggered() { - let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), false); - prog_tx.send(prog_update).unwrap_or(()); + match alarm.get_trigger() { + ALARM_TRIGGER_NONE => {} + t @ (ALARM_TRIGGER_TIMER | ALARM_TRIGGER_SIGNAL) => { + let tp = match t { + ALARM_TRIGGER_TIMER => ProgUpdateType::Periodic, + _ => ProgUpdateType::Signal, + }; + let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), tp); + prog_tx.send(prog_update).unwrap_or(()); + } + _ => {} } } - finalize(&mut o, rstat, wstat, start, &prog_tx, output_thread) + + finalize(o, rstat, wstat, start, &prog_tx, output_thread, truncate) } /// Flush output, print final stats, and join with the progress thread. fn finalize( - output: &mut Output, + mut output: BlockWriter, rstat: ReadStat, wstat: WriteStat, start: Instant, prog_tx: &mpsc::Sender, output_thread: thread::JoinHandle, -) -> std::io::Result<()> { - // Flush the output, if configured to do so. + truncate: bool, +) -> io::Result<()> { + // Flush the output in case a partial write has been buffered but + // not yet written. + let wstat_update = output.flush()?; + + // Sync the output, if configured to do so. output.sync()?; // Truncate the file to the final cursor location. - // - // Calling `set_len()` may result in an error (for example, - // when calling it on `/dev/null`), but we don't want to - // terminate the process when that happens. Instead, we - // suppress the error by calling `Result::ok()`. This matches - // the behavior of GNU `dd` when given the command-line - // argument `of=/dev/null`. - if !output.settings.oconv.notrunc { - output.dst.truncate().ok(); + if truncate { + output.truncate(); } // Print the final read/write statistics. - let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), true); + let wstat = wstat + wstat_update; + let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), ProgUpdateType::Final); prog_tx.send(prog_update).unwrap_or(()); // Wait for the output thread to finish output_thread .join() .expect("Failed to join with the output thread."); + Ok(()) } @@ -992,11 +1242,7 @@ fn make_linux_oflags(oflags: &OFlags) -> Option { flag |= libc::O_SYNC; } - if flag == 0 { - None - } else { - Some(flag) - } + if flag == 0 { None } else { Some(flag) } } /// Read from an input (that is, a source of bytes) into the given buffer. @@ -1005,7 +1251,7 @@ fn make_linux_oflags(oflags: &OFlags) -> Option { /// `conv=swab` or `conv=block` command-line arguments. This function /// mutates the `buf` argument in-place. The returned [`ReadStat`] /// indicates how many blocks were read. -fn read_helper(i: &mut Input, buf: &mut Vec, bsize: usize) -> std::io::Result { +fn read_helper(i: &mut Input, buf: &mut Vec, bsize: usize) -> io::Result { // Local Helper Fns ------------------------------------------------- fn perform_swab(buf: &mut [u8]) { for base in (1..buf.len()).step_by(2) { @@ -1055,7 +1301,7 @@ fn calc_bsize(ibs: usize, obs: usize) -> usize { // Calculate the buffer size appropriate for this loop iteration, respecting // a count=N if present. fn calc_loop_bsize( - count: &Option, + count: Option, rstat: &ReadStat, wstat: &WriteStat, ibs: usize, @@ -1068,7 +1314,7 @@ fn calc_loop_bsize( cmp::min(ideal_bsize as u64, rremain * ibs as u64) as usize } Some(Num::Bytes(bmax)) => { - let bmax: u128 = (*bmax).try_into().unwrap(); + let bmax: u128 = bmax.into(); let bremain: u128 = bmax - wstat.bytes_total; cmp::min(ideal_bsize as u128, bremain) as usize } @@ -1078,16 +1324,10 @@ fn calc_loop_bsize( // Decide if the current progress is below a count=N limit or return // true if no such limit is set. -fn below_count_limit(count: &Option, rstat: &ReadStat, wstat: &WriteStat) -> bool { +fn below_count_limit(count: Option, rstat: &ReadStat) -> bool { match count { - Some(Num::Blocks(n)) => { - let n = *n; - rstat.reads_complete + rstat.reads_partial <= n - } - Some(Num::Bytes(n)) => { - let n = (*n).try_into().unwrap(); - wstat.bytes_total <= n - } + Some(Num::Blocks(n)) => rstat.reads_complete + rstat.reads_partial < n, + Some(Num::Bytes(n)) => rstat.bytes_total < n, None => true, } } @@ -1133,6 +1373,20 @@ fn is_stdout_redirected_to_seekable_file() -> bool { } } +/// Try to get the len if it is a block device +#[cfg(unix)] +fn try_get_len_of_block_device(file: &mut File) -> io::Result> { + let ftype = file.metadata()?.file_type(); + if !ftype.is_block_device() { + return Ok(None); + } + + // FIXME: this can be replaced by file.stream_len() when stable. + let len = file.seek(SeekFrom::End(0))?; + file.rewind()?; + Ok(Some(len)) +} + /// Decide whether the named file is a named pipe, also known as a FIFO. #[cfg(unix)] fn is_fifo(filename: &str) -> bool { @@ -1146,16 +1400,12 @@ fn is_fifo(filename: &str) -> bool { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_ignore(); - let matches = uu_app().try_get_matches_from(args)?; let settings: Settings = Parser::new().parse( - &matches + matches .get_many::(options::OPERANDS) - .unwrap_or_default() - .map(|s| s.as_ref()) - .collect::>()[..], + .unwrap_or_default(), )?; let i = match settings.infile { @@ -1168,9 +1418,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { #[cfg(unix)] Some(ref outfile) if is_fifo(outfile) => Output::new_fifo(Path::new(&outfile), &settings)?, Some(ref outfile) => Output::new_file(Path::new(&outfile), &settings)?, - None if is_stdout_redirected_to_seekable_file() => { - Output::new_file(Path::new(&stdout_canonicalized()), &settings)? - } + None if is_stdout_redirected_to_seekable_file() => Output::new_file_from_stdout(&settings)?, None => Output::new_stdout(&settings)?, }; dd_copy(i, o).map_err_context(|| "IO error".to_string()) @@ -1178,7 +1426,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .after_help(AFTER_HELP) @@ -1188,7 +1436,7 @@ pub fn uu_app() -> Command { #[cfg(test)] mod tests { - use crate::{calc_bsize, Output, Parser}; + use crate::{Output, Parser, calc_bsize}; use std::path::Path; @@ -1196,8 +1444,8 @@ mod tests { fn bsize_test_primes() { let (n, m) = (7901, 7919); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, n * m); } @@ -1206,8 +1454,8 @@ mod tests { fn bsize_test_rel_prime_obs_greater() { let (n, m) = (7 * 5119, 13 * 5119); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, 7 * 13 * 5119); } @@ -1216,8 +1464,8 @@ mod tests { fn bsize_test_rel_prime_ibs_greater() { let (n, m) = (13 * 5119, 7 * 5119); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, 7 * 13 * 5119); } @@ -1226,8 +1474,8 @@ mod tests { fn bsize_test_3fac_rel_prime() { let (n, m) = (11 * 13 * 5119, 7 * 11 * 5119); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, 7 * 11 * 13 * 5119); } @@ -1236,8 +1484,8 @@ mod tests { fn bsize_test_ibs_greater() { let (n, m) = (512 * 1024, 256 * 1024); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, n); } @@ -1246,8 +1494,8 @@ mod tests { fn bsize_test_obs_greater() { let (n, m) = (256 * 1024, 512 * 1024); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, m); } @@ -1256,8 +1504,8 @@ mod tests { fn bsize_test_bs_eq() { let (n, m) = (1024, 1024); let res = calc_bsize(n, m); - assert!(res % n == 0); - assert!(res % m == 0); + assert_eq!(res % n, 0); + assert_eq!(res % m, 0); assert_eq!(res, m); } diff --git a/src/uu/dd/src/numbers.rs b/src/uu/dd/src/numbers.rs index 2911f7e58c5..b66893d8d35 100644 --- a/src/uu/dd/src/numbers.rs +++ b/src/uu/dd/src/numbers.rs @@ -2,7 +2,8 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -/// Functions for formatting a number as a magnitude and a unit suffix. + +//! Functions for formatting a number as a magnitude and a unit suffix. /// The first ten powers of 1024. const IEC_BASES: [u128; 10] = [ @@ -82,14 +83,14 @@ pub(crate) fn to_magnitude_and_suffix(n: u128, suffix_type: SuffixType) -> Strin if quotient < 10.0 { format!("{quotient:.1} {suffix}") } else { - format!("{} {}", quotient.round(), suffix) + format!("{} {suffix}", quotient.round()) } } #[cfg(test)] mod tests { - use crate::numbers::{to_magnitude_and_suffix, SuffixType}; + use crate::numbers::{SuffixType, to_magnitude_and_suffix}; #[test] fn test_to_magnitude_and_suffix_powers_of_1024() { @@ -115,6 +116,7 @@ mod tests { } #[test] + #[allow(clippy::cognitive_complexity)] fn test_to_magnitude_and_suffix_not_powers_of_1024() { assert_eq!(to_magnitude_and_suffix(1, SuffixType::Si), "1.0 B"); assert_eq!(to_magnitude_and_suffix(999, SuffixType::Si), "999 B"); diff --git a/src/uu/dd/src/parseargs.rs b/src/uu/dd/src/parseargs.rs index 53fae1b4b40..dd9a53fd884 100644 --- a/src/uu/dd/src/parseargs.rs +++ b/src/uu/dd/src/parseargs.rs @@ -9,67 +9,68 @@ mod unit_tests; use super::{ConversionMode, IConvFlags, IFlags, Num, OConvFlags, OFlags, Settings, StatusLevel}; use crate::conversion_tables::ConversionTable; -use std::error::Error; +use thiserror::Error; use uucore::display::Quotable; use uucore::error::UError; -use uucore::parse_size::{ParseSizeError, Parser as SizeParser}; +use uucore::parser::parse_size::{ParseSizeError, Parser as SizeParser}; use uucore::show_warning; /// Parser Errors describe errors with parser input -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Error)] pub enum ParseError { + #[error("Unrecognized operand '{0}'")] UnrecognizedOperand(String), + #[error("Only one of conv=ascii conv=ebcdic or conv=ibm may be specified")] MultipleFmtTable, + #[error("Only one of conv=lcase or conv=ucase may be specified")] MultipleUCaseLCase, + #[error("Only one of conv=block or conv=unblock may be specified")] MultipleBlockUnblock, + #[error("Only one ov conv=excl or conv=nocreat may be specified")] MultipleExclNoCreate, + #[error("invalid input flag: ‘{}’\nTry '{} --help' for more information.", .0, uucore::execution_phrase())] FlagNoMatch(String), + #[error("Unrecognized conv=CONV -> {0}")] ConvFlagNoMatch(String), + #[error("invalid number: ‘{0}’")] MultiplierStringParseFailure(String), + #[error("Multiplier string would overflow on current system -> {0}")] MultiplierStringOverflow(String), + #[error("conv=block or conv=unblock specified without cbs=N")] BlockUnblockWithoutCBS, + #[error("status=LEVEL not recognized -> {0}")] StatusLevelNotRecognized(String), + #[error("feature not implemented on this system -> {0}")] Unimplemented(String), + #[error("{0}=N cannot fit into memory")] BsOutOfRange(String), + #[error("invalid number: ‘{0}’")] InvalidNumber(String), } /// Contains a temporary state during parsing of the arguments -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Default)] pub struct Parser { infile: Option, outfile: Option, - ibs: usize, - obs: usize, + /// The block size option specified on the command-line, if any. + bs: Option, + /// The input block size option specified on the command-line, if any. + ibs: Option, + /// The output block size option specified on the command-line, if any. + obs: Option, cbs: Option, skip: Num, seek: Num, count: Option, conv: ConvFlags, + /// Whether a data-transforming `conv` option has been specified. + is_conv_specified: bool, iflag: IFlags, oflag: OFlags, status: Option, } -impl Default for Parser { - fn default() -> Self { - Self { - ibs: 512, - obs: 512, - cbs: None, - infile: None, - outfile: None, - skip: Num::Blocks(0), - seek: Num::Blocks(0), - count: None, - conv: ConvFlags::default(), - iflag: IFlags::default(), - oflag: OFlags::default(), - status: None, - } - } -} - #[derive(Debug, Default, PartialEq, Eq)] pub struct ConvFlags { ascii: bool, @@ -125,13 +126,19 @@ impl Parser { Self::default() } - pub(crate) fn parse(self, operands: &[&str]) -> Result { + pub(crate) fn parse( + self, + operands: impl IntoIterator>, + ) -> Result { self.read(operands)?.validate() } - pub(crate) fn read(mut self, operands: &[&str]) -> Result { + pub(crate) fn read( + mut self, + operands: impl IntoIterator>, + ) -> Result { for operand in operands { - self.parse_operand(operand)?; + self.parse_operand(operand.as_ref())?; } Ok(self) @@ -212,15 +219,34 @@ impl Parser { fsync: conv.fsync, }; + // Input and output block sizes. + // + // The `bs` option takes precedence. If either is not + // provided, `ibs` and `obs` are each 512 bytes by default. + let (ibs, obs) = match self.bs { + None => (self.ibs.unwrap_or(512), self.obs.unwrap_or(512)), + Some(bs) => (bs, bs), + }; + + // Whether to buffer partial output blocks until they are completed. + // + // From the GNU `dd` documentation for the `bs=BYTES` option: + // + // > [...] if no data-transforming 'conv' option is specified, + // > input is copied to the output as soon as it's read, even if + // > it is smaller than the block size. + // + let buffered = self.bs.is_none() || self.is_conv_specified; + let skip = self .skip .force_bytes_if(self.iflag.skip_bytes) - .to_bytes(self.ibs as u64); + .to_bytes(ibs as u64); let seek = self .seek .force_bytes_if(self.oflag.seek_bytes) - .to_bytes(self.obs as u64); + .to_bytes(obs as u64); let count = self.count.map(|c| c.force_bytes_if(self.iflag.count_bytes)); @@ -230,8 +256,9 @@ impl Parser { count, iconv, oconv, - ibs: self.ibs, - obs: self.obs, + ibs, + obs, + buffered, infile: self.infile, outfile: self.outfile, iflags: self.iflag, @@ -244,18 +271,17 @@ impl Parser { match operand.split_once('=') { None => return Err(ParseError::UnrecognizedOperand(operand.to_string())), Some((k, v)) => match k { - "bs" => { - let bs = Self::parse_bytes(k, v)?; - self.ibs = bs; - self.obs = bs; - } + "bs" => self.bs = Some(Self::parse_bytes(k, v)?), "cbs" => self.cbs = Some(Self::parse_bytes(k, v)?), - "conv" => self.parse_conv_flags(v)?, + "conv" => { + self.is_conv_specified = true; + self.parse_conv_flags(v)?; + } "count" => self.count = Some(Self::parse_n(v)?), - "ibs" => self.ibs = Self::parse_bytes(k, v)?, + "ibs" => self.ibs = Some(Self::parse_bytes(k, v)?), "if" => self.infile = Some(v.to_string()), "iflag" => self.parse_input_flags(v)?, - "obs" => self.obs = Self::parse_bytes(k, v)?, + "obs" => self.obs = Some(Self::parse_bytes(k, v)?), "of" => self.outfile = Some(v.to_string()), "oflag" => self.parse_output_flags(v)?, "seek" | "oseek" => self.seek = Self::parse_n(v)?, @@ -269,7 +295,7 @@ impl Parser { fn parse_n(val: &str) -> Result { let n = parse_bytes_with_opt_multiplier(val)?; - Ok(if val.ends_with('B') { + Ok(if val.contains('B') { Num::Bytes(n) } else { Num::Blocks(n) @@ -390,69 +416,6 @@ impl Parser { } } -impl std::fmt::Display for ParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::UnrecognizedOperand(arg) => { - write!(f, "Unrecognized operand '{arg}'") - } - Self::MultipleFmtTable => { - write!( - f, - "Only one of conv=ascii conv=ebcdic or conv=ibm may be specified" - ) - } - Self::MultipleUCaseLCase => { - write!(f, "Only one of conv=lcase or conv=ucase may be specified") - } - Self::MultipleBlockUnblock => { - write!(f, "Only one of conv=block or conv=unblock may be specified") - } - Self::MultipleExclNoCreate => { - write!(f, "Only one ov conv=excl or conv=nocreat may be specified") - } - Self::FlagNoMatch(arg) => { - // Additional message about 'dd --help' is displayed only in this situation. - write!( - f, - "invalid input flag: ‘{}’\nTry '{} --help' for more information.", - arg, - uucore::execution_phrase() - ) - } - Self::ConvFlagNoMatch(arg) => { - write!(f, "Unrecognized conv=CONV -> {arg}") - } - Self::MultiplierStringParseFailure(arg) => { - write!(f, "Unrecognized byte multiplier -> {arg}") - } - Self::MultiplierStringOverflow(arg) => { - write!( - f, - "Multiplier string would overflow on current system -> {arg}" - ) - } - Self::BlockUnblockWithoutCBS => { - write!(f, "conv=block or conv=unblock specified without cbs=N") - } - Self::StatusLevelNotRecognized(arg) => { - write!(f, "status=LEVEL not recognized -> {arg}") - } - Self::BsOutOfRange(arg) => { - write!(f, "{arg}=N cannot fit into memory") - } - Self::Unimplemented(arg) => { - write!(f, "feature not implemented on this system -> {arg}") - } - Self::InvalidNumber(arg) => { - write!(f, "invalid number: ‘{arg}’") - } - } - } -} - -impl Error for ParseError {} - impl UError for ParseError { fn code(&self) -> i32 { 1 @@ -468,8 +431,9 @@ fn show_zero_multiplier_warning() { } /// Parse bytes using str::parse, then map error if needed. -fn parse_bytes_only(s: &str) -> Result { - s.parse() +fn parse_bytes_only(s: &str, i: usize) -> Result { + s[..i] + .parse() .map_err(|_| ParseError::MultiplierStringParseFailure(s.to_string())) } @@ -484,6 +448,8 @@ fn parse_bytes_only(s: &str) -> Result { /// 512. You can also use standard block size suffixes like `'k'` for /// 1024. /// +/// If the number would be too large, return [`u64::MAX`] instead. +/// /// # Errors /// /// If a number cannot be parsed or if the multiplication would cause @@ -501,21 +467,18 @@ fn parse_bytes_only(s: &str) -> Result { fn parse_bytes_no_x(full: &str, s: &str) -> Result { let parser = SizeParser { capital_b_bytes: true, + no_empty_numeric: true, ..Default::default() }; let (num, multiplier) = match (s.find('c'), s.rfind('w'), s.rfind('b')) { - (None, None, None) => match parser.parse(s) { + (None, None, None) => match parser.parse_u64(s) { Ok(n) => (n, 1), - Err(ParseSizeError::InvalidSuffix(_) | ParseSizeError::ParseFailure(_)) => { - return Err(ParseError::InvalidNumber(full.to_string())) - } - Err(ParseSizeError::SizeTooBig(_)) => { - return Err(ParseError::MultiplierStringOverflow(full.to_string())) - } + Err(ParseSizeError::SizeTooBig(_)) => (u64::MAX, 1), + Err(_) => return Err(ParseError::InvalidNumber(full.to_string())), }, - (Some(i), None, None) => (parse_bytes_only(&s[..i])?, 1), - (None, Some(i), None) => (parse_bytes_only(&s[..i])?, 2), - (None, None, Some(i)) => (parse_bytes_only(&s[..i])?, 512), + (Some(i), None, None) => (parse_bytes_only(s, i)?, 1), + (None, Some(i), None) => (parse_bytes_only(s, i)?, 2), + (None, None, Some(i)) => (parse_bytes_only(s, i)?, 512), _ => return Err(ParseError::MultiplierStringParseFailure(full.to_string())), }; num.checked_mul(multiplier) @@ -624,22 +587,42 @@ fn conversion_mode( #[cfg(test)] mod tests { - use crate::parseargs::parse_bytes_with_opt_multiplier; + use crate::Num; + use crate::parseargs::{Parser, parse_bytes_with_opt_multiplier}; + use std::matches; + const BIG: &str = "9999999999999999999999999999999999999999999999999999999999999"; + + #[test] + fn test_parse_bytes_with_opt_multiplier_invalid() { + assert!(parse_bytes_with_opt_multiplier("123asdf").is_err()); + } #[test] - fn test_parse_bytes_with_opt_multiplier() { + fn test_parse_bytes_with_opt_multiplier_without_x() { assert_eq!(parse_bytes_with_opt_multiplier("123").unwrap(), 123); assert_eq!(parse_bytes_with_opt_multiplier("123c").unwrap(), 123); // 123 * 1 assert_eq!(parse_bytes_with_opt_multiplier("123w").unwrap(), 123 * 2); assert_eq!(parse_bytes_with_opt_multiplier("123b").unwrap(), 123 * 512); - assert_eq!(parse_bytes_with_opt_multiplier("123x3").unwrap(), 123 * 3); assert_eq!(parse_bytes_with_opt_multiplier("123k").unwrap(), 123 * 1024); - assert_eq!(parse_bytes_with_opt_multiplier("1x2x3").unwrap(), 6); // 1 * 2 * 3 + assert_eq!(parse_bytes_with_opt_multiplier(BIG).unwrap(), u64::MAX); + } + #[test] + fn test_parse_bytes_with_opt_multiplier_with_x() { + assert_eq!(parse_bytes_with_opt_multiplier("123x3").unwrap(), 123 * 3); + assert_eq!(parse_bytes_with_opt_multiplier("1x2x3").unwrap(), 6); // 1 * 2 * 3 assert_eq!( parse_bytes_with_opt_multiplier("1wx2cx3w").unwrap(), 2 * 2 * (3 * 2) // (1 * 2) * (2 * 1) * (3 * 2) ); - assert!(parse_bytes_with_opt_multiplier("123asdf").is_err()); + } + #[test] + fn test_parse_n() { + for arg in ["1x8x4", "1c", "123b", "123w"] { + assert!(matches!(Parser::parse_n(arg), Ok(Num::Blocks(_)))); + } + for arg in ["1Bx8x4", "2Bx8", "2Bx8B", "2x8B"] { + assert!(matches!(Parser::parse_n(arg), Ok(Num::Bytes(_)))); + } } } diff --git a/src/uu/dd/src/parseargs/unit_tests.rs b/src/uu/dd/src/parseargs/unit_tests.rs index a190fd75bd2..cde0ef0cc1f 100644 --- a/src/uu/dd/src/parseargs/unit_tests.rs +++ b/src/uu/dd/src/parseargs/unit_tests.rs @@ -6,13 +6,14 @@ use super::*; +use crate::StatusLevel; use crate::conversion_tables::{ ASCII_TO_EBCDIC_UCASE_TO_LCASE, ASCII_TO_IBM, EBCDIC_TO_ASCII_LCASE_TO_UCASE, }; use crate::parseargs::Parser; -use crate::StatusLevel; #[cfg(not(any(target_os = "linux", target_os = "android")))] +#[allow(clippy::useless_vec)] #[test] fn unimplemented_flags_should_error_non_linux() { let mut succeeded = Vec::new(); @@ -28,54 +29,38 @@ fn unimplemented_flags_should_error_non_linux() { "noctty", "nofollow", ] { - let args = vec![format!("iflag={}", flag)]; - - if Parser::new() - .parse(&args.iter().map(AsRef::as_ref).collect::>()[..]) - .is_ok() - { - succeeded.push(format!("iflag={}", flag)); + let arg = format!("iflag={flag}"); + if Parser::new().parse([&arg]).is_ok() { + succeeded.push(arg); } - let args = vec![format!("oflag={}", flag)]; - - if Parser::new() - .parse(&args.iter().map(AsRef::as_ref).collect::>()[..]) - .is_ok() - { - succeeded.push(format!("iflag={}", flag)); + let arg = format!("oflag={flag}"); + if Parser::new().parse([&arg]).is_ok() { + succeeded.push(arg); } } assert!( succeeded.is_empty(), - "The following flags did not panic as expected: {:?}", - succeeded + "The following flags did not panic as expected: {succeeded:?}", ); } #[test] +#[allow(clippy::useless_vec)] fn unimplemented_flags_should_error() { let mut succeeded = Vec::new(); // The following flags are not implemented for flag in ["cio", "nolinks", "text", "binary"] { - let args = vec![format!("iflag={flag}")]; - - if Parser::new() - .parse(&args.iter().map(AsRef::as_ref).collect::>()[..]) - .is_ok() - { - succeeded.push(format!("iflag={flag}")); + let arg = format!("iflag={flag}"); + if Parser::new().parse([&arg]).is_ok() { + succeeded.push(arg); } - let args = vec![format!("oflag={flag}")]; - - if Parser::new() - .parse(&args.iter().map(AsRef::as_ref).collect::>()[..]) - .is_ok() - { - succeeded.push(format!("iflag={flag}")); + let arg = format!("oflag={flag}"); + if Parser::new().parse([&arg]).is_ok() { + succeeded.push(arg); } } @@ -87,14 +72,14 @@ fn unimplemented_flags_should_error() { #[test] fn test_status_level_absent() { - let args = &["if=foo.file", "of=bar.file"]; + let args = ["if=foo.file", "of=bar.file"]; assert_eq!(Parser::new().parse(args).unwrap().status, None); } #[test] fn test_status_level_none() { - let args = &["status=none", "if=foo.file", "of=bar.file"]; + let args = ["status=none", "if=foo.file", "of=bar.file"]; assert_eq!( Parser::new().parse(args).unwrap().status, @@ -103,8 +88,9 @@ fn test_status_level_none() { } #[test] +#[allow(clippy::cognitive_complexity)] fn test_all_top_level_args_no_leading_dashes() { - let args = &[ + let args = [ "if=foo.file", "of=bar.file", "ibs=10", @@ -154,7 +140,7 @@ fn test_all_top_level_args_no_leading_dashes() { ); // no conv flags apply to output - assert_eq!(settings.oconv, OConvFlags::default(),); + assert_eq!(settings.oconv, OConvFlags::default()); // iconv=count_bytes,skip_bytes assert_eq!( @@ -179,7 +165,7 @@ fn test_all_top_level_args_no_leading_dashes() { #[test] fn test_status_level_progress() { - let args = &["if=foo.file", "of=bar.file", "status=progress"]; + let args = ["if=foo.file", "of=bar.file", "status=progress"]; let settings = Parser::new().parse(args).unwrap(); @@ -188,7 +174,7 @@ fn test_status_level_progress() { #[test] fn test_status_level_noxfer() { - let args = &["if=foo.file", "status=noxfer", "of=bar.file"]; + let args = ["if=foo.file", "status=noxfer", "of=bar.file"]; let settings = Parser::new().parse(args).unwrap(); @@ -197,7 +183,7 @@ fn test_status_level_noxfer() { #[test] fn test_multiple_flags_options() { - let args = &[ + let args = [ "iflag=fullblock,count_bytes", "iflag=skip_bytes", "oflag=append", @@ -244,7 +230,7 @@ fn test_multiple_flags_options() { #[test] fn test_override_multiple_options() { - let args = &[ + let args = [ "if=foo.file", "if=correct.file", "of=bar.file", @@ -286,31 +272,31 @@ fn test_override_multiple_options() { #[test] fn icf_ctable_error() { - let args = &["conv=ascii,ebcdic,ibm"]; + let args = ["conv=ascii,ebcdic,ibm"]; assert!(Parser::new().parse(args).is_err()); } #[test] fn icf_case_error() { - let args = &["conv=ucase,lcase"]; + let args = ["conv=ucase,lcase"]; assert!(Parser::new().parse(args).is_err()); } #[test] fn icf_block_error() { - let args = &["conv=block,unblock"]; + let args = ["conv=block,unblock"]; assert!(Parser::new().parse(args).is_err()); } #[test] fn icf_creat_error() { - let args = &["conv=excl,nocreat"]; + let args = ["conv=excl,nocreat"]; assert!(Parser::new().parse(args).is_err()); } #[test] fn parse_icf_token_ibm() { - let args = &["conv=ibm"]; + let args = ["conv=ibm"]; let settings = Parser::new().parse(args).unwrap(); assert_eq!( @@ -324,7 +310,7 @@ fn parse_icf_token_ibm() { #[test] fn parse_icf_tokens_elu() { - let args = &["conv=ebcdic,lcase"]; + let args = ["conv=ebcdic,lcase"]; let settings = Parser::new().parse(args).unwrap(); assert_eq!( @@ -338,7 +324,9 @@ fn parse_icf_tokens_elu() { #[test] fn parse_icf_tokens_remaining() { - let args = &["conv=ascii,ucase,block,sparse,swab,sync,noerror,excl,nocreat,notrunc,noerror,fdatasync,fsync"]; + let args = [ + "conv=ascii,ucase,block,sparse,swab,sync,noerror,excl,nocreat,notrunc,noerror,fdatasync,fsync", + ]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -357,6 +345,7 @@ fn parse_icf_tokens_remaining() { fsync: true, ..Default::default() }, + is_conv_specified: true, ..Default::default() }) ); @@ -364,7 +353,7 @@ fn parse_icf_tokens_remaining() { #[test] fn parse_iflag_tokens() { - let args = &["iflag=fullblock,count_bytes,skip_bytes"]; + let args = ["iflag=fullblock,count_bytes,skip_bytes"]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -381,7 +370,7 @@ fn parse_iflag_tokens() { #[test] fn parse_oflag_tokens() { - let args = &["oflag=append,seek_bytes"]; + let args = ["oflag=append,seek_bytes"]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -398,7 +387,7 @@ fn parse_oflag_tokens() { #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn parse_iflag_tokens_linux() { - let args = &["iflag=direct,directory,dsync,sync,nonblock,noatime,noctty,nofollow"]; + let args = ["iflag=direct,directory,dsync,sync,nonblock,noatime,noctty,nofollow"]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -421,7 +410,7 @@ fn parse_iflag_tokens_linux() { #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn parse_oflag_tokens_linux() { - let args = &["oflag=direct,directory,dsync,sync,nonblock,noatime,noctty,nofollow"]; + let args = ["oflag=direct,directory,dsync,sync,nonblock,noatime,noctty,nofollow"]; assert_eq!( Parser::new().read(args), Ok(Parser { @@ -504,14 +493,6 @@ mod test_64bit_arch { ); } -#[test] -#[should_panic] -fn test_overflow_panic() { - let bs_str = format!("{}KiB", u64::MAX); - - parse_bytes_with_opt_multiplier(&bs_str).unwrap(); -} - #[test] #[should_panic] fn test_neg_panic() { diff --git a/src/uu/dd/src/progress.rs b/src/uu/dd/src/progress.rs index 4fe04cb0e67..85f5fa85af8 100644 --- a/src/uu/dd/src/progress.rs +++ b/src/uu/dd/src/progress.rs @@ -11,26 +11,26 @@ //! updater that runs in its own thread. use std::io::Write; use std::sync::mpsc; +#[cfg(target_os = "linux")] +use std::thread::JoinHandle; use std::time::Duration; -use uucore::error::UResult; -use uucore::memo::sprintf; - -use crate::numbers::{to_magnitude_and_suffix, SuffixType}; - -// On Linux, we register a signal handler that prints progress updates. #[cfg(target_os = "linux")] -use signal_hook::consts::signal; -#[cfg(target_os = "linux")] -use std::{ - env, - error::Error, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, +use signal_hook::iterator::Handle; +use uucore::{ + error::UResult, + format::num_format::{FloatVariant, Formatter}, }; +use crate::numbers::{SuffixType, to_magnitude_and_suffix}; + +#[derive(PartialEq, Eq)] +pub(crate) enum ProgUpdateType { + Periodic, + Signal, + Final, +} + /// Summary statistics for read and write progress of dd for a given duration. pub(crate) struct ProgUpdate { /// Read statistics. @@ -51,7 +51,7 @@ pub(crate) struct ProgUpdate { /// The status of the write. /// /// True if the write is completed, false if still in-progress. - pub(crate) complete: bool, + pub(crate) update_type: ProgUpdateType, } impl ProgUpdate { @@ -60,13 +60,13 @@ impl ProgUpdate { read_stat: ReadStat, write_stat: WriteStat, duration: Duration, - complete: bool, + update_type: ProgUpdateType, ) -> Self { Self { read_stat, write_stat, duration, - complete, + update_type, } } @@ -152,7 +152,14 @@ impl ProgUpdate { let (carriage_return, newline) = if rewrite { ("\r", "") } else { ("", "\n") }; // The duration should be formatted as in `printf %g`. - let duration_str = sprintf("%g", &[duration.to_string()])?; + let mut duration_str = Vec::new(); + uucore::format::num_format::Float { + variant: FloatVariant::Shortest, + ..Default::default() + } + .fmt(&mut duration_str, &duration.into())?; + // We assume that printf will output valid UTF-8 + let duration_str = std::str::from_utf8(&duration_str).unwrap(); // If the number of bytes written is sufficiently large, then // print a more concise representation of the number, like @@ -379,6 +386,17 @@ impl std::ops::AddAssign for WriteStat { } } +impl std::ops::Add for WriteStat { + type Output = Self; + fn add(self, other: Self) -> Self { + Self { + writes_complete: self.writes_complete + other.writes_complete, + writes_partial: self.writes_partial + other.writes_partial, + bytes_total: self.bytes_total + other.bytes_total, + } + } +} + /// How much detail to report when printing transfer statistics. /// /// This corresponds to the available settings of the `status` @@ -413,7 +431,7 @@ pub(crate) fn gen_prog_updater( let mut progress_printed = false; while let Ok(update) = rx.recv() { // Print the final read/write statistics. - if update.complete { + if update.update_type == ProgUpdateType::Final { update.print_final_stats(print_level, progress_printed); return; } @@ -425,6 +443,49 @@ pub(crate) fn gen_prog_updater( } } +/// signal handler listens for SIGUSR1 signal and runs provided closure. +#[cfg(target_os = "linux")] +pub(crate) struct SignalHandler { + handle: Handle, + thread: Option>, +} + +#[cfg(target_os = "linux")] +impl SignalHandler { + pub(crate) fn install_signal_handler( + f: Box, + ) -> Result { + use signal_hook::consts::signal::*; + use signal_hook::iterator::Signals; + + let mut signals = Signals::new([SIGUSR1])?; + let handle = signals.handle(); + let thread = std::thread::spawn(move || { + for signal in &mut signals { + match signal { + SIGUSR1 => (*f)(), + _ => unreachable!(), + } + } + }); + + Ok(Self { + handle, + thread: Some(thread), + }) + } +} + +#[cfg(target_os = "linux")] +impl Drop for SignalHandler { + fn drop(&mut self) { + self.handle.close(); + if let Some(thread) = std::mem::take(&mut self.thread) { + thread.join().unwrap(); + } + } +} + /// Return a closure that can be used in its own thread to print progress info. /// /// This function returns a closure that receives [`ProgUpdate`] @@ -439,50 +500,31 @@ pub(crate) fn gen_prog_updater( rx: mpsc::Receiver, print_level: Option, ) -> impl Fn() { - // TODO: SIGINFO: Trigger progress line reprint. BSD-style Linux only. - const SIGUSR1_USIZE: usize = signal::SIGUSR1 as usize; - fn posixly_correct() -> bool { - env::var("POSIXLY_CORRECT").is_ok() - } - fn register_linux_signal_handler(sigval: Arc) -> Result<(), Box> { - if !posixly_correct() { - signal_hook::flag::register_usize(signal::SIGUSR1, sigval, SIGUSR1_USIZE)?; - } - - Ok(()) - } // -------------------------------------------------------------- move || { - let sigval = Arc::new(AtomicUsize::new(0)); - - register_linux_signal_handler(sigval.clone()).unwrap_or_else(|e| { - if Some(StatusLevel::None) != print_level { - eprintln!("Internal dd Warning: Unable to register signal handler \n\t{e}"); - } - }); - // Holds the state of whether we have printed the current progress. // This is needed so that we know whether or not to print a newline // character before outputting non-progress data. let mut progress_printed = false; while let Ok(update) = rx.recv() { - // Print the final read/write statistics. - if update.complete { - update.print_final_stats(print_level, progress_printed); - return; - } - // (Re)print status line if progress is requested. - if Some(StatusLevel::Progress) == print_level && !update.complete { - update.reprint_prog_line(); - progress_printed = true; - } - // Handle signals and set the signal to un-seen. - // This will print a maximum of 1 time per second, even though it - // should be printing on every SIGUSR1. - if let SIGUSR1_USIZE = sigval.swap(0, Ordering::Relaxed) { - update.print_transfer_stats(progress_printed); - // Reset the progress printed, since print_transfer_stats always prints a newline. - progress_printed = false; + match update.update_type { + ProgUpdateType::Final => { + // Print the final read/write statistics. + update.print_final_stats(print_level, progress_printed); + return; + } + ProgUpdateType::Periodic => { + // (Re)print status line if progress is requested. + if Some(StatusLevel::Progress) == print_level { + update.reprint_prog_line(); + progress_printed = true; + } + } + ProgUpdateType::Signal => { + update.print_transfer_stats(progress_printed); + // Reset the progress printed, since print_transfer_stats always prints a newline. + progress_printed = false; + } } } } @@ -504,7 +546,7 @@ mod tests { ..Default::default() }, duration: Duration::new(1, 0), // one second - complete: false, + update_type: super::ProgUpdateType::Periodic, } } @@ -513,7 +555,7 @@ mod tests { read_stat: ReadStat::default(), write_stat: WriteStat::default(), duration, - complete: false, + update_type: super::ProgUpdateType::Periodic, } } @@ -538,12 +580,12 @@ mod tests { let read_stat = ReadStat::new(1, 2, 3, 4); let write_stat = WriteStat::new(4, 5, 6); let duration = Duration::new(789, 0); - let complete = false; + let update_type = super::ProgUpdateType::Periodic; let prog_update = ProgUpdate { read_stat, write_stat, duration, - complete, + update_type, }; let mut cursor = Cursor::new(vec![]); @@ -560,7 +602,7 @@ mod tests { read_stat: ReadStat::default(), write_stat: WriteStat::default(), duration: Duration::new(1, 0), // one second - complete: false, + update_type: super::ProgUpdateType::Periodic, }; let mut cursor = Cursor::new(vec![]); @@ -616,7 +658,7 @@ mod tests { read_stat: ReadStat::default(), write_stat: WriteStat::default(), duration: Duration::new(1, 0), // one second - complete: false, + update_type: super::ProgUpdateType::Periodic, }; let mut cursor = Cursor::new(vec![]); prog_update @@ -637,7 +679,7 @@ mod tests { read_stat: ReadStat::default(), write_stat: WriteStat::default(), duration: Duration::new(1, 0), // one second - complete: false, + update_type: super::ProgUpdateType::Periodic, }; let mut cursor = Cursor::new(vec![]); let rewrite = true; diff --git a/src/uu/df/Cargo.toml b/src/uu/df/Cargo.toml index b547a35d5ee..6585b0abc6a 100644 --- a/src/uu/df/Cargo.toml +++ b/src/uu/df/Cargo.toml @@ -1,26 +1,30 @@ [package] name = "uu_df" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "df ~ (uutils) display file system information" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/df" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/df.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true, features = ["libc", "fsext"] } +uucore = { workspace = true, features = ["libc", "fsext", "parser"] } unicode-width = { workspace = true } +thiserror = { workspace = true } [dev-dependencies] -tempfile = "3" +tempfile = { workspace = true } [[bin]] name = "df" diff --git a/src/uu/df/src/blocks.rs b/src/uu/df/src/blocks.rs index 9bc16b7820a..26b763cac1a 100644 --- a/src/uu/df/src/blocks.rs +++ b/src/uu/df/src/blocks.rs @@ -9,7 +9,7 @@ use std::{env, fmt}; use uucore::{ display::Quotable, - parse_size::{parse_size, ParseSizeError}, + parser::parse_size::{ParseSizeError, parse_size_u64}, }; /// The first ten powers of 1024. @@ -98,9 +98,9 @@ pub(crate) fn to_magnitude_and_suffix(n: u128, suffix_type: SuffixType) -> Strin if rem % (bases[i] / 10) == 0 { format!("{quot}.{tenths_place}{suffix}") } else if tenths_place + 1 == 10 || quot >= 10 { - format!("{}{}", quot + 1, suffix) + format!("{}{suffix}", quot + 1) } else { - format!("{}.{}{}", quot, tenths_place + 1, suffix) + format!("{quot}.{}{suffix}", tenths_place + 1) } } } @@ -165,7 +165,7 @@ impl Default for BlockSize { pub(crate) fn read_block_size(matches: &ArgMatches) -> Result { if matches.contains_id(OPT_BLOCKSIZE) { let s = matches.get_one::(OPT_BLOCKSIZE).unwrap(); - let bytes = parse_size(s)?; + let bytes = parse_size_u64(s)?; if bytes > 0 { Ok(BlockSize::Bytes(bytes)) @@ -184,11 +184,7 @@ pub(crate) fn read_block_size(matches: &ArgMatches) -> Result Option { for env_var in ["DF_BLOCK_SIZE", "BLOCK_SIZE", "BLOCKSIZE"] { if let Ok(env_size) = env::var(env_var) { - if let Ok(size) = parse_size(&env_size) { - return Some(size); - } else { - return None; - } + return parse_size_u64(&env_size).ok(); } } @@ -216,7 +212,7 @@ mod tests { use std::env; - use crate::blocks::{to_magnitude_and_suffix, BlockSize, SuffixType}; + use crate::blocks::{BlockSize, SuffixType, to_magnitude_and_suffix}; #[test] fn test_to_magnitude_and_suffix_powers_of_1024() { @@ -239,6 +235,7 @@ mod tests { } #[test] + #[allow(clippy::cognitive_complexity)] fn test_to_magnitude_and_suffix_not_powers_of_1024() { assert_eq!(to_magnitude_and_suffix(1, SuffixType::Si), "1B"); assert_eq!(to_magnitude_and_suffix(999, SuffixType::Si), "999B"); @@ -293,8 +290,8 @@ mod tests { #[test] fn test_default_block_size() { assert_eq!(BlockSize::Bytes(1024), BlockSize::default()); - env::set_var("POSIXLY_CORRECT", "1"); + unsafe { env::set_var("POSIXLY_CORRECT", "1") }; assert_eq!(BlockSize::Bytes(512), BlockSize::default()); - env::remove_var("POSIXLY_CORRECT"); + unsafe { env::remove_var("POSIXLY_CORRECT") }; } } diff --git a/src/uu/df/src/columns.rs b/src/uu/df/src/columns.rs index 0659d7f7da7..0d2d121a3d1 100644 --- a/src/uu/df/src/columns.rs +++ b/src/uu/df/src/columns.rs @@ -4,7 +4,9 @@ // file that was distributed with this source code. // spell-checker:ignore itotal iused iavail ipcent pcent squashfs use crate::{OPT_INODES, OPT_OUTPUT, OPT_PRINT_TYPE}; -use clap::{parser::ValueSource, ArgMatches}; +use clap::{ArgMatches, parser::ValueSource}; +use thiserror::Error; +use uucore::display::Quotable; /// The columns in the output table produced by `df`. /// @@ -56,9 +58,10 @@ pub(crate) enum Column { } /// An error while defining which columns to display in the output table. -#[derive(Debug)] +#[derive(Debug, Error)] pub(crate) enum ColumnError { /// If a column appears more than once in the `--output` argument. + #[error("{}", .0.quote())] MultipleColumns(String), } @@ -201,6 +204,7 @@ impl Column { // 14 = length of "Filesystem" plus 4 spaces Self::Source => 14, Self::Used => 5, + Self::Size => 5, // the shortest headers have a length of 4 chars so we use that as the minimum width _ => 4, } diff --git a/src/uu/df/src/df.rs b/src/uu/df/src/df.rs index 78325f3d2ad..9e2bb6920af 100644 --- a/src/uu/df/src/df.rs +++ b/src/uu/df/src/df.rs @@ -12,22 +12,21 @@ use blocks::HumanReadable; use clap::builder::ValueParser; use table::HeaderMode; use uucore::display::Quotable; -use uucore::error::FromIo; -use uucore::error::{UError, UResult, USimpleError}; -use uucore::fsext::{read_fs_list, MountInfo}; -use uucore::parse_size::ParseSizeError; +use uucore::error::{UError, UResult, USimpleError, get_exit_code}; +use uucore::fsext::{MountInfo, read_fs_list}; +use uucore::parser::parse_size::ParseSizeError; use uucore::{format_usage, help_about, help_section, help_usage, show}; -use clap::{crate_version, parser::ValueSource, Arg, ArgAction, ArgMatches, Command}; +use clap::{Arg, ArgAction, ArgMatches, Command, parser::ValueSource}; -use std::error::Error; use std::ffi::OsString; -use std::fmt; use std::path::Path; +use thiserror::Error; -use crate::blocks::{read_block_size, BlockSize}; +use crate::blocks::{BlockSize, read_block_size}; use crate::columns::{Column, ColumnError}; use crate::filesystem::Filesystem; +use crate::filesystem::FsError; use crate::table::Table; const ABOUT: &str = help_about!("df.md"); @@ -114,52 +113,32 @@ impl Default for Options { } } -#[derive(Debug)] +#[derive(Debug, Error)] enum OptionsError { + // TODO This needs to vary based on whether `--block-size` + // or `-B` were provided. + #[error("--block-size argument '{0}' too large")] BlockSizeTooLarge(String), + // TODO This needs to vary based on whether `--block-size` + // or `-B` were provided., + #[error("invalid --block-size argument {0}")] InvalidBlockSize(String), + // TODO This needs to vary based on whether `--block-size` + // or `-B` were provided. + #[error("invalid suffix in --block-size argument {0}")] InvalidSuffix(String), /// An error getting the columns to display in the output table. + #[error("option --output: field {0} used more than once")] ColumnError(ColumnError), + #[error("{}", .0.iter() + .map(|t| format!("file system type {} both selected and excluded", t.quote())) + .collect::>() + .join(format!("\n{}: ", uucore::util_name()).as_str()))] FilesystemTypeBothSelectedAndExcluded(Vec), } -impl fmt::Display for OptionsError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - // TODO This needs to vary based on whether `--block-size` - // or `-B` were provided. - Self::BlockSizeTooLarge(s) => { - write!(f, "--block-size argument {} too large", s.quote()) - } - // TODO This needs to vary based on whether `--block-size` - // or `-B` were provided. - Self::InvalidBlockSize(s) => write!(f, "invalid --block-size argument {s}"), - // TODO This needs to vary based on whether `--block-size` - // or `-B` were provided. - Self::InvalidSuffix(s) => write!(f, "invalid suffix in --block-size argument {s}"), - Self::ColumnError(ColumnError::MultipleColumns(s)) => write!( - f, - "option --output: field {} used more than once", - s.quote() - ), - #[allow(clippy::print_in_format_impl)] - Self::FilesystemTypeBothSelectedAndExcluded(types) => { - for t in types { - eprintln!( - "{}: file system type {} both selected and excluded", - uucore::util_name(), - t.quote() - ); - } - Ok(()) - } - } - } -} - impl Options { /// Convert command-line arguments into [`Options`]. fn from(matches: &ArgMatches) -> Result { @@ -189,6 +168,7 @@ impl Options { .to_string(), ), ParseSizeError::ParseFailure(s) => OptionsError::InvalidBlockSize(s), + ParseSizeError::PhysicalMem(s) => OptionsError::InvalidBlockSize(s), })?, header_mode: { if matches.get_flag(OPT_HUMAN_READABLE_BINARY) @@ -242,7 +222,10 @@ fn is_included(mi: &MountInfo, opt: &Options) -> bool { } // Don't show pseudo filesystems unless `--all` has been given. - if mi.dummy && !opt.show_all_fs { + // The "lofs" filesystem is a loopback + // filesystem present on Solaris and FreeBSD systems. It + // is similar to a symbolic link. + if (mi.dummy || mi.fs_type == "lofs") && !opt.show_all_fs { return false; } @@ -305,38 +288,14 @@ fn is_best(previous: &[MountInfo], mi: &MountInfo) -> bool { true } -/// Keep only the specified subset of [`MountInfo`] instances. -/// -/// The `opt` argument specifies a variety of ways of excluding -/// [`MountInfo`] instances; see [`Options`] for more information. -/// -/// Finally, if there are duplicate entries, the one with the shorter -/// path is kept. - -fn filter_mount_list(vmi: Vec, opt: &Options) -> Vec { - let mut result = vec![]; - for mi in vmi { - // TODO The running time of the `is_best()` function is linear - // in the length of `result`. That makes the running time of - // this loop quadratic in the length of `vmi`. This could be - // improved by a more efficient implementation of `is_best()`, - // but `vmi` is probably not very long in practice. - if is_included(&mi, opt) && is_best(&result, &mi) { - result.push(mi); - } - } - result -} - /// Get all currently mounted filesystems. /// /// `opt` excludes certain filesystems from consideration and allows for the synchronization of filesystems before running; see /// [`Options`] for more information. - -fn get_all_filesystems(opt: &Options) -> Result, std::io::Error> { +fn get_all_filesystems(opt: &Options) -> UResult> { // Run a sync call before any operation if so instructed. if opt.sync { - #[cfg(not(windows))] + #[cfg(not(any(windows, target_os = "redox")))] unsafe { #[cfg(not(target_os = "android"))] uucore::libc::sync(); @@ -345,88 +304,102 @@ fn get_all_filesystems(opt: &Options) -> Result, std::io::Error> } } - // The list of all mounted filesystems. - // - // Filesystems excluded by the command-line options are - // not considered. - let mounts: Vec = filter_mount_list(read_fs_list()?, opt); + let mut mounts = vec![]; + for mi in read_fs_list()? { + // TODO The running time of the `is_best()` function is linear + // in the length of `result`. That makes the running time of + // this loop quadratic in the length of `vmi`. This could be + // improved by a more efficient implementation of `is_best()`, + // but `vmi` is probably not very long in practice. + if is_included(&mi, opt) && is_best(&mounts, &mi) { + mounts.push(mi); + } + } // Convert each `MountInfo` into a `Filesystem`, which contains // both the mount information and usage information. - Ok(mounts - .into_iter() - .filter_map(|m| Filesystem::new(m, None)) - .filter(|fs| opt.show_all_fs || fs.usage.blocks > 0) - .collect()) + #[cfg(not(windows))] + { + let maybe_mount = |m| Filesystem::from_mount(&mounts, &m, None).ok(); + Ok(mounts + .clone() + .into_iter() + .filter_map(maybe_mount) + .filter(|fs| opt.show_all_fs || fs.usage.blocks > 0) + .collect()) + } + #[cfg(windows)] + { + let maybe_mount = |m| Filesystem::from_mount(&m, None).ok(); + Ok(mounts + .into_iter() + .filter_map(maybe_mount) + .filter(|fs| opt.show_all_fs || fs.usage.blocks > 0) + .collect()) + } } /// For each path, get the filesystem that contains that path. -fn get_named_filesystems

(paths: &[P], opt: &Options) -> Result, std::io::Error> +fn get_named_filesystems

(paths: &[P], opt: &Options) -> UResult> where P: AsRef, { // The list of all mounted filesystems. - // - // Filesystems marked as `dummy` or of type "lofs" are not - // considered. The "lofs" filesystem is a loopback - // filesystem present on Solaris and FreeBSD systems. It - // is similar to a symbolic link. - let mounts: Vec = filter_mount_list(read_fs_list()?, opt) - .into_iter() - .filter(|mi| mi.fs_type != "lofs" && !mi.dummy) - .collect(); + let mounts: Vec = read_fs_list()?; let mut result = vec![]; - // this happens if the file system type doesn't exist - if mounts.is_empty() { - show!(USimpleError::new(1, "no file systems processed")); - return Ok(result); - } - // Convert each path into a `Filesystem`, which contains // both the mount information and usage information. for path in paths { match Filesystem::from_path(&mounts, path) { - Some(fs) => result.push(fs), - None => { - // this happens if specified file system type != file system type of the file - if path.as_ref().exists() { - show!(USimpleError::new(1, "no file systems processed")); - } else { - show!(USimpleError::new( - 1, - format!("{}: No such file or directory", path.as_ref().display()) - )); + Ok(fs) => { + if is_included(&fs.mount_info, opt) { + result.push(fs); } } + Err(FsError::InvalidPath) => { + show!(USimpleError::new( + 1, + format!("{}: No such file or directory", path.as_ref().display()) + )); + } + Err(FsError::MountMissing) => { + show!(USimpleError::new(1, "no file systems processed")); + } + #[cfg(not(windows))] + Err(FsError::OverMounted) => { + show!(USimpleError::new( + 1, + format!( + "cannot access {}: over-mounted by another device", + path.as_ref().quote() + ) + )); + } } } + if get_exit_code() == 0 && result.is_empty() { + show!(USimpleError::new(1, "no file systems processed")); + return Ok(result); + } + Ok(result) } -#[derive(Debug)] +#[derive(Debug, Error)] enum DfError { /// A problem while parsing command-line options. + #[error("{}", .0)] OptionsError(OptionsError), } -impl Error for DfError {} - impl UError for DfError { fn usage(&self) -> bool { matches!(self, Self::OptionsError(OptionsError::ColumnError(_))) } } -impl fmt::Display for DfError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::OptionsError(e) => e.fmt(f), - } - } -} - #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let matches = uu_app().try_get_matches_from(args)?; @@ -443,8 +416,10 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { // Get the list of filesystems to display in the output table. let filesystems: Vec = match matches.get_many::(OPT_PATHS) { None => { - let filesystems = get_all_filesystems(&opt) - .map_err_context(|| "cannot read table of mounted file systems".into())?; + let filesystems = get_all_filesystems(&opt).map_err(|e| { + let context = "cannot read table of mounted file systems"; + USimpleError::new(e.code(), format!("{context}: {e}")) + })?; if filesystems.is_empty() { return Err(USimpleError::new(1, "no file systems processed")); @@ -454,8 +429,10 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } Some(paths) => { let paths: Vec<_> = paths.collect(); - let filesystems = get_named_filesystems(&paths, &opt) - .map_err_context(|| "cannot read table of mounted file systems".into())?; + let filesystems = get_named_filesystems(&paths, &opt).map_err(|e| { + let context = "cannot read table of mounted file systems"; + USimpleError::new(e.code(), format!("{context}: {e}")) + })?; // This can happen if paths are given as command-line arguments // but none of the paths exist. @@ -474,7 +451,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) .after_help(AFTER_HELP) @@ -724,7 +701,7 @@ mod tests { mod is_included { - use crate::{is_included, Options}; + use crate::{Options, is_included}; use uucore::fsext::MountInfo; /// Instantiate a [`MountInfo`] with the given fields. @@ -858,16 +835,4 @@ mod tests { assert!(is_included(&m, &opt)); } } - - mod filter_mount_list { - - use crate::{filter_mount_list, Options}; - - #[test] - fn test_empty() { - let opt = Options::default(); - let mount_infos = vec![]; - assert!(filter_mount_list(mount_infos, &opt).is_empty()); - } - } } diff --git a/src/uu/df/src/filesystem.rs b/src/uu/df/src/filesystem.rs index ef5107958a2..43b1deb36c2 100644 --- a/src/uu/df/src/filesystem.rs +++ b/src/uu/df/src/filesystem.rs @@ -37,6 +37,33 @@ pub(crate) struct Filesystem { pub usage: FsUsage, } +#[derive(Debug, PartialEq)] +pub(crate) enum FsError { + #[cfg(not(windows))] + OverMounted, + InvalidPath, + MountMissing, +} + +/// Check whether `mount` has been over-mounted. +/// +/// `mount` is considered over-mounted if it there is an element in +/// `mounts` after mount that has the same mount_dir. +#[cfg(not(windows))] +fn is_over_mounted(mounts: &[MountInfo], mount: &MountInfo) -> bool { + let last_mount_for_dir = mounts + .iter() + .filter(|m| m.mount_dir == mount.mount_dir) + .next_back(); + + if let Some(lmi) = last_mount_for_dir { + lmi.dev_name != mount.dev_name + } else { + // Should be unreachable if `mount` is in `mounts` + false + } +} + /// Find the mount info that best matches a given filesystem path. /// /// This function returns the element of `mounts` on which `path` is @@ -56,14 +83,16 @@ fn mount_info_from_path

( path: P, // This is really only used for testing purposes. canonicalize: bool, -) -> Option<&MountInfo> +) -> Result<&MountInfo, FsError> where P: AsRef, { // TODO Refactor this function with `Stater::find_mount_point()` // in the `stat` crate. let path = if canonicalize { - path.as_ref().canonicalize().ok()? + path.as_ref() + .canonicalize() + .map_err(|_| FsError::InvalidPath)? } else { path.as_ref().to_path_buf() }; @@ -82,12 +111,14 @@ where .find(|m| m.1.eq(&path)) .map(|m| m.0); - maybe_mount_point.or_else(|| { - mounts - .iter() - .filter(|mi| path.starts_with(&mi.mount_dir)) - .max_by_key(|mi| mi.mount_dir.len()) - }) + maybe_mount_point + .or_else(|| { + mounts + .iter() + .filter(|mi| path.starts_with(&mi.mount_dir)) + .max_by_key(|mi| mi.mount_dir.len()) + }) + .ok_or(FsError::MountMissing) } impl Filesystem { @@ -109,14 +140,35 @@ impl Filesystem { #[cfg(unix)] let usage = FsUsage::new(statfs(_stat_path).ok()?); #[cfg(windows)] - let usage = FsUsage::new(Path::new(&_stat_path)); + let usage = FsUsage::new(Path::new(&_stat_path)).ok()?; Some(Self { + file, mount_info, usage, - file, }) } + /// Find and create the filesystem from the given mount + /// after checking that the it hasn't been over-mounted + #[cfg(not(windows))] + pub(crate) fn from_mount( + mounts: &[MountInfo], + mount: &MountInfo, + file: Option, + ) -> Result { + if is_over_mounted(mounts, mount) { + Err(FsError::OverMounted) + } else { + Self::new(mount.clone(), file).ok_or(FsError::MountMissing) + } + } + + /// Find and create the filesystem from the given mount. + #[cfg(windows)] + pub(crate) fn from_mount(mount: &MountInfo, file: Option) -> Result { + Self::new(mount.clone(), file).ok_or(FsError::MountMissing) + } + /// Find and create the filesystem that best matches a given path. /// /// This function returns a new `Filesystem` derived from the @@ -133,16 +185,18 @@ impl Filesystem { /// * [`Path::canonicalize`] /// * [`MountInfo::mount_dir`] /// - pub(crate) fn from_path

(mounts: &[MountInfo], path: P) -> Option + pub(crate) fn from_path

(mounts: &[MountInfo], path: P) -> Result where P: AsRef, { let file = path.as_ref().display().to_string(); let canonicalize = true; - let mount_info = mount_info_from_path(mounts, path, canonicalize)?; - // TODO Make it so that we do not need to clone the `mount_info`. - let mount_info = (*mount_info).clone(); - Self::new(mount_info, Some(file)) + + let result = mount_info_from_path(mounts, path, canonicalize); + #[cfg(windows)] + return result.and_then(|mount_info| Self::from_mount(mount_info, Some(file))); + #[cfg(not(windows))] + return result.and_then(|mount_info| Self::from_mount(mounts, mount_info, Some(file))); } } @@ -153,7 +207,7 @@ mod tests { use uucore::fsext::MountInfo; - use crate::filesystem::mount_info_from_path; + use crate::filesystem::{FsError, mount_info_from_path}; // Create a fake `MountInfo` with the given directory name. fn mount_info(mount_dir: &str) -> MountInfo { @@ -183,7 +237,19 @@ mod tests { #[test] fn test_empty_mounts() { - assert!(mount_info_from_path(&[], "/", false).is_none()); + assert_eq!( + mount_info_from_path(&[], "/", false).unwrap_err(), + FsError::MountMissing + ); + } + + #[test] + fn test_bad_path() { + assert_eq!( + // This path better not exist.... + mount_info_from_path(&[], "/non-existent-path", true).unwrap_err(), + FsError::InvalidPath + ); } #[test] @@ -210,16 +276,25 @@ mod tests { #[test] fn test_no_match() { let mounts = [mount_info("/foo")]; - assert!(mount_info_from_path(&mounts, "/bar", false).is_none()); + assert_eq!( + mount_info_from_path(&mounts, "/bar", false).unwrap_err(), + FsError::MountMissing + ); } #[test] fn test_partial_match() { let mounts = [mount_info("/foo/bar")]; - assert!(mount_info_from_path(&mounts, "/foo/baz", false).is_none()); + assert_eq!( + mount_info_from_path(&mounts, "/foo/baz", false).unwrap_err(), + FsError::MountMissing + ); } #[test] + // clippy::assigning_clones added with Rust 1.78 + // Rust version = 1.76 on OpenBSD stable/7.5 + #[cfg_attr(not(target_os = "openbsd"), allow(clippy::assigning_clones))] fn test_dev_name_match() { let tmp = tempfile::TempDir::new().expect("Failed to create temp dir"); let dev_name = std::fs::canonicalize(tmp.path()) @@ -234,4 +309,52 @@ mod tests { assert!(mount_info_eq(actual, &mounts[0])); } } + + #[cfg(not(windows))] + mod over_mount { + use crate::filesystem::{Filesystem, FsError, is_over_mounted}; + use uucore::fsext::MountInfo; + + fn mount_info_with_dev_name(mount_dir: &str, dev_name: Option<&str>) -> MountInfo { + MountInfo { + dev_id: String::default(), + dev_name: dev_name.map(String::from).unwrap_or_default(), + fs_type: String::default(), + mount_dir: String::from(mount_dir), + mount_option: String::default(), + mount_root: String::default(), + remote: Default::default(), + dummy: Default::default(), + } + } + + #[test] + fn test_over_mount() { + let mount_info1 = mount_info_with_dev_name("/foo", Some("dev_name_1")); + let mount_info2 = mount_info_with_dev_name("/foo", Some("dev_name_2")); + let mounts = [mount_info1, mount_info2]; + assert!(is_over_mounted(&mounts, &mounts[0])); + } + + #[test] + fn test_over_mount_not_over_mounted() { + let mount_info1 = mount_info_with_dev_name("/foo", Some("dev_name_1")); + let mount_info2 = mount_info_with_dev_name("/foo", Some("dev_name_2")); + let mounts = [mount_info1, mount_info2]; + assert!(!is_over_mounted(&mounts, &mounts[1])); + } + + #[test] + fn test_from_mount_over_mounted() { + let mount_info1 = mount_info_with_dev_name("/foo", Some("dev_name_1")); + let mount_info2 = mount_info_with_dev_name("/foo", Some("dev_name_2")); + + let mounts = [mount_info1, mount_info2]; + + assert_eq!( + Filesystem::from_mount(&mounts, &mounts[0], None).unwrap_err(), + FsError::OverMounted + ); + } + } } diff --git a/src/uu/df/src/table.rs b/src/uu/df/src/table.rs index 4c3d08f4533..be7eb8557f9 100644 --- a/src/uu/df/src/table.rs +++ b/src/uu/df/src/table.rs @@ -2,14 +2,14 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore tmpfs Pcent Itotal Iused Iavail Ipcent +// spell-checker:ignore tmpfs Pcent Itotal Iused Iavail Ipcent nosuid nodev //! The filesystem usage data table. //! //! A table ([`Table`]) comprises a header row ([`Header`]) and a //! collection of data rows ([`Row`]), one per filesystem. use unicode_width::UnicodeWidthStr; -use crate::blocks::{to_magnitude_and_suffix, SuffixType}; +use crate::blocks::{SuffixType, to_magnitude_and_suffix}; use crate::columns::{Alignment, Column}; use crate::filesystem::Filesystem; use crate::{BlockSize, Options}; @@ -58,13 +58,13 @@ pub(crate) struct Row { bytes_capacity: Option, /// Total number of inodes in the filesystem. - inodes: u64, + inodes: u128, /// Number of used inodes. - inodes_used: u64, + inodes_used: u128, /// Number of free inodes. - inodes_free: u64, + inodes_free: u128, /// Percentage of inodes that are used, given as a float between 0 and 1. /// @@ -152,8 +152,10 @@ impl From for Row { ffree, .. } = fs.usage; - let bused = blocks - bfree; - let fused = files - ffree; + + // On Windows WSL, files can be less than ffree. Protect such cases via saturating_sub. + let bused = blocks.saturating_sub(bfree); + let fused = files.saturating_sub(ffree); Self { file: fs.file, fs_device: dev_name, @@ -176,9 +178,9 @@ impl From for Row { } else { Some(bavail as f64 / ((bused + bavail) as f64)) }, - inodes: files, - inodes_used: fused, - inodes_free: ffree, + inodes: files as u128, + inodes_used: fused as u128, + inodes_free: ffree as u128, inodes_usage: if files == 0 { None } else { @@ -233,9 +235,9 @@ impl<'a> RowFormatter<'a> { /// Get a string giving the scaled version of the input number. /// /// The scaling factor is defined in the `options` field. - fn scaled_inodes(&self, size: u64) -> String { + fn scaled_inodes(&self, size: u128) -> String { if let Some(h) = self.options.human_readable { - to_magnitude_and_suffix(size.into(), SuffixType::HumanReadable(h)) + to_magnitude_and_suffix(size, SuffixType::HumanReadable(h)) } else { size.to_string() } @@ -393,12 +395,6 @@ impl Table { let values = fmt.get_values(); total += row; - for (i, value) in values.iter().enumerate() { - if UnicodeWidthStr::width(value.as_str()) > widths[i] { - widths[i] = UnicodeWidthStr::width(value.as_str()); - } - } - rows.push(values); } } @@ -408,6 +404,16 @@ impl Table { rows.push(total_row.get_values()); } + // extend the column widths (in chars) for long values in rows + // do it here, after total row was added to the list of rows + for row in &rows { + for (i, value) in row.iter().enumerate() { + if UnicodeWidthStr::width(value.as_str()) > widths[i] { + widths[i] = UnicodeWidthStr::width(value.as_str()); + } + } + } + Self { rows, widths, @@ -464,9 +470,11 @@ impl fmt::Display for Table { #[cfg(test)] mod tests { + use std::vec; + use crate::blocks::HumanReadable; use crate::columns::Column; - use crate::table::{Header, HeaderMode, Row, RowFormatter}; + use crate::table::{Header, HeaderMode, Row, RowFormatter, Table}; use crate::{BlockSize, Options}; const COLUMNS_WITH_FS_TYPE: [Column; 7] = [ @@ -815,4 +823,120 @@ mod tests { assert_eq!(get_formatted_values(1000, 1000, 0), vec!("1", "1", "0")); assert_eq!(get_formatted_values(1001, 1000, 1), vec!("2", "1", "1")); } + + #[test] + fn test_row_converter_with_invalid_numbers() { + // copy from wsl linux + let d = crate::Filesystem { + file: None, + mount_info: crate::MountInfo { + dev_id: "28".to_string(), + dev_name: "none".to_string(), + fs_type: "9p".to_string(), + mount_dir: "/usr/lib/wsl/drivers".to_string(), + mount_option: "ro,nosuid,nodev,noatime".to_string(), + mount_root: "/".to_string(), + remote: false, + dummy: false, + }, + usage: crate::table::FsUsage { + blocksize: 4096, + blocks: 244_029_695, + bfree: 125_085_030, + bavail: 125_085_030, + bavail_top_bit_set: false, + files: 999, + ffree: 1_000_000, + }, + }; + + let row = Row::from(d); + + assert_eq!(row.inodes_used, 0); + } + + #[test] + fn test_table_column_width_computation_include_total_row() { + let d1 = crate::Filesystem { + file: None, + mount_info: crate::MountInfo { + dev_id: "28".to_string(), + dev_name: "none".to_string(), + fs_type: "9p".to_string(), + mount_dir: "/usr/lib/wsl/drivers".to_string(), + mount_option: "ro,nosuid,nodev,noatime".to_string(), + mount_root: "/".to_string(), + remote: false, + dummy: false, + }, + usage: crate::table::FsUsage { + blocksize: 4096, + blocks: 244_029_695, + bfree: 125_085_030, + bavail: 125_085_030, + bavail_top_bit_set: false, + files: 99_999_999_999, + ffree: 999_999, + }, + }; + + let filesystems = vec![d1.clone(), d1]; + + let mut options = Options { + show_total: true, + columns: vec![ + Column::Source, + Column::Itotal, + Column::Iused, + Column::Iavail, + ], + ..Default::default() + }; + + let table_w_total = Table::new(&options, filesystems.clone()); + assert_eq!( + table_w_total.to_string(), + "Filesystem Inodes IUsed IFree\n\ + none 99999999999 99999000000 999999\n\ + none 99999999999 99999000000 999999\n\ + total 199999999998 199998000000 1999998" + ); + + options.show_total = false; + + let table_w_o_total = Table::new(&options, filesystems); + assert_eq!( + table_w_o_total.to_string(), + "Filesystem Inodes IUsed IFree\n\ + none 99999999999 99999000000 999999\n\ + none 99999999999 99999000000 999999" + ); + } + + #[test] + fn test_row_accumulation_u64_overflow() { + let total = u64::MAX as u128; + let used1 = 3000u128; + let used2 = 50000u128; + + let mut row1 = Row { + inodes: total, + inodes_used: used1, + inodes_free: total - used1, + ..Default::default() + }; + + let row2 = Row { + inodes: total, + inodes_used: used2, + inodes_free: total - used2, + ..Default::default() + }; + + row1 += row2; + + assert_eq!(row1.inodes, total * 2); + assert_eq!(row1.inodes_used, used1 + used2); + assert_eq!(row1.inodes_free, total * 2 - used1 - used2); + } } diff --git a/src/uu/dir/Cargo.toml b/src/uu/dir/Cargo.toml index 9cc5fbde4ca..9bbec793b40 100644 --- a/src/uu/dir/Cargo.toml +++ b/src/uu/dir/Cargo.toml @@ -1,22 +1,25 @@ [package] name = "uu_dir" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "shortcut to ls -C -b" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/ls" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/dir.rs" [dependencies] clap = { workspace = true, features = ["env"] } -uucore = { workspace = true, features = ["entries", "fs"] } +uucore = { workspace = true, features = ["entries", "fs", "quoting-style"] } uu_ls = { workspace = true } [[bin]] diff --git a/src/uu/dir/src/dir.rs b/src/uu/dir/src/dir.rs index e255295119a..0a8a71c228a 100644 --- a/src/uu/dir/src/dir.rs +++ b/src/uu/dir/src/dir.rs @@ -6,7 +6,7 @@ use clap::Command; use std::ffi::OsString; use std::path::Path; -use uu_ls::{options, Config, Format}; +use uu_ls::{Config, Format, options}; use uucore::error::UResult; use uucore::quoting_style::{Quotes, QuotingStyle}; diff --git a/src/uu/dircolors/Cargo.toml b/src/uu/dircolors/Cargo.toml index 9248f5ea12f..5403cd1b40f 100644 --- a/src/uu/dircolors/Cargo.toml +++ b/src/uu/dircolors/Cargo.toml @@ -1,22 +1,25 @@ [package] name = "uu_dircolors" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "dircolors ~ (uutils) display commands to set LS_COLORS" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/dircolors" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/dircolors.rs" [dependencies] clap = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["colors", "parser"] } [[bin]] name = "dircolors" diff --git a/src/uu/dircolors/README.md b/src/uu/dircolors/README.md index ce8aa965f93..62944d4907b 100644 --- a/src/uu/dircolors/README.md +++ b/src/uu/dircolors/README.md @@ -15,4 +15,4 @@ Run the tests: cargo test --features "dircolors" --no-default-features ``` -Edit `/PATH_TO_COREUTILS/src/uu/dircolors/src/colors.rs` until the tests pass. +Edit `/PATH_TO_COREUTILS/src/uu/dircolors/src/dircolors.rs` until the tests pass. diff --git a/src/uu/dircolors/src/colors.rs b/src/uu/dircolors/src/colors.rs deleted file mode 100644 index c0a981db89c..00000000000 --- a/src/uu/dircolors/src/colors.rs +++ /dev/null @@ -1,225 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. -// spell-checker:ignore (ToDO) EIGHTBIT ETERM MULTIHARDLINK cpio dtterm jfbterm konsole kterm mlterm rmvb rxvt stat'able svgz tmux webm xspf COLORTERM tzst avif tzst mjpg mjpeg webp dpkg rpmnew rpmorig rpmsave - -pub const INTERNAL_DB: &str = r#"# Configuration file for dircolors, a utility to help you set the -# LS_COLORS environment variable used by GNU ls with the --color option. -# Copyright (C) 1996-2022 Free Software Foundation, Inc. -# Copying and distribution of this file, with or without modification, -# are permitted provided the copyright notice and this notice are preserved. -# The keywords COLOR, OPTIONS, and EIGHTBIT (honored by the -# slackware version of dircolors) are recognized but ignored. -# Global config options can be specified before TERM or COLORTERM entries -# Below are TERM or COLORTERM entries, which can be glob patterns, which -# restrict following config to systems with matching environment variables. -COLORTERM ?* -TERM Eterm -TERM ansi -TERM *color* -TERM con[0-9]*x[0-9]* -TERM cons25 -TERM console -TERM cygwin -TERM *direct* -TERM dtterm -TERM gnome -TERM hurd -TERM jfbterm -TERM konsole -TERM kterm -TERM linux -TERM linux-c -TERM mlterm -TERM putty -TERM rxvt* -TERM screen* -TERM st -TERM terminator -TERM tmux* -TERM vt100 -TERM xterm* -# Below are the color init strings for the basic file types. -# One can use codes for 256 or more colors supported by modern terminals. -# The default color codes use the capabilities of an 8 color terminal -# with some additional attributes as per the following codes: -# Attribute codes: -# 00=none 01=bold 04=underscore 05=blink 07=reverse 08=concealed -# Text color codes: -# 30=black 31=red 32=green 33=yellow 34=blue 35=magenta 36=cyan 37=white -# Background color codes: -# 40=black 41=red 42=green 43=yellow 44=blue 45=magenta 46=cyan 47=white -#NORMAL 00 # no color code at all -#FILE 00 # regular file: use no color at all -RESET 0 # reset to "normal" color -DIR 01;34 # directory -LINK 01;36 # symbolic link. (If you set this to 'target' instead of a - # numerical value, the color is as for the file pointed to.) -MULTIHARDLINK 00 # regular file with more than one link -FIFO 40;33 # pipe -SOCK 01;35 # socket -DOOR 01;35 # door -BLK 40;33;01 # block device driver -CHR 40;33;01 # character device driver -ORPHAN 40;31;01 # symlink to nonexistent file, or non-stat'able file ... -MISSING 00 # ... and the files they point to -SETUID 37;41 # file that is setuid (u+s) -SETGID 30;43 # file that is setgid (g+s) -CAPABILITY 00 # file with capability (very expensive to lookup) -STICKY_OTHER_WRITABLE 30;42 # dir that is sticky and other-writable (+t,o+w) -OTHER_WRITABLE 34;42 # dir that is other-writable (o+w) and not sticky -STICKY 37;44 # dir with the sticky bit set (+t) and not other-writable -# This is for files with execute permission: -EXEC 01;32 -# List any file extensions like '.gz' or '.tar' that you would like ls -# to color below. Put the extension, a space, and the color init string. -# (and any comments you want to add after a '#') -# If you use DOS-style suffixes, you may want to uncomment the following: -#.cmd 01;32 # executables (bright green) -#.exe 01;32 -#.com 01;32 -#.btm 01;32 -#.bat 01;32 -# Or if you want to color scripts even if they do not have the -# executable bit actually set. -#.sh 01;32 -#.csh 01;32 - # archives or compressed (bright red) -.tar 01;31 -.tgz 01;31 -.arc 01;31 -.arj 01;31 -.taz 01;31 -.lha 01;31 -.lz4 01;31 -.lzh 01;31 -.lzma 01;31 -.tlz 01;31 -.txz 01;31 -.tzo 01;31 -.t7z 01;31 -.zip 01;31 -.z 01;31 -.dz 01;31 -.gz 01;31 -.lrz 01;31 -.lz 01;31 -.lzo 01;31 -.xz 01;31 -.zst 01;31 -.tzst 01;31 -.bz2 01;31 -.bz 01;31 -.tbz 01;31 -.tbz2 01;31 -.tz 01;31 -.deb 01;31 -.rpm 01;31 -.jar 01;31 -.war 01;31 -.ear 01;31 -.sar 01;31 -.rar 01;31 -.alz 01;31 -.ace 01;31 -.zoo 01;31 -.cpio 01;31 -.7z 01;31 -.rz 01;31 -.cab 01;31 -.wim 01;31 -.swm 01;31 -.dwm 01;31 -.esd 01;31 -# image formats -.avif 01;35 -.jpg 01;35 -.jpeg 01;35 -.mjpg 01;35 -.mjpeg 01;35 -.gif 01;35 -.bmp 01;35 -.pbm 01;35 -.pgm 01;35 -.ppm 01;35 -.tga 01;35 -.xbm 01;35 -.xpm 01;35 -.tif 01;35 -.tiff 01;35 -.png 01;35 -.svg 01;35 -.svgz 01;35 -.mng 01;35 -.pcx 01;35 -.mov 01;35 -.mpg 01;35 -.mpeg 01;35 -.m2v 01;35 -.mkv 01;35 -.webm 01;35 -.webp 01;35 -.ogm 01;35 -.mp4 01;35 -.m4v 01;35 -.mp4v 01;35 -.vob 01;35 -.qt 01;35 -.nuv 01;35 -.wmv 01;35 -.asf 01;35 -.rm 01;35 -.rmvb 01;35 -.flc 01;35 -.avi 01;35 -.fli 01;35 -.flv 01;35 -.gl 01;35 -.dl 01;35 -.xcf 01;35 -.xwd 01;35 -.yuv 01;35 -.cgm 01;35 -.emf 01;35 -# https://wiki.xiph.org/MIME_Types_and_File_Extensions -.ogv 01;35 -.ogx 01;35 -# audio formats -.aac 00;36 -.au 00;36 -.flac 00;36 -.m4a 00;36 -.mid 00;36 -.midi 00;36 -.mka 00;36 -.mp3 00;36 -.mpc 00;36 -.ogg 00;36 -.ra 00;36 -.wav 00;36 -# https://wiki.xiph.org/MIME_Types_and_File_Extensions -.oga 00;36 -.opus 00;36 -.spx 00;36 -.xspf 00;36 -# backup files -*~ 00;90 -*# 00;90 -.bak 00;90 -.old 00;90 -.orig 00;90 -.part 00;90 -.rej 00;90 -.swp 00;90 -.tmp 00;90 -.dpkg-dist 00;90 -.dpkg-old 00;90 -.ucf-dist 00;90 -.ucf-new 00;90 -.ucf-old 00;90 -.rpmnew 00;90 -.rpmorig 00;90 -.rpmsave 00;90 -# Subsequent TERM or COLORTERM entries, can be used to add / override -# config specific to those matching environment variables."#; diff --git a/src/uu/dircolors/src/dircolors.rs b/src/uu/dircolors/src/dircolors.rs index 2e3087d810b..4fb9228eb5f 100644 --- a/src/uu/dircolors/src/dircolors.rs +++ b/src/uu/dircolors/src/dircolors.rs @@ -3,18 +3,20 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) clrtoeol dircolors eightbit endcode fnmatch leftcode multihardlink rightcode setenv sgid suid colorterm +// spell-checker:ignore (ToDO) clrtoeol dircolors eightbit endcode fnmatch leftcode multihardlink rightcode setenv sgid suid colorterm disp use std::borrow::Borrow; use std::env; +use std::fmt::Write as _; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::Path; -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; +use uucore::colors::{FILE_ATTRIBUTE_CODES, FILE_COLORS, FILE_TYPES, TERMS}; use uucore::display::Quotable; use uucore::error::{UResult, USimpleError, UUsageError}; -use uucore::{help_about, help_section, help_usage}; +use uucore::{format_usage, help_about, help_section, help_usage, parser::parse_glob}; mod options { pub const BOURNE_SHELL: &str = "bourne-shell"; @@ -28,9 +30,6 @@ const USAGE: &str = help_usage!("dircolors.md"); const ABOUT: &str = help_about!("dircolors.md"); const AFTER_HELP: &str = help_section!("after help", "dircolors.md"); -mod colors; -use self::colors::INTERNAL_DB; - #[derive(PartialEq, Eq, Debug)] pub enum OutputFmt { Shell, @@ -57,10 +56,72 @@ pub fn guess_syntax() -> OutputFmt { } } +fn get_colors_format_strings(fmt: &OutputFmt) -> (String, String) { + let prefix = match fmt { + OutputFmt::Shell => "LS_COLORS='".to_string(), + OutputFmt::CShell => "setenv LS_COLORS '".to_string(), + OutputFmt::Display => String::new(), + OutputFmt::Unknown => unreachable!(), + }; + + let suffix = match fmt { + OutputFmt::Shell => "';\nexport LS_COLORS".to_string(), + OutputFmt::CShell => "'".to_string(), + OutputFmt::Display => String::new(), + OutputFmt::Unknown => unreachable!(), + }; + + (prefix, suffix) +} + +pub fn generate_type_output(fmt: &OutputFmt) -> String { + match fmt { + OutputFmt::Display => FILE_TYPES + .iter() + .map(|&(_, key, val)| format!("\x1b[{val}m{key}\t{val}\x1b[0m")) + .collect::>() + .join("\n"), + _ => { + // Existing logic for other formats + FILE_TYPES + .iter() + .map(|&(_, v1, v2)| format!("{v1}={v2}")) + .collect::>() + .join(":") + } + } +} + +fn generate_ls_colors(fmt: &OutputFmt, sep: &str) -> String { + match fmt { + OutputFmt::Display => { + let mut display_parts = vec![]; + let type_output = generate_type_output(fmt); + display_parts.push(type_output); + for &(extension, code) in FILE_COLORS { + let prefix = if extension.starts_with('*') { "" } else { "*" }; + let formatted_extension = format!("\x1b[{code}m{prefix}{extension}\t{code}\x1b[0m"); + display_parts.push(formatted_extension); + } + display_parts.join("\n") + } + _ => { + // existing logic for other formats + let mut parts = vec![]; + for &(extension, code) in FILE_COLORS { + let prefix = if extension.starts_with('*') { "" } else { "*" }; + let formatted_extension = format!("{prefix}{extension}"); + parts.push(format!("{formatted_extension}={code}")); + } + let (prefix, suffix) = get_colors_format_strings(fmt); + let ls_colors = parts.join(sep); + format!("{prefix}{}:{ls_colors}:{suffix}", generate_type_output(fmt),) + } + } +} + #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_ignore(); - let matches = uu_app().try_get_matches_from(args)?; let files = matches @@ -97,7 +158,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { ), )); } - println!("{INTERNAL_DB}"); + + println!("{}", generate_dircolors_config()); return Ok(()); } @@ -125,7 +187,20 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let result; if files.is_empty() { - result = parse(INTERNAL_DB.lines(), &out_format, ""); + println!("{}", generate_ls_colors(&out_format, ":")); + return Ok(()); + /* + // Check if data is being piped into the program + if std::io::stdin().is_terminal() { + // No data piped, use default behavior + println!("{}", generate_ls_colors(&out_format, ":")); + return Ok(()); + } else { + // Data is piped, process the input from stdin + let fin = BufReader::new(std::io::stdin()); + result = parse(fin.lines().map_while(Result::ok), &out_format, "-"); + } + */ } else if files.len() > 1 { return Err(UUsageError::new( 1, @@ -133,6 +208,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { )); } else if files[0].eq("-") { let fin = BufReader::new(std::io::stdin()); + // For example, for echo "owt 40;33"|dircolors -b - result = parse(fin.lines().map_while(Result::ok), &out_format, files[0]); } else { let path = Path::new(files[0]); @@ -152,10 +228,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { ); } Err(e) => { - return Err(USimpleError::new( - 1, - format!("{}: {}", path.maybe_quote(), e), - )); + return Err(USimpleError::new(1, format!("{}: {e}", path.maybe_quote()))); } } } @@ -171,10 +244,11 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) - .version(crate_version!()) + .version(uucore::crate_version!()) .about(ABOUT) .after_help(AFTER_HELP) .override_usage(format_usage(USAGE)) + .args_override_self(true) .infer_long_args(true) .arg( Arg::new(options::BOURNE_SHELL) @@ -276,69 +350,24 @@ enum ParseState { Pass, } -use std::collections::HashMap; -use uucore::{format_usage, parse_glob}; - -#[allow(clippy::cognitive_complexity)] -fn parse(lines: T, fmt: &OutputFmt, fp: &str) -> Result +fn parse(user_input: T, fmt: &OutputFmt, fp: &str) -> Result where T: IntoIterator, T::Item: Borrow, { - // 1790 > $(dircolors | wc -m) let mut result = String::with_capacity(1790); - match fmt { - OutputFmt::Shell => result.push_str("LS_COLORS='"), - OutputFmt::CShell => result.push_str("setenv LS_COLORS '"), - OutputFmt::Display => (), - OutputFmt::Unknown => unreachable!(), - } + let (prefix, suffix) = get_colors_format_strings(fmt); - let mut table: HashMap<&str, &str> = HashMap::with_capacity(48); - table.insert("normal", "no"); - table.insert("norm", "no"); - table.insert("file", "fi"); - table.insert("reset", "rs"); - table.insert("dir", "di"); - table.insert("lnk", "ln"); - table.insert("link", "ln"); - table.insert("symlink", "ln"); - table.insert("orphan", "or"); - table.insert("missing", "mi"); - table.insert("fifo", "pi"); - table.insert("pipe", "pi"); - table.insert("sock", "so"); - table.insert("blk", "bd"); - table.insert("block", "bd"); - table.insert("chr", "cd"); - table.insert("char", "cd"); - table.insert("door", "do"); - table.insert("exec", "ex"); - table.insert("left", "lc"); - table.insert("leftcode", "lc"); - table.insert("right", "rc"); - table.insert("rightcode", "rc"); - table.insert("end", "ec"); - table.insert("endcode", "ec"); - table.insert("suid", "su"); - table.insert("setuid", "su"); - table.insert("sgid", "sg"); - table.insert("setgid", "sg"); - table.insert("sticky", "st"); - table.insert("other_writable", "ow"); - table.insert("owr", "ow"); - table.insert("sticky_other_writable", "tw"); - table.insert("owt", "tw"); - table.insert("capability", "ca"); - table.insert("multihardlink", "mh"); - table.insert("clrtoeol", "cl"); + result.push_str(&prefix); + // Get environment variables once at the start let term = env::var("TERM").unwrap_or_else(|_| "none".to_owned()); - let term = term.as_str(); + let colorterm = env::var("COLORTERM").unwrap_or_default(); let mut state = ParseState::Global; + let mut saw_colorterm_match = false; - for (num, line) in lines.into_iter().enumerate() { + for (num, line) in user_input.into_iter().enumerate() { let num = num + 1; let line = line.borrow().purify(); if line.is_empty() { @@ -350,71 +379,97 @@ where let (key, val) = line.split_two(); if val.is_empty() { return Err(format!( - "{}:{}: invalid line; missing second token", + // The double space is what GNU is doing + "{}:{num}: invalid line; missing second token", fp.maybe_quote(), - num )); } - let lower = key.to_lowercase(); - if lower == "term" || lower == "colorterm" { - if term.fnmatch(val) { - state = ParseState::Matched; - } else if state != ParseState::Matched { - state = ParseState::Pass; - } - } else { - if state == ParseState::Matched { - // prevent subsequent mismatched TERM from - // cancelling the input - state = ParseState::Continue; + let lower = key.to_lowercase(); + match lower.as_str() { + "term" => { + if term.fnmatch(val) { + state = ParseState::Matched; + } else if state == ParseState::Global { + state = ParseState::Pass; + } } - if state != ParseState::Pass { - if key.starts_with('.') { - if *fmt == OutputFmt::Display { - result.push_str(format!("\x1b[{val}m*{key}\t{val}\x1b[0m\n").as_str()); - } else { - result.push_str(format!("*{key}={val}:").as_str()); - } - } else if key.starts_with('*') { - if *fmt == OutputFmt::Display { - result.push_str(format!("\x1b[{val}m{key}\t{val}\x1b[0m\n").as_str()); - } else { - result.push_str(format!("{key}={val}:").as_str()); - } - } else if lower == "options" || lower == "color" || lower == "eightbit" { - // Slackware only. Ignore - } else if let Some(s) = table.get(lower.as_str()) { - if *fmt == OutputFmt::Display { - result.push_str(format!("\x1b[{val}m{s}\t{val}\x1b[0m\n").as_str()); - } else { - result.push_str(format!("{s}={val}:").as_str()); - } + "colorterm" => { + // For COLORTERM ?*, only match if COLORTERM is non-empty + let matches = if val == "?*" { + !colorterm.is_empty() } else { - return Err(format!( - "{}:{}: unrecognized keyword {}", - fp.maybe_quote(), - num, - key - )); + colorterm.fnmatch(val) + }; + if matches { + state = ParseState::Matched; + saw_colorterm_match = true; + } else if !saw_colorterm_match && state == ParseState::Global { + state = ParseState::Pass; + } + } + _ => { + if state == ParseState::Matched { + // prevent subsequent mismatched TERM from + // cancelling the input + state = ParseState::Continue; + } + if state != ParseState::Pass { + append_entry(&mut result, fmt, key, &lower, val)?; } } } } - match fmt { - OutputFmt::Shell => result.push_str("';\nexport LS_COLORS"), - OutputFmt::CShell => result.push('\''), - OutputFmt::Display => { - // remove latest "\n" - result.pop(); - } - OutputFmt::Unknown => unreachable!(), + if fmt == &OutputFmt::Display { + // remove latest "\n" + result.pop(); } + result.push_str(&suffix); Ok(result) } +fn append_entry( + result: &mut String, + fmt: &OutputFmt, + key: &str, + lower: &str, + val: &str, +) -> Result<(), String> { + if key.starts_with(['.', '*']) { + let entry = if key.starts_with('.') { + format!("*{key}") + } else { + key.to_string() + }; + let disp = if *fmt == OutputFmt::Display { + format!("\x1b[{val}m{entry}\t{val}\x1b[0m\n") + } else { + format!("{entry}={val}:") + }; + result.push_str(&disp); + return Ok(()); + } + + match lower { + "options" | "color" | "eightbit" => Ok(()), // Slackware only, ignore + _ => { + if let Some((_, s)) = FILE_ATTRIBUTE_CODES.iter().find(|&&(key, _)| key == lower) { + let disp = if *fmt == OutputFmt::Display { + format!("\x1b[{val}m{s}\t{val}\x1b[0m\n") + } else { + format!("{s}={val}:") + }; + result.push_str(&disp); + Ok(()) + } else { + Err(format!("unrecognized keyword {key}")) + } + } + } +} + /// Escape single quotes because they are not allowed between single quotes in shell code, and code /// enclosed by single quotes is what is returned by `parse()`. /// @@ -428,7 +483,7 @@ fn escape(s: &str) -> String { match c { '\'' => result.push_str("'\\''"), ':' if previous != '\\' => result.push_str("\\:"), - _ => result.push_str(&c.to_string()), + _ => result.push(c), } previous = c; } @@ -436,6 +491,58 @@ fn escape(s: &str) -> String { result } +pub fn generate_dircolors_config() -> String { + let mut config = String::new(); + + config.push_str( + "\ + # Configuration file for dircolors, a utility to help you set the\n\ + # LS_COLORS environment variable used by GNU ls with the --color option.\n\ + # The keywords COLOR, OPTIONS, and EIGHTBIT (honored by the\n\ + # slackware version of dircolors) are recognized but ignored.\n\ + # Global config options can be specified before TERM or COLORTERM entries\n\ + # Below are TERM or COLORTERM entries, which can be glob patterns, which\n\ + # restrict following config to systems with matching environment variables.\n\ + ", + ); + config.push_str("COLORTERM ?*\n"); + for term in TERMS { + let _ = writeln!(config, "TERM {term}"); + } + + config.push_str( + "\ + # Below are the color init strings for the basic file types.\n\ + # One can use codes for 256 or more colors supported by modern terminals.\n\ + # The default color codes use the capabilities of an 8 color terminal\n\ + # with some additional attributes as per the following codes:\n\ + # Attribute codes:\n\ + # 00=none 01=bold 04=underscore 05=blink 07=reverse 08=concealed\n\ + # Text color codes:\n\ + # 30=black 31=red 32=green 33=yellow 34=blue 35=magenta 36=cyan 37=white\n\ + # Background color codes:\n\ + # 40=black 41=red 42=green 43=yellow 44=blue 45=magenta 46=cyan 47=white\n\ + #NORMAL 00 # no color code at all\n\ + #FILE 00 # regular file: use no color at all\n\ + ", + ); + + for (name, _, code) in FILE_TYPES { + let _ = writeln!(config, "{name} {code}"); + } + + config.push_str("# List any file extensions like '.gz' or '.tar' that you would like ls\n"); + config.push_str("# to color below. Put the extension, a space, and the color init string.\n"); + + for (ext, color) in FILE_COLORS { + let _ = writeln!(config, "{ext} {color}"); + } + config.push_str("# Subsequent TERM or COLORTERM entries, can be used to add / override\n"); + config.push_str("# config specific to those matching environment variables."); + + config +} + #[cfg(test)] mod tests { use super::escape; diff --git a/src/uu/dirname/Cargo.toml b/src/uu/dirname/Cargo.toml index 6de444499db..7e505a37b9f 100644 --- a/src/uu/dirname/Cargo.toml +++ b/src/uu/dirname/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_dirname" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "dirname ~ (uutils) display parent directory of PATHNAME" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/dirname" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/dirname.rs" diff --git a/src/uu/dirname/src/dirname.rs b/src/uu/dirname/src/dirname.rs index 9a56e9f5bd6..de8740f8970 100644 --- a/src/uu/dirname/src/dirname.rs +++ b/src/uu/dirname/src/dirname.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -use clap::{crate_version, Arg, ArgAction, Command}; +use clap::{Arg, ArgAction, Command}; use std::path::Path; use uucore::display::print_verbatim; use uucore::error::{UResult, UUsageError}; @@ -21,8 +21,6 @@ mod options { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let args = args.collect_lossy(); - let matches = uu_app().after_help(AFTER_HELP).try_get_matches_from(args)?; let line_ending = LineEnding::from_zero_flag(matches.get_flag(options::ZERO)); @@ -30,7 +28,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let dirnames: Vec = matches .get_many::(options::DIR) .unwrap_or_default() - .map(|s| s.to_owned()) + .cloned() .collect(); if dirnames.is_empty() { @@ -64,8 +62,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app() -> Command { Command::new(uucore::util_name()) .about(ABOUT) - .version(crate_version!()) + .version(uucore::crate_version!()) .override_usage(format_usage(USAGE)) + .args_override_self(true) .infer_long_args(true) .arg( Arg::new(options::ZERO) diff --git a/src/uu/du/Cargo.toml b/src/uu/du/Cargo.toml index c87595db6ca..5b0d3f5e8ea 100644 --- a/src/uu/du/Cargo.toml +++ b/src/uu/du/Cargo.toml @@ -1,15 +1,18 @@ [package] name = "uu_du" -version = "0.0.21" -authors = ["uutils developers"] -license = "MIT" description = "du ~ (uutils) display disk usage" - -homepage = "https://github.com/uutils/coreutils" repository = "https://github.com/uutils/coreutils/tree/main/src/uu/du" -keywords = ["coreutils", "uutils", "cross-platform", "cli", "utility"] -categories = ["command-line-utilities"] -edition = "2021" +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true +edition.workspace = true +readme.workspace = true + +[lints] +workspace = true [lib] path = "src/du.rs" @@ -19,7 +22,8 @@ chrono = { workspace = true } # For the --exclude & --exclude-from options glob = { workspace = true } clap = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["format", "parser"] } +thiserror = { workspace = true } [target.'cfg(target_os = "windows")'.dependencies] windows-sys = { workspace = true, features = [ diff --git a/src/uu/du/src/du.rs b/src/uu/du/src/du.rs index 5be2a8a2b34..0b268888136 100644 --- a/src/uu/du/src/du.rs +++ b/src/uu/du/src/du.rs @@ -3,47 +3,40 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -use chrono::prelude::DateTime; -use chrono::Local; -use clap::ArgAction; -use clap::{crate_version, Arg, ArgMatches, Command}; +use chrono::{DateTime, Local}; +use clap::{Arg, ArgAction, ArgMatches, Command, builder::PossibleValue}; use glob::Pattern; use std::collections::HashSet; use std::env; -use std::fs; -use std::fs::File; #[cfg(not(windows))] use std::fs::Metadata; -use std::io::BufRead; -use std::io::BufReader; -use std::io::Result; -use std::iter; +use std::fs::{self, DirEntry, File}; +use std::io::{BufRead, BufReader}; #[cfg(not(windows))] use std::os::unix::fs::MetadataExt; #[cfg(windows)] use std::os::windows::fs::MetadataExt; #[cfg(windows)] use std::os::windows::io::AsRawHandle; -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::str::FromStr; +use std::sync::mpsc; +use std::thread; use std::time::{Duration, UNIX_EPOCH}; -use std::{error::Error, fmt::Display}; -use uucore::display::{print_verbatim, Quotable}; -use uucore::error::FromIo; -use uucore::error::{set_exit_code, UError, UResult}; +use thiserror::Error; +use uucore::display::{Quotable, print_verbatim}; +use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code}; use uucore::line_ending::LineEnding; -use uucore::parse_glob; -use uucore::parse_size::{parse_size, ParseSizeError}; -use uucore::{ - crash, format_usage, help_about, help_section, help_usage, show, show_error, show_warning, -}; +use uucore::parser::parse_glob; +use uucore::parser::parse_size::{ParseSizeError, parse_size_u64}; +use uucore::parser::shortcut_value_parser::ShortcutValueParser; +use uucore::{format_usage, help_about, help_section, help_usage, show, show_error, show_warning}; #[cfg(windows)] use windows_sys::Win32::Foundation::HANDLE; #[cfg(windows)] use windows_sys::Win32::Storage::FileSystem::{ - FileIdInfo, FileStandardInfo, GetFileInformationByHandleEx, FILE_ID_128, FILE_ID_INFO, - FILE_STANDARD_INFO, + FILE_ID_128, FILE_ID_INFO, FILE_STANDARD_INFO, FileIdInfo, FileStandardInfo, + GetFileInformationByHandleEx, }; mod options { @@ -68,9 +61,11 @@ mod options { pub const ONE_FILE_SYSTEM: &str = "one-file-system"; pub const DEREFERENCE: &str = "dereference"; pub const DEREFERENCE_ARGS: &str = "dereference-args"; + pub const NO_DEREFERENCE: &str = "no-dereference"; pub const INODES: &str = "inodes"; pub const EXCLUDE: &str = "exclude"; pub const EXCLUDE_FROM: &str = "exclude-from"; + pub const FILES0_FROM: &str = "files0-from"; pub const VERBOSE: &str = "verbose"; pub const FILE: &str = "FILE"; } @@ -79,27 +74,50 @@ const ABOUT: &str = help_about!("du.md"); const AFTER_HELP: &str = help_section!("after help", "du.md"); const USAGE: &str = help_usage!("du.md"); -// TODO: Support Z & Y (currently limited by size of u64) -const UNITS: [(char, u32); 6] = [('E', 6), ('P', 5), ('T', 4), ('G', 3), ('M', 2), ('K', 1)]; - -struct Options { +struct TraversalOptions { all: bool, - max_depth: Option, - total: bool, separate_dirs: bool, one_file_system: bool, dereference: Deref, - inodes: bool, + count_links: bool, verbose: bool, + excludes: Vec, } -#[derive(PartialEq)] +struct StatPrinter { + total: bool, + inodes: bool, + max_depth: Option, + threshold: Option, + apparent_size: bool, + size_format: SizeFormat, + time: Option