From d0d660d035905d9c49fc0f8dafb579d2cc68c0c8 Mon Sep 17 00:00:00 2001 From: ossdev07 <39188636+ossdev07@users.noreply.github.com> Date: Fri, 20 Mar 2020 00:19:38 +0530 Subject: [PATCH 01/21] Add ARM64 jobs in Travis-CI (#366) --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index b3185f95..5270ecb4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,15 @@ matrix: env: TOXENV=py38 - python: 3.8-dev env: TOXENV=py38 + - python: 3.7 + arch: arm64 + env: TOXENV=py37 + - python: 3.8 + arch: arm64 + env: TOXENV=py38 + - python: 3.8-dev + arch: arm64 + env: TOXENV=py38 - python: pypy env: TOXENV=pypy From 3effceca2c2a583b400cb78e9dcdd205e785c9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Wed, 1 Apr 2020 00:57:16 +0200 Subject: [PATCH 02/21] Update list of maintainers Remove myself --- announcement.msg | 1 - 1 file changed, 1 deletion(-) diff --git a/announcement.msg b/announcement.msg index 1c0bcddd..1e32b3c4 100644 --- a/announcement.msg +++ b/announcement.msg @@ -79,7 +79,6 @@ Maintainers The following people are currently responsible for maintaining PyYAML: -* Tina Mueller * Ingy döt Net * Matt Davis From 89f608599d700bf2976bf9307f9f86565f27a80e Mon Sep 17 00:00:00 2001 From: Brad Solomon Date: Fri, 11 Dec 2020 21:12:08 -0500 Subject: [PATCH 03/21] Build modernization (GHA, wheels, setuptools) (#407) * Move most CI to GitHub Actions * Build sdist * Build manylinux1 wheels with libyaml ext (also tested with 2010 and 2014) * Build MacOS x86_64 wheels with libyaml ext * Windows wheel builds remain on AppVeyor until we drop 2.7 support in 6.0 * Smoke tests of all post-build artifacts * Add PEP517/518 build declaration (pyproject.toml with setuptools backend) * Fully move build to setuptools * Drop Python 3.5 support * Declare Python 3.9 support * Update PyPI metadata now that setuptools lets it flow through Co-authored-by: Matt Davis --- .appveyor.yml | 2 +- .github/workflows/ci.yml | 236 +++++++++++++++++++++++++++++++++++ .gitignore | 9 +- .travis.yml | 54 -------- MANIFEST.in | 5 +- lib/_yaml/__init__.py | 31 +++++ lib/yaml/__init__.py | 2 +- lib/yaml/cyaml.py | 2 +- lib3/_yaml/__init__.py | 31 +++++ lib3/yaml/__init__.py | 2 +- lib3/yaml/cyaml.py | 2 +- packaging/build/appveyor.ps1 | 14 +-- packaging/build/libyaml.sh | 26 ++++ packaging/build/macos.sh | 43 +++++++ packaging/build/manylinux.sh | 62 +++++++++ packaging/build/smoketest.py | 22 ++++ pyproject.toml | 3 + setup.py | 71 ++++------- tests/lib/test_yaml_ext.py | 15 ++- tests/lib3/test_yaml_ext.py | 11 +- yaml/__init__.pxd | 0 {ext => yaml}/_yaml.h | 0 {ext => yaml}/_yaml.pxd | 0 {ext => yaml}/_yaml.pyx | 0 24 files changed, 522 insertions(+), 121 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml create mode 100644 lib/_yaml/__init__.py create mode 100644 lib3/_yaml/__init__.py create mode 100755 packaging/build/libyaml.sh create mode 100755 packaging/build/macos.sh create mode 100755 packaging/build/manylinux.sh create mode 100644 packaging/build/smoketest.py create mode 100644 pyproject.toml create mode 100644 yaml/__init__.pxd rename {ext => yaml}/_yaml.h (100%) rename {ext => yaml}/_yaml.pxd (100%) rename {ext => yaml}/_yaml.pyx (100%) diff --git a/.appveyor.yml b/.appveyor.yml index 8c752749..bcefe02e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -10,7 +10,7 @@ image: environment: libyaml_repo_url: https://github.com/yaml/libyaml.git - libyaml_refspec: 0.2.2 + libyaml_refspec: 0.2.5 PYYAML_TEST_GROUP: all # matrix: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..772959c5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,236 @@ +--- + +name: PyYAML CI +on: + push: + pull_request: + workflow_dispatch: + +env: + LIBYAML_REPO: https://github.com/yaml/libyaml + LIBYAML_REF: '0.2.5' +jobs: + python_sdist: + name: pyyaml sdist + runs-on: ubuntu-latest + steps: + - name: checkout pyyaml + uses: actions/checkout@v2 + + - name: install a python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: install build deps + run: | + python -V + + python -m pip install build + + - name: build sdist + run: | + export PYYAML_FORCE_CYTHON=1 # we DO want to force Cythoning, at least until 6.0 + export PYYAML_FORCE_LIBYAML=0 # we don't actually want to build the lib + + python -m build . + + # ensure exactly one artifact was produced + shopt -s nullglob + DISTFILES=(dist/*.tar.gz) + if [[ ${DISTFILES[@]} -ne 1 ]]; then + echo "unexpected content in dist dir: $(ls dist/*.tar.gz)" + exit 1 + fi + + - name: test sdist + run: | + # install some libyaml headers + # TODO: should we smoke test the sdist against the libyaml we built? + sudo apt update + sudo apt install libyaml-dev -y + + # ensure Cython is not present so we use only what's in the sdist + python -m pip uninstall Cython -y || true + + # pass no extra args- we should auto-install with libyaml since it's present + python -m pip install dist/*.tar.gz -v + + python packaging/build/smoketest.py + + - name: upload sdist artifact + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.tar.gz + + + linux_libyaml: + name: libyaml ${{ matrix.arch }} ${{ matrix.platform }} + runs-on: ubuntu-latest + strategy: + matrix: + platform: + # manylinux1 is forward-compatible to 2010/2014 + #- manylinux2014 + #- manylinux2010 + - manylinux1 + arch: + - x86_64 + env: + DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }} + steps: + - name: check cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: | + libyaml + key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + + - name: checkout pyyaml + uses: actions/checkout@v2 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: build libyaml + run: | + docker run --rm -v $(pwd):/io -e LIBYAML_REF -e LIBYAML_REPO --workdir /io "$DOCKER_IMAGE" /io/packaging/build/libyaml.sh + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + linux_pyyaml: + needs: linux_libyaml + name: pyyaml ${{ matrix.arch }} ${{ matrix.platform }} ${{ matrix.python_tag }} + runs-on: ubuntu-latest + strategy: + matrix: + platform: + # so long as manylinux1 container builds work, they're forward-compatible to 2010/2014 + # - manylinux2014 + # - manylinux2010 + - manylinux1 + arch: + - x86_64 + python_tag: + # NB: manylinux >=2014 containers don't have Python 2.7, so we have to use exclude to skip it + - cp27-cp27mu + - cp36-cp36m + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 +# exclude: +# - platform: manylinux2014 +# arch: x86_64 +# python_tag: cp27-cp27mu + env: + AW_PLAT: ${{ matrix.platform }}_${{ matrix.arch }} + DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }} + PYTHON_TAG: ${{ matrix.python_tag }} + PYYAML_BUILD_WHEELS: 1 + steps: + - uses: actions/checkout@v2 + + - name: fetch cached libyaml + id: cached_libyaml + uses: actions/cache@v2 + with: + path: | + libyaml + key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + + - name: ensure libyaml fetched + run: exit 1 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: start container + run: | + docker run --name worker -t -d --rm -v $(pwd):/io "$DOCKER_IMAGE" bash + + - name: build/test/package + run: | + docker exec -e PYTHON_TAG -e PYYAML_RUN_TESTS -e PYYAML_BUILD_WHEELS -e AW_PLAT --workdir /io worker \ + /io/packaging/build/manylinux.sh + + - uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.whl + + macos_libyaml: + name: libyaml ${{ matrix.arch }} ${{ matrix.platform }} + runs-on: ${{ matrix.platform }} + strategy: + matrix: + platform: + - macos-10.15 + arch: + - x86_64 + steps: + - name: check cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: | + libyaml + key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + + - name: checkout pyyaml + uses: actions/checkout@v2 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: build libyaml + env: + MACOSX_DEPLOYMENT_TARGET: '10.9' + run: | + brew install automake coreutils + bash ./packaging/build/libyaml.sh + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + + macos_pyyaml: + needs: macos_libyaml + name: pyyaml ${{ matrix.arch }} ${{ matrix.platform }} ${{ matrix.python_tag }} + runs-on: ${{ matrix.platform }} + strategy: + matrix: + platform: + - macos-10.15 + arch: + - x86_64 + python_tag: + - cp27* + - cp36* + - cp37* + - cp38* + - cp39* + steps: + - name: checkout pyyaml + uses: actions/checkout@v2 + + - name: get cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: | + libyaml + key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + + - name: ensure libyaml fetched + run: exit 1 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: install a python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: build/test/package + env: + CIBW_BUILD: ${{ matrix.python_tag }} + CIBW_BUILD_VERBOSITY: 1 + run: | + bash ./packaging/build/macos.sh + + - uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.whl diff --git a/.gitignore b/.gitignore index 4bf45542..b59e62a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,15 @@ # build outputs /dist/* /build/* -/ext/_yaml.c +/lib/PyYAML.egg-info/* +/lib3/PyYAML.egg-info/* +/wheelhouse/* +/yaml/_yaml.c MANIFEST +**/*.so +**/*.dylib +**/*.pyd + # cached Python binaries *.py[cdo] diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5270ecb4..00000000 --- a/.travis.yml +++ /dev/null @@ -1,54 +0,0 @@ -# dist: xenial - -language: python - -cache: pip - -env: - global: - - PYYAML_TEST_GROUP=all - -matrix: - include: - - python: 2.7 - env: TOXENV=py27 - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - - python: 3.8 - env: TOXENV=py38 - - python: 3.8-dev - env: TOXENV=py38 - - python: 3.7 - arch: arm64 - env: TOXENV=py37 - - python: 3.8 - arch: arm64 - env: TOXENV=py38 - - python: 3.8-dev - arch: arm64 - env: TOXENV=py38 - - python: pypy - env: TOXENV=pypy - -# build libyaml -before_script: -- >- - cd /tmp - && git clone https://github.com/yaml/libyaml.git libyaml - && cd libyaml - && git reset --hard 0.2.2 - && ./bootstrap - && ./configure - && make - && make test-all - && sudo make install - && sudo ldconfig - && cd "$TRAVIS_BUILD_DIR" - -install: pip install cython tox - -script: tox diff --git a/MANIFEST.in b/MANIFEST.in index 185e7807..f4051a11 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,10 @@ -include README LICENSE CHANGES setup.py +include CHANGES README LICENSE Makefile pyproject.toml setup.py recursive-include lib/yaml *.py +recursive-include lib/_yaml *.py recursive-include lib3/yaml *.py +recursive-include lib3/_yaml *.py recursive-include examples *.py *.cfg *.yaml recursive-include tests/data * recursive-include tests/lib *.py recursive-include tests/lib3 *.py +recursive-include yaml * diff --git a/lib/_yaml/__init__.py b/lib/_yaml/__init__.py new file mode 100644 index 00000000..2dadafc1 --- /dev/null +++ b/lib/_yaml/__init__.py @@ -0,0 +1,31 @@ +# This is a stub package designed to roughly emulate the _yaml +# extension module, which previously existed as a standalone module +# and has been moved into the `yaml` package namespace. +# It does not perfectly mimic its old counterpart, but should get +# close enough for anyone who's relying on it even when they shouldn't. +import yaml + +if not yaml.__with_libyaml__: + from sys import version_info + + exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError + raise exc("No module named '_yaml'") +else: + from yaml._yaml import * + import warnings + warnings.warn( + 'The _yaml extension module is now located at yaml._yaml' + ' and its location is subject to change. To use the' + ' LibYAML-based parser and emitter, import from `yaml`:' + ' `from yaml import CLoader as Loader, CDumper as Dumper`.', + DeprecationWarning + ) + del warnings + # Don't `del yaml` here because yaml is actually an existing + # namespace member of _yaml. + +__name__ = '_yaml' +# If the module is top-level (i.e. not a part of any specific package) +# then the attribute should be set to ''. +# https://docs.python.org/3.8/library/types.html +__package__ = '' diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 211fc866..6da15d87 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '5.3.1' +__version__ = '5.4.0a0' try: from cyaml import * diff --git a/lib/yaml/cyaml.py b/lib/yaml/cyaml.py index ebb89593..768b49d6 100644 --- a/lib/yaml/cyaml.py +++ b/lib/yaml/cyaml.py @@ -4,7 +4,7 @@ 'CBaseDumper', 'CSafeDumper', 'CDumper' ] -from _yaml import CParser, CEmitter +from yaml._yaml import CParser, CEmitter from constructor import * diff --git a/lib3/_yaml/__init__.py b/lib3/_yaml/__init__.py new file mode 100644 index 00000000..2dadafc1 --- /dev/null +++ b/lib3/_yaml/__init__.py @@ -0,0 +1,31 @@ +# This is a stub package designed to roughly emulate the _yaml +# extension module, which previously existed as a standalone module +# and has been moved into the `yaml` package namespace. +# It does not perfectly mimic its old counterpart, but should get +# close enough for anyone who's relying on it even when they shouldn't. +import yaml + +if not yaml.__with_libyaml__: + from sys import version_info + + exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError + raise exc("No module named '_yaml'") +else: + from yaml._yaml import * + import warnings + warnings.warn( + 'The _yaml extension module is now located at yaml._yaml' + ' and its location is subject to change. To use the' + ' LibYAML-based parser and emitter, import from `yaml`:' + ' `from yaml import CLoader as Loader, CDumper as Dumper`.', + DeprecationWarning + ) + del warnings + # Don't `del yaml` here because yaml is actually an existing + # namespace member of _yaml. + +__name__ = '_yaml' +# If the module is top-level (i.e. not a part of any specific package) +# then the attribute should be set to ''. +# https://docs.python.org/3.8/library/types.html +__package__ = '' diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 13d687c5..98b662c1 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -8,7 +8,7 @@ from .loader import * from .dumper import * -__version__ = '5.3.1' +__version__ = '5.4.0a0' try: from .cyaml import * __with_libyaml__ = True diff --git a/lib3/yaml/cyaml.py b/lib3/yaml/cyaml.py index 1e606c74..0c213458 100644 --- a/lib3/yaml/cyaml.py +++ b/lib3/yaml/cyaml.py @@ -4,7 +4,7 @@ 'CBaseDumper', 'CSafeDumper', 'CDumper' ] -from _yaml import CParser, CEmitter +from yaml._yaml import CParser, CEmitter from .constructor import * diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index a60d0bbb..d4cc6cf7 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -14,16 +14,16 @@ Function Invoke-Exe([scriptblock]$sb) { } Function Bootstrap() { -<# - # ensure python 3.9 prerelease is present (current Appveyor VS2015 image doesn't include it) + + # ensure python 3.9 is present (current Appveyor VS2015 image doesn't include it) If(-not $(Test-Path C:\Python39)) { - Invoke-Exe { choco.exe install python3 --version=3.9.0-a1 --forcex86 --force --params="/InstallDir:C:\Python39" --no-progress } + Invoke-Exe { choco.exe install python3 --version=3.9.1 -i --forcex86 --force --params="/InstallDir:C:\Python39" --no-progress } } If(-not $(Test-Path C:\Python39-x64)) { - Invoke-Exe { choco.exe install python3 --version=3.9.0-a1 --force --params="/InstallDir:C:\Python39-x64" --no-progress } + Invoke-Exe { choco.exe install python3 --version=3.9.1 -i --force --params="/InstallDir:C:\Python39-x64" --no-progress } } -#> + Write-Output "patching Windows SDK bits for distutils" # patch 7.0/7.1 vcvars SDK bits up to work with distutils query @@ -119,14 +119,14 @@ Bootstrap $pythons = @( "C:\Python27" "C:\Python27-x64" -"C:\Python35" -"C:\Python35-x64" "C:\Python36" "C:\Python36-x64" "C:\Python37" "C:\Python37-x64" "C:\Python38" "C:\Python38-x64" +"C:\Python39" +"C:\Python39-x64" ) #$pythons = @("C:\$($env:PYTHON_VER)") diff --git a/packaging/build/libyaml.sh b/packaging/build/libyaml.sh new file mode 100755 index 00000000..f33cffe9 --- /dev/null +++ b/packaging/build/libyaml.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eux + +# build the requested version of libyaml locally +echo "::group::fetch libyaml ${LIBYAML_REF}" +git config --global advice.detachedHead false +git clone --branch "$LIBYAML_REF" "$LIBYAML_REPO" libyaml +pushd libyaml +git reset --hard "$LIBYAML_REF" +echo "::endgroup::" + +echo "::group::autoconf libyaml w/ static only" +./bootstrap +# build only a static library- reduces our reliance on auditwheel/delocate magic +./configure --disable-dependency-tracking --with-pic --enable-shared=no +echo "::endgroup::" + +echo "::group::build libyaml" +make +echo "::endgroup::" + +echo "::group::test built libyaml" +make test-all +echo "::endgroup::" +popd diff --git a/packaging/build/macos.sh b/packaging/build/macos.sh new file mode 100755 index 00000000..3e629ab4 --- /dev/null +++ b/packaging/build/macos.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -eux + +# doesn't really matter which Python we use, so long as it can run cibuildwheels, and we're consistent within the +# build, since cibuildwheel is internally managing looping over all the Pythons for us. +export PYBIN=/usr/bin/python3 + +${PYBIN} -V +${PYBIN} -m pip install -U --user cibuildwheel +# run cibuildwheel; we can skip CIBW_ENVIRONMENT since the Mac version will directly inherit the envvars we set to +# force Cython and --with-libyaml. cibuildwheel will install Cython before each version is built. We expect that +# the calling environment will set CIBW_SKIP or CIBW_BUILD to control which Pythons we build for. (eg, CIBW_SKIP='pp* cp27* cp35*') + +# we're using a private build of libyaml, so set paths to favor that instead of whatever's laying around +export C_INCLUDE_PATH=$(cd libyaml/include; pwd):${C_INCLUDE_PATH:-} +export LIBRARY_PATH=$(cd libyaml/src/.libs; pwd):${LIBRARY_PATH:-} +export LD_LIBRARY_PATH=$(cd libyaml/src/.libs; pwd):${LD_LIBRARY_PATH:-} + +export PYYAML_FORCE_CYTHON=1 +export PYYAML_FORCE_LIBYAML=1 + +if [[ ${PYYAML_RUN_TESTS:-1} -eq 1 ]]; then + # tweak CIBW behavior to run our tests for us + export CIBW_BEFORE_BUILD='pip install Cython && make testall PYTHON=python' +else + echo "skipping test suite..." +fi + +export CIBW_TEST_COMMAND='python {project}/packaging/build/smoketest.py' + +${PYBIN} -m cibuildwheel --platform macos . + +mkdir -p dist +mv wheelhouse/* dist/ + +# ensure exactly one artifact +shopt -s nullglob +DISTFILES=(dist/*.whl) +if [[ ${#DISTFILES[@]} -ne 1 ]]; then + echo -e "unexpected dist content:\n\n$(ls)" + exit 1 +fi diff --git a/packaging/build/manylinux.sh b/packaging/build/manylinux.sh new file mode 100755 index 00000000..46f5dec2 --- /dev/null +++ b/packaging/build/manylinux.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -eux + +PYBIN="/opt/python/${PYTHON_TAG}/bin/python" + +# modern tools don't allow us to pass eg, --with-libyaml, so we force it via env +export PYYAML_FORCE_CYTHON=1 +export PYYAML_FORCE_LIBYAML=1 + +# we're using a private build of libyaml, so set paths to favor that instead of whatever's laying around +export C_INCLUDE_PATH=libyaml/include:${C_INCLUDE_PATH:-} +export LIBRARY_PATH=libyaml/src/.libs:${LIBRARY_PATH:-} +export LD_LIBRARY_PATH=libyaml/src/.libs:${LD_LIBRARY_PATH:-} + +# install deps +echo "::group::installing build deps" +# FIXME: installing Cython here won't be necessary once we fix tests, since the build is PEP517 and declares its own deps +"${PYBIN}" -m pip install build==0.1.0 Cython +echo "::endgroup::" + +if [[ ${PYYAML_RUN_TESTS:-1} -eq 1 ]]; then + echo "::group::running test suite" + # FIXME: split tests out for easier direct execution w/o Makefile + # run full test suite + make testall PYTHON="${PYBIN}" + echo "::endgroup::" +else + echo "skipping test suite..." +fi + + +if [[ ${PYYAML_BUILD_WHEELS:-0} -eq 1 ]]; then + echo "::group::building wheels" + "${PYBIN}" -m build -w -o tempwheel . + echo "::endgroup::" + + echo "::group::validating wheels" + + for whl in tempwheel/*.whl; do + auditwheel repair --plat "${AW_PLAT}" "$whl" -w dist/ + done + + # ensure exactly one finished artifact + shopt -s nullglob + DISTFILES=(dist/*.whl) + if [[ ${#DISTFILES[@]} -ne 1 ]]; then + echo -e "unexpected dist content:\n\n$(ls)" + exit 1 + fi + + "${PYBIN}" -m pip install dist/*.whl + + "${PYBIN}" packaging/build/smoketest.py + + ls -1 dist/ + + echo "::endgroup::" + +else + echo "skipping wheel build..." +fi diff --git a/packaging/build/smoketest.py b/packaging/build/smoketest.py new file mode 100644 index 00000000..7d799cea --- /dev/null +++ b/packaging/build/smoketest.py @@ -0,0 +1,22 @@ +import sys +import yaml + + +def main(): + # various smoke tests on an installed PyYAML with extension + if not getattr(yaml, '_yaml', None): + raise Exception('C extension is not available at `yaml._yaml`') + + print('embedded libyaml version is {0}'.format(yaml._yaml.get_version_string())) + + for loader, dumper in [(yaml.CLoader, yaml.CDumper), (yaml.Loader, yaml.Dumper)]: + testyaml = 'dude: mar' + loaded = yaml.load(testyaml, Loader=loader) + dumped = yaml.dump(loaded, Dumper=dumper) + if testyaml != dumped.strip(): + raise Exception('roundtrip failed with {0}/{1}'.format(loader, dumper)) + print('smoke test passed for {0}'.format(sys.executable)) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..3ac661b1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel", "Cython"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.py b/setup.py index 5e34adfb..296b5992 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '5.3.1' +VERSION = '5.4.0a0' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability @@ -18,7 +18,7 @@ AUTHOR_EMAIL = 'xi@resolvent.net' LICENSE = "MIT" PLATFORMS = "Any" -URL = "https://github.com/yaml/pyyaml" +URL = "https://pyyaml.org/" DOWNLOAD_URL = "https://pypi.org/project/PyYAML/" CLASSIFIERS = [ "Development Status :: 5 - Production/Stable", @@ -30,16 +30,22 @@ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Text Processing :: Markup", ] - +PROJECT_URLS = { + 'Bug Tracker': 'https://github.com/yaml/pyyaml/issues', + 'CI': 'https://github.com/yaml/pyyaml/actions', + 'Documentation': 'https://pyyaml.org/wiki/PyYAMLDocumentation', + 'Mailing lists': 'http://lists.sourceforge.net/lists/listinfo/yaml-core', + 'Source Code': 'https://github.com/yaml/pyyaml', +} LIBYAML_CHECK = """ #include @@ -59,24 +65,15 @@ """ -import sys, os.path, platform, warnings +import sys, os, os.path, platform, warnings from distutils import log -from distutils.core import setup, Command -from distutils.core import Distribution as _Distribution -from distutils.core import Extension as _Extension -from distutils.command.build_ext import build_ext as _build_ext -from distutils.command.bdist_rpm import bdist_rpm as _bdist_rpm +from setuptools import setup, Command, Distribution as _Distribution, Extension as _Extension +from setuptools.command.build_ext import build_ext as _build_ext from distutils.errors import DistutilsError, CompileError, LinkError, DistutilsPlatformError -if 'setuptools.extension' in sys.modules: - _Extension = sys.modules['setuptools.extension']._Extension - sys.modules['distutils.core'].Extension = _Extension - sys.modules['distutils.extension'].Extension = _Extension - sys.modules['distutils.command.build_ext'].Extension = _Extension - with_cython = False -if 'sdist' in sys.argv: +if 'sdist' in sys.argv or os.environ.get('PYYAML_FORCE_CYTHON') == '1': # we need cython here with_cython = True try: @@ -106,8 +103,8 @@ for w in windows_ignore_warnings: warnings.filterwarnings('ignore', w) -class Distribution(_Distribution): +class Distribution(_Distribution): def __init__(self, attrs=None): _Distribution.__init__(self, attrs) if not self.ext_modules: @@ -138,10 +135,15 @@ def has_ext_modules(self): def ext_status(self, ext): implementation = platform.python_implementation() - if implementation != 'CPython': + if implementation not in ['CPython', 'PyPy']: return False if isinstance(ext, Extension): - with_ext = getattr(self, ext.attr_name) + # the "build by default" behavior is implemented by this returning None + with_ext = getattr(self, ext.attr_name) or os.environ.get('PYYAML_FORCE_{0}'.format(ext.feature_name.upper())) + try: + with_ext = int(with_ext) # attempt coerce envvar to int + except TypeError: + pass return with_ext else: return True @@ -233,27 +235,6 @@ def build_extensions(self): log.warn("Error compiling module, falling back to pure Python") -class bdist_rpm(_bdist_rpm): - - def _make_spec_file(self): - argv0 = sys.argv[0] - features = [] - for ext in self.distribution.ext_modules: - if not isinstance(ext, Extension): - continue - with_ext = getattr(self.distribution, ext.attr_name) - if with_ext is None: - continue - if with_ext: - features.append('--'+ext.option_name) - else: - features.append('--'+ext.neg_option_name) - sys.argv[0] = ' '.join([argv0]+features) - spec_file = _bdist_rpm._make_spec_file(self) - sys.argv[0] = argv0 - return spec_file - - class test(Command): user_options = [] @@ -279,7 +260,6 @@ def run(self): cmdclass = { 'build_ext': build_ext, - 'bdist_rpm': bdist_rpm, 'test': test, } if bdist_wheel: @@ -300,16 +280,17 @@ def run(self): url=URL, download_url=DOWNLOAD_URL, classifiers=CLASSIFIERS, + project_urls=PROJECT_URLS, package_dir={'': {2: 'lib', 3: 'lib3'}[sys.version_info[0]]}, - packages=['yaml'], + packages=['yaml', '_yaml'], ext_modules=[ - Extension('_yaml', ['ext/_yaml.pyx'], + Extension('yaml._yaml', ['yaml/_yaml.pyx'], 'libyaml', "LibYAML bindings", LIBYAML_CHECK, libraries=['yaml']), ], distclass=Distribution, cmdclass=cmdclass, - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', ) diff --git a/tests/lib/test_yaml_ext.py b/tests/lib/test_yaml_ext.py index 7f9357f6..dfe26182 100644 --- a/tests/lib/test_yaml_ext.py +++ b/tests/lib/test_yaml_ext.py @@ -1,5 +1,5 @@ -import _yaml, yaml +import yaml._yaml, yaml import types, pprint, tempfile, sys, os yaml.PyBaseLoader = yaml.BaseLoader @@ -117,10 +117,15 @@ def _tear_down(): def test_c_version(verbose=False): if verbose: - print _yaml.get_version() - print _yaml.get_version_string() - assert ("%s.%s.%s" % _yaml.get_version()) == _yaml.get_version_string(), \ - (_yaml.get_version(), _yaml.get_version_string()) + print yaml._yaml.get_version() + print yaml._yaml.get_version_string() + assert ("%s.%s.%s" % yaml._yaml.get_version()) == yaml._yaml.get_version_string(), \ + (_yaml.get_version(), yaml._yaml.get_version_string()) + +def test_deprecate_yaml_module(): + import _yaml + assert _yaml.__package__ == '' + assert isinstance(_yaml.get_version(), str) def _compare_scanners(py_data, c_data, verbose): py_tokens = list(yaml.scan(py_data, Loader=yaml.PyLoader)) diff --git a/tests/lib3/test_yaml_ext.py b/tests/lib3/test_yaml_ext.py index d902214e..264df0df 100644 --- a/tests/lib3/test_yaml_ext.py +++ b/tests/lib3/test_yaml_ext.py @@ -1,5 +1,5 @@ -import _yaml, yaml +import yaml._yaml, yaml import types, pprint, tempfile, sys, os yaml.PyBaseLoader = yaml.BaseLoader @@ -119,8 +119,13 @@ def test_c_version(verbose=False): if verbose: print(_yaml.get_version()) print(_yaml.get_version_string()) - assert ("%s.%s.%s" % _yaml.get_version()) == _yaml.get_version_string(), \ - (_yaml.get_version(), _yaml.get_version_string()) + assert ("%s.%s.%s" % yaml._yaml.get_version()) == yaml._yaml.get_version_string(), \ + (_yaml.get_version(), yaml._yaml.get_version_string()) + +def test_deprecate_yaml_module(): + import _yaml + assert _yaml.__package__ == '' + assert isinstance(_yaml.get_version(), str) def _compare_scanners(py_data, c_data, verbose): py_tokens = list(yaml.scan(py_data, Loader=yaml.PyLoader)) diff --git a/yaml/__init__.pxd b/yaml/__init__.pxd new file mode 100644 index 00000000..e69de29b diff --git a/ext/_yaml.h b/yaml/_yaml.h similarity index 100% rename from ext/_yaml.h rename to yaml/_yaml.h diff --git a/ext/_yaml.pxd b/yaml/_yaml.pxd similarity index 100% rename from ext/_yaml.pxd rename to yaml/_yaml.pxd diff --git a/ext/_yaml.pyx b/yaml/_yaml.pyx similarity index 100% rename from ext/_yaml.pyx rename to yaml/_yaml.pyx From 4927e75d99e5d21e5e578c97e54c962b7fc0e169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 10:21:57 -0500 Subject: [PATCH 04/21] Add py29 to tox.ini envlist --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 867f1251..8694f6db 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,pypy,py35,py36,py37,py38 +envlist = py27,pypy,py35,py36,py37,py38,py39 [testenv] deps = From 6a19fd77a0cdf81525b03cfea751ca46666e9137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 10:24:14 -0500 Subject: [PATCH 05/21] Rename ci.yml to YAML preferred ci.yaml --- .github/workflows/{ci.yml => ci.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci.yml => ci.yaml} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yaml similarity index 100% rename from .github/workflows/ci.yml rename to .github/workflows/ci.yaml From 219fe65b66d8d658119d1e2b48bf7313701352db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 09:53:43 -0500 Subject: [PATCH 06/21] Don't overindent sequences in maps --- .github/workflows/ci.yaml | 321 +++++++++++++++++++------------------- 1 file changed, 160 insertions(+), 161 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 772959c5..4aabc02c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,5 +1,4 @@ --- - name: PyYAML CI on: push: @@ -14,55 +13,55 @@ jobs: name: pyyaml sdist runs-on: ubuntu-latest steps: - - name: checkout pyyaml - uses: actions/checkout@v2 + - name: checkout pyyaml + uses: actions/checkout@v2 - - name: install a python - uses: actions/setup-python@v2 - with: - python-version: 3.x + - name: install a python + uses: actions/setup-python@v2 + with: + python-version: 3.x - - name: install build deps - run: | - python -V + - name: install build deps + run: | + python -V - python -m pip install build + python -m pip install build - - name: build sdist - run: | - export PYYAML_FORCE_CYTHON=1 # we DO want to force Cythoning, at least until 6.0 - export PYYAML_FORCE_LIBYAML=0 # we don't actually want to build the lib + - name: build sdist + run: | + export PYYAML_FORCE_CYTHON=1 # we DO want to force Cythoning, at least until 6.0 + export PYYAML_FORCE_LIBYAML=0 # we don't actually want to build the lib - python -m build . + python -m build . - # ensure exactly one artifact was produced - shopt -s nullglob - DISTFILES=(dist/*.tar.gz) - if [[ ${DISTFILES[@]} -ne 1 ]]; then - echo "unexpected content in dist dir: $(ls dist/*.tar.gz)" - exit 1 - fi + # ensure exactly one artifact was produced + shopt -s nullglob + DISTFILES=(dist/*.tar.gz) + if [[ ${DISTFILES[@]} -ne 1 ]]; then + echo "unexpected content in dist dir: $(ls dist/*.tar.gz)" + exit 1 + fi - - name: test sdist - run: | - # install some libyaml headers - # TODO: should we smoke test the sdist against the libyaml we built? - sudo apt update - sudo apt install libyaml-dev -y + - name: test sdist + run: | + # install some libyaml headers + # TODO: should we smoke test the sdist against the libyaml we built? + sudo apt update + sudo apt install libyaml-dev -y - # ensure Cython is not present so we use only what's in the sdist - python -m pip uninstall Cython -y || true + # ensure Cython is not present so we use only what's in the sdist + python -m pip uninstall Cython -y || true - # pass no extra args- we should auto-install with libyaml since it's present - python -m pip install dist/*.tar.gz -v + # pass no extra args- we should auto-install with libyaml since it's present + python -m pip install dist/*.tar.gz -v - python packaging/build/smoketest.py + python packaging/build/smoketest.py - - name: upload sdist artifact - uses: actions/upload-artifact@v2 - with: - name: dist - path: dist/*.tar.gz + - name: upload sdist artifact + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.tar.gz linux_libyaml: @@ -71,31 +70,31 @@ jobs: strategy: matrix: platform: - # manylinux1 is forward-compatible to 2010/2014 - #- manylinux2014 - #- manylinux2010 - - manylinux1 + # manylinux1 is forward-compatible to 2010/2014 + #- manylinux2014 + #- manylinux2010 + - manylinux1 arch: - - x86_64 + - x86_64 env: DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }} steps: - - name: check cached libyaml state - id: cached_libyaml - uses: actions/cache@v2 - with: - path: | - libyaml - key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} - - - name: checkout pyyaml - uses: actions/checkout@v2 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: build libyaml - run: | - docker run --rm -v $(pwd):/io -e LIBYAML_REF -e LIBYAML_REPO --workdir /io "$DOCKER_IMAGE" /io/packaging/build/libyaml.sh - if: steps.cached_libyaml.outputs.cache-hit != 'true' + - name: check cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: | + libyaml + key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + + - name: checkout pyyaml + uses: actions/checkout@v2 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: build libyaml + run: | + docker run --rm -v $(pwd):/io -e LIBYAML_REF -e LIBYAML_REPO --workdir /io "$DOCKER_IMAGE" /io/packaging/build/libyaml.sh + if: steps.cached_libyaml.outputs.cache-hit != 'true' linux_pyyaml: needs: linux_libyaml @@ -104,56 +103,56 @@ jobs: strategy: matrix: platform: - # so long as manylinux1 container builds work, they're forward-compatible to 2010/2014 - # - manylinux2014 - # - manylinux2010 - - manylinux1 + # so long as manylinux1 container builds work, they're forward-compatible to 2010/2014 + # - manylinux2014 + # - manylinux2010 + - manylinux1 arch: - - x86_64 + - x86_64 python_tag: - # NB: manylinux >=2014 containers don't have Python 2.7, so we have to use exclude to skip it - - cp27-cp27mu - - cp36-cp36m - - cp37-cp37m - - cp38-cp38 - - cp39-cp39 + # NB: manylinux >=2014 containers don't have Python 2.7, so we have to use exclude to skip it + - cp27-cp27mu + - cp36-cp36m + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 # exclude: -# - platform: manylinux2014 -# arch: x86_64 -# python_tag: cp27-cp27mu +# - platform: manylinux2014 +# arch: x86_64 +# python_tag: cp27-cp27mu env: AW_PLAT: ${{ matrix.platform }}_${{ matrix.arch }} DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }} PYTHON_TAG: ${{ matrix.python_tag }} PYYAML_BUILD_WHEELS: 1 steps: - - uses: actions/checkout@v2 - - - name: fetch cached libyaml - id: cached_libyaml - uses: actions/cache@v2 - with: - path: | - libyaml - key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} - - - name: ensure libyaml fetched - run: exit 1 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: start container - run: | - docker run --name worker -t -d --rm -v $(pwd):/io "$DOCKER_IMAGE" bash - - - name: build/test/package - run: | - docker exec -e PYTHON_TAG -e PYYAML_RUN_TESTS -e PYYAML_BUILD_WHEELS -e AW_PLAT --workdir /io worker \ - /io/packaging/build/manylinux.sh - - - uses: actions/upload-artifact@v2 - with: - name: dist - path: dist/*.whl + - uses: actions/checkout@v2 + + - name: fetch cached libyaml + id: cached_libyaml + uses: actions/cache@v2 + with: + path: | + libyaml + key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + + - name: ensure libyaml fetched + run: exit 1 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: start container + run: | + docker run --name worker -t -d --rm -v $(pwd):/io "$DOCKER_IMAGE" bash + + - name: build/test/package + run: | + docker exec -e PYTHON_TAG -e PYYAML_RUN_TESTS -e PYYAML_BUILD_WHEELS -e AW_PLAT --workdir /io worker \ + /io/packaging/build/manylinux.sh + + - uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.whl macos_libyaml: name: libyaml ${{ matrix.arch }} ${{ matrix.platform }} @@ -161,30 +160,29 @@ jobs: strategy: matrix: platform: - - macos-10.15 + - macos-10.15 arch: - - x86_64 + - x86_64 steps: - - name: check cached libyaml state - id: cached_libyaml - uses: actions/cache@v2 - with: - path: | - libyaml - key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} - - - name: checkout pyyaml - uses: actions/checkout@v2 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: build libyaml - env: - MACOSX_DEPLOYMENT_TARGET: '10.9' - run: | - brew install automake coreutils - bash ./packaging/build/libyaml.sh - if: steps.cached_libyaml.outputs.cache-hit != 'true' - + - name: check cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: | + libyaml + key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + + - name: checkout pyyaml + uses: actions/checkout@v2 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: build libyaml + env: + MACOSX_DEPLOYMENT_TARGET: '10.9' + run: | + brew install automake coreutils + bash ./packaging/build/libyaml.sh + if: steps.cached_libyaml.outputs.cache-hit != 'true' macos_pyyaml: needs: macos_libyaml @@ -193,44 +191,45 @@ jobs: strategy: matrix: platform: - - macos-10.15 + - macos-10.15 arch: - - x86_64 + - x86_64 python_tag: - - cp27* - - cp36* - - cp37* - - cp38* - - cp39* + - cp27* + - cp36* + - cp37* + - cp38* + - cp39* steps: - - name: checkout pyyaml - uses: actions/checkout@v2 - - - name: get cached libyaml state - id: cached_libyaml - uses: actions/cache@v2 - with: - path: | - libyaml - key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} - - - name: ensure libyaml fetched - run: exit 1 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: install a python - uses: actions/setup-python@v2 - with: - python-version: 3.x - - - name: build/test/package - env: - CIBW_BUILD: ${{ matrix.python_tag }} - CIBW_BUILD_VERBOSITY: 1 - run: | - bash ./packaging/build/macos.sh - - - uses: actions/upload-artifact@v2 - with: - name: dist - path: dist/*.whl + - name: checkout pyyaml + uses: actions/checkout@v2 + + - name: get cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: | + libyaml + key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + + - name: ensure libyaml fetched + run: exit 1 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: install a python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: build/test/package + env: + CIBW_BUILD: ${{ matrix.python_tag }} + CIBW_BUILD_VERBOSITY: 1 + run: | + bash ./packaging/build/macos.sh + + - uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.whl +... From 13c7aec48d90420ade8f206727315d3868d97b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 10:03:03 -0500 Subject: [PATCH 07/21] Reduce long lines and adjust blank lines for clarity --- .github/workflows/ci.yaml | 46 +++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4aabc02c..27632e04 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,5 +1,6 @@ --- name: PyYAML CI + on: push: pull_request: @@ -8,6 +9,7 @@ on: env: LIBYAML_REPO: https://github.com/yaml/libyaml LIBYAML_REF: '0.2.5' + jobs: python_sdist: name: pyyaml sdist @@ -29,8 +31,10 @@ jobs: - name: build sdist run: | - export PYYAML_FORCE_CYTHON=1 # we DO want to force Cythoning, at least until 6.0 - export PYYAML_FORCE_LIBYAML=0 # we don't actually want to build the lib + # we DO want to force Cythoning, at least until 6.0 + export PYYAML_FORCE_CYTHON=1 + # we don't actually want to build the lib + export PYYAML_FORCE_LIBYAML=0 python -m build . @@ -45,14 +49,15 @@ jobs: - name: test sdist run: | # install some libyaml headers - # TODO: should we smoke test the sdist against the libyaml we built? + # TODO should we smoke test the sdist against the libyaml we built? sudo apt update sudo apt install libyaml-dev -y # ensure Cython is not present so we use only what's in the sdist python -m pip uninstall Cython -y || true - # pass no extra args- we should auto-install with libyaml since it's present + # pass no extra args- we should auto-install with libyaml since it's + # present python -m pip install dist/*.tar.gz -v python packaging/build/smoketest.py @@ -92,8 +97,14 @@ jobs: if: steps.cached_libyaml.outputs.cache-hit != 'true' - name: build libyaml - run: | - docker run --rm -v $(pwd):/io -e LIBYAML_REF -e LIBYAML_REPO --workdir /io "$DOCKER_IMAGE" /io/packaging/build/libyaml.sh + run: > + docker run --rm + -v $(pwd):/io + -e LIBYAML_REF + -e LIBYAML_REPO + --workdir /io + "$DOCKER_IMAGE" + /io/packaging/build/libyaml.sh if: steps.cached_libyaml.outputs.cache-hit != 'true' linux_pyyaml: @@ -103,14 +114,16 @@ jobs: strategy: matrix: platform: - # so long as manylinux1 container builds work, they're forward-compatible to 2010/2014 + # so long as manylinux1 container builds work, they're + # forward-compatible to 2010/2014 # - manylinux2014 # - manylinux2010 - manylinux1 arch: - x86_64 python_tag: - # NB: manylinux >=2014 containers don't have Python 2.7, so we have to use exclude to skip it + # NB manylinux >=2014 containers don't have Python 2.7, so we have to + # use exclude to skip it - cp27-cp27mu - cp36-cp36m - cp37-cp37m @@ -141,12 +154,21 @@ jobs: if: steps.cached_libyaml.outputs.cache-hit != 'true' - name: start container - run: | - docker run --name worker -t -d --rm -v $(pwd):/io "$DOCKER_IMAGE" bash + run: > + docker run -t -d --rm + --name worker + -v $(pwd):/io + "$DOCKER_IMAGE" + bash - name: build/test/package - run: | - docker exec -e PYTHON_TAG -e PYYAML_RUN_TESTS -e PYYAML_BUILD_WHEELS -e AW_PLAT --workdir /io worker \ + run: > + docker exec + -e PYTHON_TAG + -e PYYAML_RUN_TESTS + -e PYYAML_BUILD_WHEELS + -e AW_PLAT + --workdir /io worker /io/packaging/build/manylinux.sh - uses: actions/upload-artifact@v2 From c851ff7eadc86ddb49cce6702c6bd4c9e7f5e474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 10:07:24 -0500 Subject: [PATCH 08/21] Replace ${{ x }} with ${{x}} Spaces in the syntax make it harder to reason if there will be spaces in the rendering or not. --- .github/workflows/ci.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 27632e04..fc24f173 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -70,7 +70,7 @@ jobs: linux_libyaml: - name: libyaml ${{ matrix.arch }} ${{ matrix.platform }} + name: libyaml ${{matrix.arch}} ${{matrix.platform}} runs-on: ubuntu-latest strategy: matrix: @@ -82,7 +82,7 @@ jobs: arch: - x86_64 env: - DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }} + DOCKER_IMAGE: quay.io/pypa/${{matrix.platform}}_${{matrix.arch}} steps: - name: check cached libyaml state id: cached_libyaml @@ -90,7 +90,7 @@ jobs: with: path: | libyaml - key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: checkout pyyaml uses: actions/checkout@v2 @@ -109,7 +109,7 @@ jobs: linux_pyyaml: needs: linux_libyaml - name: pyyaml ${{ matrix.arch }} ${{ matrix.platform }} ${{ matrix.python_tag }} + name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.python_tag}} runs-on: ubuntu-latest strategy: matrix: @@ -134,9 +134,9 @@ jobs: # arch: x86_64 # python_tag: cp27-cp27mu env: - AW_PLAT: ${{ matrix.platform }}_${{ matrix.arch }} - DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }} - PYTHON_TAG: ${{ matrix.python_tag }} + AW_PLAT: ${{matrix.platform}}_${{matrix.arch}} + DOCKER_IMAGE: quay.io/pypa/${{matrix.platform}}_${{matrix.arch}} + PYTHON_TAG: ${{matrix.python_tag}} PYYAML_BUILD_WHEELS: 1 steps: - uses: actions/checkout@v2 @@ -147,7 +147,7 @@ jobs: with: path: | libyaml - key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: ensure libyaml fetched run: exit 1 @@ -177,8 +177,8 @@ jobs: path: dist/*.whl macos_libyaml: - name: libyaml ${{ matrix.arch }} ${{ matrix.platform }} - runs-on: ${{ matrix.platform }} + name: libyaml ${{matrix.arch}} ${{matrix.platform}} + runs-on: ${{matrix.platform}} strategy: matrix: platform: @@ -192,7 +192,7 @@ jobs: with: path: | libyaml - key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: checkout pyyaml uses: actions/checkout@v2 @@ -208,8 +208,8 @@ jobs: macos_pyyaml: needs: macos_libyaml - name: pyyaml ${{ matrix.arch }} ${{ matrix.platform }} ${{ matrix.python_tag }} - runs-on: ${{ matrix.platform }} + name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.python_tag}} + runs-on: ${{matrix.platform}} strategy: matrix: platform: @@ -232,7 +232,7 @@ jobs: with: path: | libyaml - key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: ensure libyaml fetched run: exit 1 @@ -245,7 +245,7 @@ jobs: - name: build/test/package env: - CIBW_BUILD: ${{ matrix.python_tag }} + CIBW_BUILD: ${{matrix.python_tag}} CIBW_BUILD_VERBOSITY: 1 run: | bash ./packaging/build/macos.sh From 492bcbaa1385f11c98ce15a8e184d7002ba9339f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 10:28:17 -0500 Subject: [PATCH 09/21] Better (non)use of literal form scalars --- .github/workflows/ci.yaml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fc24f173..0bada62f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,7 +26,6 @@ jobs: - name: install build deps run: | python -V - python -m pip install build - name: build sdist @@ -88,8 +87,7 @@ jobs: id: cached_libyaml uses: actions/cache@v2 with: - path: | - libyaml + path: libyaml key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: checkout pyyaml @@ -145,8 +143,7 @@ jobs: id: cached_libyaml uses: actions/cache@v2 with: - path: | - libyaml + path: libyaml key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: ensure libyaml fetched @@ -190,8 +187,7 @@ jobs: id: cached_libyaml uses: actions/cache@v2 with: - path: | - libyaml + path: libyaml key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: checkout pyyaml @@ -230,8 +226,7 @@ jobs: id: cached_libyaml uses: actions/cache@v2 with: - path: | - libyaml + path: libyaml key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: ensure libyaml fetched @@ -247,8 +242,7 @@ jobs: env: CIBW_BUILD: ${{matrix.python_tag}} CIBW_BUILD_VERBOSITY: 1 - run: | - bash ./packaging/build/macos.sh + run: bash ./packaging/build/macos.sh - uses: actions/upload-artifact@v2 with: From c5fb9097983aa98f6e6c95e5d6cf989a6289588f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 10:32:36 -0500 Subject: [PATCH 10/21] Use long forms for docker run options --- .github/workflows/ci.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0bada62f..54a80aac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -97,9 +97,9 @@ jobs: - name: build libyaml run: > docker run --rm - -v $(pwd):/io - -e LIBYAML_REF - -e LIBYAML_REPO + --volume $(pwd):/io + --env LIBYAML_REF + --env LIBYAML_REPO --workdir /io "$DOCKER_IMAGE" /io/packaging/build/libyaml.sh @@ -152,19 +152,19 @@ jobs: - name: start container run: > - docker run -t -d --rm + docker run --rm --tty --detach --name worker - -v $(pwd):/io + --volume $(pwd):/io "$DOCKER_IMAGE" bash - name: build/test/package run: > docker exec - -e PYTHON_TAG - -e PYYAML_RUN_TESTS - -e PYYAML_BUILD_WHEELS - -e AW_PLAT + --env PYTHON_TAG + --env PYYAML_RUN_TESTS + --env PYYAML_BUILD_WHEELS + --env AW_PLAT --workdir /io worker /io/packaging/build/manylinux.sh From d6572c3a80607bd095b7288426a4d1352349f961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 10:34:52 -0500 Subject: [PATCH 11/21] Remove unneeded quotes --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 54a80aac..86dfe4ce 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,7 +8,7 @@ on: env: LIBYAML_REPO: https://github.com/yaml/libyaml - LIBYAML_REF: '0.2.5' + LIBYAML_REF: 0.2.5 jobs: python_sdist: From c97691596eec279ef9191a9b3bba583a17139d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 10:38:09 -0500 Subject: [PATCH 12/21] Shell code improvements --- .github/workflows/ci.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 86dfe4ce..9ef4ecf4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,12 +38,10 @@ jobs: python -m build . # ensure exactly one artifact was produced - shopt -s nullglob - DISTFILES=(dist/*.tar.gz) - if [[ ${DISTFILES[@]} -ne 1 ]]; then + [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { echo "unexpected content in dist dir: $(ls dist/*.tar.gz)" exit 1 - fi + } - name: test sdist run: | @@ -97,7 +95,7 @@ jobs: - name: build libyaml run: > docker run --rm - --volume $(pwd):/io + --volume "$(pwd):/io" --env LIBYAML_REF --env LIBYAML_REPO --workdir /io @@ -154,7 +152,7 @@ jobs: run: > docker run --rm --tty --detach --name worker - --volume $(pwd):/io + --volume "$(pwd):/io" "$DOCKER_IMAGE" bash From 0b6b7d61719fbe0a11f0980489f1bf8ce746c164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 10:49:12 -0500 Subject: [PATCH 13/21] Start sentences and phrases for capital letters End sentences with periods. --- .github/workflows/ci.yaml | 70 +++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9ef4ecf4..b939ee15 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,54 +12,54 @@ env: jobs: python_sdist: - name: pyyaml sdist + name: PyYAML sdist runs-on: ubuntu-latest steps: - - name: checkout pyyaml + - name: Checkout pyyaml uses: actions/checkout@v2 - - name: install a python + - name: Install a python uses: actions/setup-python@v2 with: python-version: 3.x - - name: install build deps + - name: Install build deps run: | python -V python -m pip install build - - name: build sdist + - name: Build sdist run: | - # we DO want to force Cythoning, at least until 6.0 + # We DO want to force Cythoning, at least until 6.0. export PYYAML_FORCE_CYTHON=1 - # we don't actually want to build the lib + # We don't actually want to build the lib. export PYYAML_FORCE_LIBYAML=0 python -m build . - # ensure exactly one artifact was produced + # Ensure exactly one artifact was produced. [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { - echo "unexpected content in dist dir: $(ls dist/*.tar.gz)" + echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." exit 1 } - - name: test sdist + - name: Test sdist run: | - # install some libyaml headers - # TODO should we smoke test the sdist against the libyaml we built? + # Install some libyaml headers. + # TODO Should we smoke test the sdist against the libyaml we built? sudo apt update sudo apt install libyaml-dev -y - # ensure Cython is not present so we use only what's in the sdist + # Ensure Cython is not present so we use only what's in the sdist. python -m pip uninstall Cython -y || true - # pass no extra args- we should auto-install with libyaml since it's - # present + # Pass no extra args. + # We should auto-install with libyaml since it's present. python -m pip install dist/*.tar.gz -v python packaging/build/smoketest.py - - name: upload sdist artifact + - name: Upload sdist artifact uses: actions/upload-artifact@v2 with: name: dist @@ -72,7 +72,7 @@ jobs: strategy: matrix: platform: - # manylinux1 is forward-compatible to 2010/2014 + # manylinux1 is forward-compatible to 2010/2014. #- manylinux2014 #- manylinux2010 - manylinux1 @@ -81,18 +81,18 @@ jobs: env: DOCKER_IMAGE: quay.io/pypa/${{matrix.platform}}_${{matrix.arch}} steps: - - name: check cached libyaml state + - name: Check cached libyaml state id: cached_libyaml uses: actions/cache@v2 with: path: libyaml key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - - name: checkout pyyaml + - name: Checkout pyyaml uses: actions/checkout@v2 if: steps.cached_libyaml.outputs.cache-hit != 'true' - - name: build libyaml + - name: Build libyaml run: > docker run --rm --volume "$(pwd):/io" @@ -110,8 +110,8 @@ jobs: strategy: matrix: platform: - # so long as manylinux1 container builds work, they're - # forward-compatible to 2010/2014 + # So long as manylinux1 container builds work, they're + # forward-compatible to 2010/2014. # - manylinux2014 # - manylinux2010 - manylinux1 @@ -119,7 +119,7 @@ jobs: - x86_64 python_tag: # NB manylinux >=2014 containers don't have Python 2.7, so we have to - # use exclude to skip it + # use exclude to skip it. - cp27-cp27mu - cp36-cp36m - cp37-cp37m @@ -137,18 +137,18 @@ jobs: steps: - uses: actions/checkout@v2 - - name: fetch cached libyaml + - name: Fetch cached libyaml id: cached_libyaml uses: actions/cache@v2 with: path: libyaml key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - - name: ensure libyaml fetched + - name: Ensure libyaml fetched run: exit 1 if: steps.cached_libyaml.outputs.cache-hit != 'true' - - name: start container + - name: Start container run: > docker run --rm --tty --detach --name worker @@ -156,7 +156,7 @@ jobs: "$DOCKER_IMAGE" bash - - name: build/test/package + - name: Build/Test/Package run: > docker exec --env PYTHON_TAG @@ -181,18 +181,18 @@ jobs: arch: - x86_64 steps: - - name: check cached libyaml state + - name: Check cached libyaml state id: cached_libyaml uses: actions/cache@v2 with: path: libyaml key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - - name: checkout pyyaml + - name: Checkout pyyaml uses: actions/checkout@v2 if: steps.cached_libyaml.outputs.cache-hit != 'true' - - name: build libyaml + - name: Build libyaml env: MACOSX_DEPLOYMENT_TARGET: '10.9' run: | @@ -217,26 +217,26 @@ jobs: - cp38* - cp39* steps: - - name: checkout pyyaml + - name: Checkout pyyaml uses: actions/checkout@v2 - - name: get cached libyaml state + - name: Get cached libyaml state id: cached_libyaml uses: actions/cache@v2 with: path: libyaml key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - - name: ensure libyaml fetched + - name: Ensure libyaml fetched run: exit 1 if: steps.cached_libyaml.outputs.cache-hit != 'true' - - name: install a python + - name: Install a python uses: actions/setup-python@v2 with: python-version: 3.x - - name: build/test/package + - name: Build/Test/Package env: CIBW_BUILD: ${{matrix.python_tag}} CIBW_BUILD_VERBOSITY: 1 From 1e1c7fb7c09e9149967c208a6fd07276a6140d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 11:05:57 -0500 Subject: [PATCH 14/21] Add a newline character to end of pyproject.toml Is this TOML file actually needed? I'd prefer to remove it since it does so little, and stands out so prominiently. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3ac661b1..2bf5ec80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] requires = ["setuptools", "wheel", "Cython"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" From fe150624146ee631bb0f95e45731e8b01281fed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Wed, 23 Dec 2020 11:23:58 -0500 Subject: [PATCH 15/21] Add 3.9 to appveyor file for completeness sake Are we done with appveyor now? Can we just remove this file? --- .appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index bcefe02e..fbc41c36 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -24,6 +24,8 @@ environment: # - PYTHON_VER: Python37-x64 # - PYTHON_VER: Python38 # - PYTHON_VER: Python38-x64 +# - PYTHON_VER: Python39 +# - PYTHON_VER: Python39-x64 #init: #- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) From a001f2782501ad2d24986959f0239a354675f9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Sat, 9 Jan 2021 10:53:23 -0500 Subject: [PATCH 16/21] Fix for CVE-2020-14343 Per suggestion https://github.com/yaml/pyyaml/issues/420#issuecomment-663888344 move a few constructors from full_load to unsafe_load. --- lib/yaml/constructor.py | 24 ++++++++++++------------ lib3/yaml/constructor.py | 24 ++++++++++++------------ tests/lib/test_recursive.py | 2 +- tests/lib3/test_recursive.py | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 794681cb..c42ee344 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -722,18 +722,6 @@ def construct_python_object_new(self, suffix, node): u'tag:yaml.org,2002:python/name:', FullConstructor.construct_python_name) -FullConstructor.add_multi_constructor( - u'tag:yaml.org,2002:python/module:', - FullConstructor.construct_python_module) - -FullConstructor.add_multi_constructor( - u'tag:yaml.org,2002:python/object:', - FullConstructor.construct_python_object) - -FullConstructor.add_multi_constructor( - u'tag:yaml.org,2002:python/object/new:', - FullConstructor.construct_python_object_new) - class UnsafeConstructor(FullConstructor): def find_python_module(self, name, mark): @@ -750,6 +738,18 @@ def set_python_instance_state(self, instance, state): return super(UnsafeConstructor, self).set_python_instance_state( instance, state, unsafe=True) +UnsafeConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/module:', + UnsafeConstructor.construct_python_module) + +UnsafeConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object:', + UnsafeConstructor.construct_python_object) + +UnsafeConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object/new:', + UnsafeConstructor.construct_python_object_new) + UnsafeConstructor.add_multi_constructor( u'tag:yaml.org,2002:python/object/apply:', UnsafeConstructor.construct_python_object_apply) diff --git a/lib3/yaml/constructor.py b/lib3/yaml/constructor.py index 1948b125..619acd30 100644 --- a/lib3/yaml/constructor.py +++ b/lib3/yaml/constructor.py @@ -710,18 +710,6 @@ def construct_python_object_new(self, suffix, node): 'tag:yaml.org,2002:python/name:', FullConstructor.construct_python_name) -FullConstructor.add_multi_constructor( - 'tag:yaml.org,2002:python/module:', - FullConstructor.construct_python_module) - -FullConstructor.add_multi_constructor( - 'tag:yaml.org,2002:python/object:', - FullConstructor.construct_python_object) - -FullConstructor.add_multi_constructor( - 'tag:yaml.org,2002:python/object/new:', - FullConstructor.construct_python_object_new) - class UnsafeConstructor(FullConstructor): def find_python_module(self, name, mark): @@ -738,6 +726,18 @@ def set_python_instance_state(self, instance, state): return super(UnsafeConstructor, self).set_python_instance_state( instance, state, unsafe=True) +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/module:', + UnsafeConstructor.construct_python_module) + +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object:', + UnsafeConstructor.construct_python_object) + +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object/new:', + UnsafeConstructor.construct_python_object_new) + UnsafeConstructor.add_multi_constructor( 'tag:yaml.org,2002:python/object/apply:', UnsafeConstructor.construct_python_object_apply) diff --git a/tests/lib/test_recursive.py b/tests/lib/test_recursive.py index 312204ea..04c57985 100644 --- a/tests/lib/test_recursive.py +++ b/tests/lib/test_recursive.py @@ -30,7 +30,7 @@ def test_recursive(recursive_filename, verbose=False): output2 = None try: output1 = yaml.dump(value1) - value2 = yaml.load(output1, yaml.FullLoader) + value2 = yaml.load(output1, yaml.UnsafeLoader) output2 = yaml.dump(value2) assert output1 == output2, (output1, output2) finally: diff --git a/tests/lib3/test_recursive.py b/tests/lib3/test_recursive.py index 74c2ee65..08042c81 100644 --- a/tests/lib3/test_recursive.py +++ b/tests/lib3/test_recursive.py @@ -31,7 +31,7 @@ def test_recursive(recursive_filename, verbose=False): output2 = None try: output1 = yaml.dump(value1) - value2 = yaml.full_load(output1) + value2 = yaml.unsafe_load(output1) output2 = yaml.dump(value2) assert output1 == output2, (output1, output2) finally: From fc914d52c43f499224f7fb4c2d4c47623adc5b33 Mon Sep 17 00:00:00 2001 From: Phil Sphicas Date: Fri, 18 Sep 2020 00:29:02 -0700 Subject: [PATCH 17/21] Avoid repeatedly appending to yaml_implicit_resolvers Repeated calls to `resolve` can experience performance degredation, if `add_implicit_resolver` has been called with `first=None` (to add an implicit resolver with an unspecified first character). For example, every time `foo` is encountered, the "wildcard implicit resolvers" (with `first=None`) will be appended to the list of implicit resolvers for strings starting with `f`, which will normally be the resolver for booleans. The list `yaml_implicit_resolvers['f']` will keep getting longer. The same behavior applies for any first-letter matches with existing implicit resolvers. This change avoids unintentionally mutating the lists in the class-level dict `yaml_implicit_resolvers` by looping through a temporary copy. Fixes: #439 --- lib/yaml/resolver.py | 4 ++-- lib3/yaml/resolver.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/yaml/resolver.py b/lib/yaml/resolver.py index 528fbc0e..ba9aeab2 100644 --- a/lib/yaml/resolver.py +++ b/lib/yaml/resolver.py @@ -146,8 +146,8 @@ def resolve(self, kind, value, implicit): resolvers = self.yaml_implicit_resolvers.get(u'', []) else: resolvers = self.yaml_implicit_resolvers.get(value[0], []) - resolvers += self.yaml_implicit_resolvers.get(None, []) - for tag, regexp in resolvers: + wildcard_resolvers = self.yaml_implicit_resolvers.get(None, []) + for tag, regexp in resolvers + wildcard_resolvers: if regexp.match(value): return tag implicit = implicit[1] diff --git a/lib3/yaml/resolver.py b/lib3/yaml/resolver.py index 02b82e73..013896d2 100644 --- a/lib3/yaml/resolver.py +++ b/lib3/yaml/resolver.py @@ -146,8 +146,8 @@ def resolve(self, kind, value, implicit): resolvers = self.yaml_implicit_resolvers.get('', []) else: resolvers = self.yaml_implicit_resolvers.get(value[0], []) - resolvers += self.yaml_implicit_resolvers.get(None, []) - for tag, regexp in resolvers: + wildcard_resolvers = self.yaml_implicit_resolvers.get(None, []) + for tag, regexp in resolvers + wildcard_resolvers: if regexp.match(value): return tag implicit = implicit[1] From ddf20330be1fae8813b8ce1789c48f244746d252 Mon Sep 17 00:00:00 2001 From: Ovv Date: Mon, 6 Apr 2020 11:14:45 +0200 Subject: [PATCH 18/21] constructor.timezone: __copy_ & __deepcopy__ close #387 --- lib/yaml/constructor.py | 6 ++++++ tests/lib/test_constructor.py | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index c42ee344..ff4e3682 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -38,6 +38,12 @@ def utcoffset(self, dt=None): def dst(self, dt=None): return datetime.timedelta(0) + def __copy__(self): + return self.__deepcopy__() + + def __deepcopy__(self, memodict={}): + return self.__class__(self.utcoffset()) + __repr__ = __str__ = tzname diff --git a/tests/lib/test_constructor.py b/tests/lib/test_constructor.py index 5a8cce21..c76df5ed 100644 --- a/tests/lib/test_constructor.py +++ b/tests/lib/test_constructor.py @@ -305,6 +305,18 @@ def test_subclass_blacklist_types(data_filename, verbose=False): test_subclass_blacklist_types.unittest = ['.subclass_blacklist'] +def test_timezone_copy(verbose=False): + import copy + tzinfo = yaml.constructor.timezone(datetime.timedelta(0)) + + tz_copy = copy.copy(tzinfo) + tz_deepcopy = copy.deepcopy(tzinfo) + + if tzinfo.tzname() != tz_copy.tzname() != tz_deepcopy.tzname(): + raise AssertionError("Timezones should be equal") + +test_timezone_copy.unittest = [] + if __name__ == '__main__': import sys, test_constructor sys.modules['test_constructor'] = sys.modules['__main__'] From ee98abd7d7bd2ca9c7b98aa19164fd0306a3f3d2 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Wed, 13 Jan 2021 13:44:38 -0800 Subject: [PATCH 19/21] Run CI on PR base branch changes --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b939ee15..c67847b2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,6 +4,7 @@ name: PyYAML CI on: push: pull_request: + types: [opened, synchronize, edited, reopened] workflow_dispatch: env: From a60f7a19c0b418fe95fcf2ec0957005ae39e1090 Mon Sep 17 00:00:00 2001 From: Anish Athalye Date: Wed, 22 Jan 2020 16:07:06 -0500 Subject: [PATCH 20/21] Fix compatibility with Jython MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch was taken from https://github.com/yaml/pyyaml/issues/369#issuecomment-571596545, authored by Pekka Klärck . In short, Jython doesn't support lone surrogates, so importing yaml (and in particular, loading `reader.py`) caused a UnicodeDecodeError. This patch works around this through a clever use of `eval` to defer evaluation of the string containing the lone surrogates, only doing it on non-Jython platforms. This is only done in `lib/yaml/reader.py` and not `lib3/yaml/reader.py` because Jython does not support Python 3. With this patch, Jython's behavior with respect to Unicode code points over 0xFFFF becomes as it was before 0716ae21a1e7ab6b4ef73428c0c8fff49685d057. It still does not pass all the unit tests on Jython (passes 1275, fails 3, errors on 1); all the failing tests are related to unicode. Still, this is better than simply crashing upon `import yaml`. With this patch, all tests continue to pass on Python 2 / Python 3. --- lib/yaml/reader.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/yaml/reader.py b/lib/yaml/reader.py index 4b377d61..4c421509 100644 --- a/lib/yaml/reader.py +++ b/lib/yaml/reader.py @@ -137,9 +137,14 @@ def determine_encoding(self): self.update(1) if has_ucs4: - NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]') + NON_PRINTABLE = u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]' + elif sys.platform.startswith('java'): + # Jython doesn't support lone surrogates https://bugs.jython.org/issue2048 + NON_PRINTABLE = u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]' else: - NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uFFFD]|(?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF](?:[^\uDC00-\uDFFF]|$)') + # Need to use eval here due to the above Jython issue + NON_PRINTABLE = eval(r"u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uFFFD]|(?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF](?:[^\uDC00-\uDFFF]|$)'") + NON_PRINTABLE = re.compile(NON_PRINTABLE) def check_printable(self, data): match = self.NON_PRINTABLE.search(data) if match: From 58d0cb7ee09954c67fabfbd714c5673b03e7a9e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Tue, 19 Jan 2021 14:07:59 -0500 Subject: [PATCH 21/21] 5.4 release --- CHANGES | 14 +++++++++++--- LICENSE | 2 +- announcement.msg | 38 ++++++++++++++++++++++++-------------- lib/yaml/__init__.py | 2 +- lib3/yaml/__init__.py | 2 +- setup.py | 2 +- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/CHANGES b/CHANGES index f3facb14..876bd33c 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,14 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/commits/ * https://bitbucket.org/xi/pyyaml/commits/ +5.4 (2021-01-19) + +* https://github.com/yaml/pyyaml/pull/407 -- Build modernization, remove distutils, fix metadata, build wheels, CI to GHA +* https://github.com/yaml/pyyaml/pull/472 -- Fix for CVE-2020-14343, moves arbitrary python tags to UnsafeLoader +* https://github.com/yaml/pyyaml/pull/441 -- Fix memory leak in implicit resolver setup +* https://github.com/yaml/pyyaml/pull/392 -- Fix py2 copy support for timezone objects +* https://github.com/yaml/pyyaml/pull/378 -- Fix compatibility with Jython + 5.3.1 (2020-03-18) * https://github.com/yaml/pyyaml/pull/386 -- Prevents arbitrary code execution during python/object/new constructor @@ -11,7 +19,7 @@ For a complete changelog, see: 5.3 (2020-01-06) * https://github.com/yaml/pyyaml/pull/290 -- Use `is` instead of equality for comparing with `None` -* https://github.com/yaml/pyyaml/pull/270 -- fix typos and stylistic nit +* https://github.com/yaml/pyyaml/pull/270 -- Fix typos and stylistic nit * https://github.com/yaml/pyyaml/pull/309 -- Fix up small typo * https://github.com/yaml/pyyaml/pull/161 -- Fix handling of __slots__ * https://github.com/yaml/pyyaml/pull/358 -- Allow calling add_multi_constructor with None @@ -21,8 +29,8 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/pull/359 -- Use full_load in yaml-highlight example * https://github.com/yaml/pyyaml/pull/244 -- Document that PyYAML is implemented with Cython * https://github.com/yaml/pyyaml/pull/329 -- Fix for Python 3.10 -* https://github.com/yaml/pyyaml/pull/310 -- increase size of index, line, and column fields -* https://github.com/yaml/pyyaml/pull/260 -- remove some unused imports +* https://github.com/yaml/pyyaml/pull/310 -- Increase size of index, line, and column fields +* https://github.com/yaml/pyyaml/pull/260 -- Remove some unused imports * https://github.com/yaml/pyyaml/pull/163 -- Create timezone-aware datetimes when parsed as such * https://github.com/yaml/pyyaml/pull/363 -- Add tests for timezone diff --git a/LICENSE b/LICENSE index 3d82c281..2f1b8e15 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2017-2020 Ingy döt Net +Copyright (c) 2017-2021 Ingy döt Net Copyright (c) 2006-2016 Kirill Simonov Permission is hereby granted, free of charge, to any person obtaining a copy of diff --git a/announcement.msg b/announcement.msg index 1e32b3c4..118d5e35 100644 --- a/announcement.msg +++ b/announcement.msg @@ -1,25 +1,34 @@ -From: Tina Müller +From: Ingy döt Net To: python-list@python.org, python-announce@python.org, yaml-core@lists.sourceforge.net -Subject: [ANN] PyYAML-5.3.1: YAML parser and emitter for Python +Subject: [ANN] PyYAML-5.4 Released -======================= -Announcing PyYAML-5.3.1 -======================= +===================== +Announcing PyYAML-5.4 +===================== A new release of PyYAML is now available: -https://pypi.org/project/PyYAML/ +https://github.com/yaml/pyyaml/releases/tag/5.4 -This release contains a security fix for CVE-2020-1747. FullLoader was still -exploitable for arbitrary command execution. -https://bugzilla.redhat.com/show_bug.cgi?id=1807367 +This release contains a security fix for CVE-2020-14343. It removes the +python/module, python/object, and python/object/new tags from the FullLoader. +YAML that uses these tags must be loaded by UnsafeLoader, or a custom loader +that has explicitly enabled them. + +This release also adds Python wheels for manylinux1 (x86_64) and MacOS (x86_64) +with the libyaml extension included (built on libyaml 0.2.5). + +PyYAML 5.4 will be the last release to support Python 2.7 (except for possible +critical bug fix releases). -Thanks to Riccardo Schirone (https://github.com/ret2libc) for both reporting -this and providing the fixes to resolve it. Changes ======= -* https://github.com/yaml/pyyaml/pull/386 -- Prevents arbitrary code execution during python/object/new constructor +* https://github.com/yaml/pyyaml/pull/407 -- build modernization, remove distutils, fix metadata, build wheels, CI to GHA +* https://github.com/yaml/pyyaml/pull/472 -- fix for CVE-2020-14343, moves arbitrary python tags to UnsafeLoader +* https://github.com/yaml/pyyaml/pull/441 -- fix memory leak in implicit resolver setup +* https://github.com/yaml/pyyaml/pull/392 -- fix py2 copy support for timezone objects +* https://github.com/yaml/pyyaml/pull/378 -- fix compatibility with Jython Resources @@ -55,6 +64,7 @@ files to object serialization and persistence. Example ======= +``` >>> import yaml >>> yaml.full_load(""" @@ -72,7 +82,7 @@ name: PyYAML homepage: https://github.com/yaml/pyyaml description: YAML parser and emitter for Python keywords: [YAML, serialization, configuration, persistence, pickle] - +``` Maintainers =========== @@ -89,7 +99,7 @@ See: https://github.com/yaml/pyyaml/pulls Copyright ========= -Copyright (c) 2017-2020 Ingy döt Net +Copyright (c) 2017-2021 Ingy döt Net Copyright (c) 2006-2016 Kirill Simonov The PyYAML module was written by Kirill Simonov . diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 6da15d87..1263d96f 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '5.4.0a0' +__version__ = '5.4' try: from cyaml import * diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 98b662c1..ee3d4b3e 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -8,7 +8,7 @@ from .loader import * from .dumper import * -__version__ = '5.4.0a0' +__version__ = '5.4' try: from .cyaml import * __with_libyaml__ = True diff --git a/setup.py b/setup.py index 296b5992..ed2487d9 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '5.4.0a0' +VERSION = '5.4' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability