diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 0000000..bad03bc --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,54 @@ +name: Documentation + +on: + pull_request: + push: + branches: + - main + release: + types: + - published + +jobs: + build-docs: + runs-on: ubuntu-latest + + steps: + - name: checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Micromamba + uses: mamba-org/provision-with-micromamba@main + with: + environment-file: false + + - name: Build environment + shell: bash -l {0} + run: | + micromamba create --name TEST python=3 --file requirements.txt --file requirements-dev.txt --channel conda-forge + micromamba activate TEST + python -m pip install -e . --no-deps --force-reinstall + + - name: Get the version + id: get_version + run: echo ::set-output name=VERSION::$(python setup.py --version) + + - name: Build documentation + shell: bash -l {0} + run: | + set -e + micromamba activate TEST + jupyter nbconvert --to notebook --execute notebooks/quick_intro.ipynb --output=quick_intro-output.ipynb + mv notebooks/*output.ipynb docs/source/ + pushd docs + make clean html linkcheck + popd + + - name: Deploy + if: github.event_name == 'release' || github.event_name == 'push' + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: docs/build/html diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml deleted file mode 100644 index 48bb600..0000000 --- a/.github/workflows/pip.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Pip - -on: - pull_request: - push: - branches: [master] - -jobs: - run: - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: ["3.7", "3.8", "3.9"] - os: [windows-latest, ubuntu-latest, macos-latest] - - steps: - - uses: actions/checkout@v2 - - - name: Setup python - uses: actions/setup-python@v2 - - - name: Python ${{ matrix.python-version }} - run: | - pip install -r requirements.txt -r requirements-dev.txt - pip install -e . --no-deps --force-reinstall - - - name: pip-tests - run: | - pytest -rxs tests diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 7233479..0000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: pre-commit - -on: - pull_request: - push: - branches: [master] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 0000000..6676875 --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,52 @@ +name: Publish to PyPI + +on: + pull_request: + push: + branches: + - main + release: + types: + - published + +jobs: + packages: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.x" + + - name: Get tags + run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* + shell: bash + + - name: Install build tools + run: | + python -m pip install --upgrade pip wheel setuptools setuptools_scm build twine + + shell: bash + + - name: Build binary wheel + run: python -m build --sdist --wheel . --outdir dist + + - name: CheckFiles + run: | + ls dist + shell: bash + + - name: Test wheels + run: | + cd dist && python -m pip install ctd*.whl + python -m twine check * + shell: bash + + - name: Publish a Python distribution to PyPI + if: success() && github.event_name == 'release' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/test-conda.yml b/.github/workflows/test-conda.yml deleted file mode 100644 index 5fa1a20..0000000 --- a/.github/workflows/test-conda.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Test-Conda - -on: - pull_request: - push: - branches: [master] - -jobs: - run: - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: ["3.7", "3.8", "3.9"] - os: [windows-latest, ubuntu-latest, macos-latest] - - steps: - - uses: actions/checkout@v2 - - - name: Setup Conda - uses: s-weigand/setup-conda@v1 - with: - activate-conda: false - conda-channels: conda-forge - - - name: Python ${{ matrix.python-version }} - shell: bash -l {0} - run: | - conda create --name TEST python=${{ matrix.python-version }} --file requirements.txt --file requirements-dev.txt - source activate TEST - pip install -e . --no-deps --force-reinstall - conda info --all - conda list - - - name: Tests - shell: bash -l {0} - run: | - source activate TEST - pytest -rxs tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..f4a3336 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,36 @@ +name: Tests + +on: + pull_request: + push: + branches: [main] + +jobs: + run: + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] + os: [windows-latest, ubuntu-latest, macos-latest] + fail-fast: false + + steps: + - uses: actions/checkout@v3 + + - name: Setup Micromamba + uses: mamba-org/provision-with-micromamba@main + with: + environment-file: false + + - name: Python ${{ matrix.python-version }} + shell: bash -l {0} + run: | + micromamba create --name TEST python=${{ matrix.python-version }} --file requirements.txt --file requirements-dev.txt --channel conda-forge + micromamba activate TEST + python -m pip install -e . --no-deps --force-reinstall + + - name: Tests + shell: bash -l {0} + run: | + micromamba activate TEST + python -m pytest -rxs tests diff --git a/.gitignore b/.gitignore index 6abc136..9dbacdb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ build/ ctd/_version.py dist/ +*-output.ipynb +*.pyc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b2c5e08..6f7a75c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.2.0 hooks: - id: trailing-whitespace exclude: tests/data @@ -14,38 +14,32 @@ repos: files: requirements-dev.txt - repo: https://github.com/econchick/interrogate - rev: 1.4.0 + rev: 1.5.0 hooks: - id: interrogate exclude: ^(docs|setup.py|tests) args: [--config=pyproject.toml] - repo: https://github.com/PyCQA/flake8 - rev: 3.9.2 + rev: 4.0.1 hooks: - id: flake8 exclude: docs/source/conf.py args: [--max-line-length=105, --ignore=E203] - repo: https://github.com/pycqa/isort - rev: 5.9.3 + rev: 5.10.1 hooks: - id: isort additional_dependencies: [toml] args: ["--profile", "black", "--filter-files"] - repo: https://github.com/psf/black - rev: 21.8b0 + rev: 22.3.0 hooks: - id: black language_version: python3 -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.910 - hooks: - - id: mypy - exclude: docs/source/conf.py - args: [--ignore-missing-imports] - repo: https://github.com/codespell-project/codespell rev: v2.1.0 @@ -55,13 +49,13 @@ repos: - --ignore-words-list=pres,nd,te,afe,ue,wil,tey,te,ot,fo - repo: https://github.com/asottile/pyupgrade - rev: v2.25.0 + rev: v2.32.1 hooks: - id: pyupgrade args: - --py36-plus - repo: https://github.com/asottile/add-trailing-comma - rev: v2.1.0 + rev: v2.2.3 hooks: - id: add-trailing-comma diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d7435ab..0000000 --- a/.travis.yml +++ /dev/null @@ -1,99 +0,0 @@ -dist: xenial - -language: python - -sudo: false - -services: - - xvfb - -env: - global: - - secure: "Tlgd7sOn9XoITQ69wWen9XLLzVqz8IXKtA4B+KXaRP6vTliP6XPrEDldOrOkNYwQFE7P5p57H3irp2fMLIsvxd2JEuN17i+6flB5JrBX+YmVV6tQ/DdDHUUc55CmO1bJefNpWK3kc/Z3TXdKA1k881sTGe/0BHTHV73CYj9ARrs=" - -matrix: - fast_finish: true - include: - - name: "python-3.6" - env: PY=3.6 - - name: "python-3.7" - env: PY=3.7 - - name: "python-3.8" - env: PY=3.8 - - name: "tarball" - env: PY=3 - - name: "docs" - env: PY=3 - - name: "doctest" - env: PY=3 - -before_install: - # Install micromamba and create TEST env. - - | - wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba --strip-components=1 - ./micromamba shell init -s bash -p ~/micromamba - export MAMBA_ROOT_PREFIX=~/micromamba - export MAMBA_EXE=$(pwd)/micromamba - . ${MAMBA_ROOT_PREFIX}/etc/profile.d/mamba.sh - - echo "micromamba version $(micromamba --version)" - micromamba create --yes --name TEST python=$PY pip --file requirements.txt --file requirements-dev.txt --channel conda-forge - micromamba activate TEST - -install: - - pip install -e . --no-deps --force-reinstall - -script: - - | - if [[ $TRAVIS_JOB_NAME == python-* ]]; then - cp -r tests/ /tmp - pushd /tmp - pytest -n 2 -rxs --cov=ctd tests - popd - fi - - - if [[ $TRAVIS_JOB_NAME == "doctest" ]]; then - pytest -s -rxs --doctest-modules -vv ctd ; - fi - - - - if [[ $TRAVIS_JOB_NAME == 'tarball' ]]; then - python setup.py --version ; - pip wheel . -w dist --no-deps ; - check-manifest --verbose ; - twine check dist/* ; - fi - - - | - if [[ $TRAVIS_JOB_NAME == 'docs' ]]; then - set -e - travis_wait jupyter nbconvert --to notebook --execute notebooks/quick_intro.ipynb --output=quick_intro-output.ipynb - mv notebooks/quick_intro-output.ipynb docs/source/ - pushd docs - make clean html linkcheck - popd - if [[ -z "$TRAVIS_TAG" ]]; then - python -m doctr deploy --build-tags --key-path github_deploy_key_pyoceans_python_ctd.enc --built-docs docs/build/html/ dev - else - python -m doctr deploy --build-tags --key-path github_deploy_key_pyoceans_python_ctd.enc --built-docs docs/build/html/ "version-$TRAVIS_TAG" - python -m doctr deploy --build-tags --key-path github_deploy_key_pyoceans_python_ctd.enc --built-docs docs/build/html/ . - fi - fi - -doctr: - require-master: true - sync: false - -deploy: - skip_cleanup: true - provider: pypi - user: "__token__" - password: - secure: "MdQV2uZYrHZ8NAC3jSsEU9BVaYd5OCEvu5+NhglkBDUSLcrmr+dJni0c0BoMCh1D026ynNt1joVxk2euo1Rpbqa1jiMr5ADAvh1VdLjiqdAnr9TVy9dFVk85/qscnP3LWZ1DCiWorujFbSyObsU+grNwjxjvV+7NW0rXrpHNI+8=" - distributions: sdist bdist_wheel - upload_docs: false - on: - repo: pyoceans/python-ctd - tags: true - all_branches: master - condition: '$TRAVIS_JOB_NAME == "tarball"' diff --git a/ctd/processing.py b/ctd/processing.py index 4d9ff51..6582d5e 100644 --- a/ctd/processing.py +++ b/ctd/processing.py @@ -255,10 +255,8 @@ def movingaverage(df, window_size=48): """ Moving average on a data frame or series. - Inputs - ------ - windows_size : integer - Size of the window. + Inputs: + windows_size : integer """ diff --git a/ctd/read.py b/ctd/read.py index 5eae33c..7805f08 100644 --- a/ctd/read.py +++ b/ctd/read.py @@ -13,6 +13,7 @@ from io import StringIO from pathlib import Path +import chardet import gsw import numpy as np import pandas as pd @@ -50,9 +51,7 @@ def _open_compressed(fname): cfile = zfile.open(name) else: raise ValueError( - "Unrecognized file extension. Expected .gzip, .bz2, or .zip, got {}".format( - extension, - ), + f"Unrecognized file extension. Expected .gzip, .bz2, or .zip, got {extension}", ) contents = cfile.read() cfile.close() @@ -60,7 +59,12 @@ def _open_compressed(fname): def _read_file(fname): - """Read file contents.""" + """Read file contents, or read from StringIO object.""" + if isinstance(fname, StringIO): + fname.seek(0) + text = fname.read() + return StringIO(text) + if not isinstance(fname, Path): fname = Path(fname).resolve() @@ -74,7 +78,8 @@ def _read_file(fname): f"Unrecognized file extension. Expected .cnv, .edf, .txt, .ros, or .btl got {extension}", ) # Read as bytes but we need to return strings for the parsers. - text = contents.decode(encoding="utf-8", errors="replace") + encoding = chardet.detect(contents)["encoding"] + text = contents.decode(encoding=encoding, errors="replace") return StringIO(text) @@ -90,9 +95,10 @@ def _remane_duplicate_columns(names): if count > 1: dup.append(item) - second_occurrences = [names[::-1].index(item) for item in dup] + # since we can assume there are only two instances of a word in the list, how about we find the last + # index of an instance, which will be the second occurrence of the item + second_occurrences = [len(names) - names[::-1].index(item) - 1 for item in dup] for idx in second_occurrences: - idx += 1 names[idx] = f"{names[idx]}_" return names @@ -101,6 +107,7 @@ def _parse_seabird(lines, ftype): """Parse searbird formats.""" # Initialize variables. lon = lat = time = None, None, None + fname = None skiprows = 0 metadata = {} @@ -118,6 +125,9 @@ def _parse_seabird(lines, ftype): # Seabird headers starts with *. if line.startswith("*"): header.append(line) + if "FileName" in line: + file_path = line.split("=")[-1].strip() + fname = Path(file_path).stem # Seabird configuration starts with #. if line.startswith("#"): @@ -171,6 +181,7 @@ def _parse_seabird(lines, ftype): names.append("Statistic") metadata.update( { + "name": fname if fname else "unknown", "header": "\n".join(header), "config": "\n".join(config), "names": _remane_duplicate_columns(names), @@ -245,8 +256,9 @@ def from_btl(fname): # Get row types, see what you have: avg, std, min, max or just avg, std. rowtypes = df[df.columns[-1]].unique() # Get times and dates which occur on second line of each bottle. - dates = df.iloc[:: len(rowtypes), 1].reset_index(drop=True) - times = df.iloc[1 :: len(rowtypes), 1].reset_index(drop=True) + date_idx = metadata["names"].index("Date") + dates = df.iloc[:: len(rowtypes), date_idx].reset_index(drop=True) + times = df.iloc[1 :: len(rowtypes), date_idx].reset_index(drop=True) datetimes = dates + " " + times # Fill the Date column with datetimes. @@ -259,7 +271,9 @@ def from_btl(fname): df["Statistic"] = df["Statistic"].str.replace(r"\(|\)", "") # (avg) to avg - name = _basename(fname)[1] + if "name" not in metadata: + name = _basename(fname)[1] + metadata["name"] = str(name) dtypes = { "bpos": int, @@ -280,7 +294,6 @@ def from_btl(fname): warnings.warn("Could not convert %s to float." % column) df["Date"] = pd.to_datetime(df["Date"]) - metadata["name"] = str(name) setattr(df, "_metadata", metadata) return df diff --git a/docs/source/index.rst b/docs/source/index.rst index 24eb5b9..44e8d42 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,7 +7,7 @@ Load and pre-process CTD/XBT casts as pandas DataFrames. .. toctree:: :maxdepth: 3 - quick_intro.ipynb + quick_intro-output.ipynb ctd Indices and tables diff --git a/github_deploy_key_pyoceans_python_ctd.enc b/github_deploy_key_pyoceans_python_ctd.enc deleted file mode 100644 index 468c235..0000000 --- a/github_deploy_key_pyoceans_python_ctd.enc +++ /dev/null @@ -1 +0,0 @@ -gAAAAABcksnrznRahez_EoXZFuM--QLw571JcTmBChG3FHtIT58M9kbyywC55snxWenuEYm2my2Z_9er_DXYRPcCcCJtQzx0jwM-4NkXgcwz5Pq-zolDbRQOK6mtAs7VAQgznj_R34EZLPwdELDbK2xZ0AhLdGCh5vuMkisHGLUrSCPQ3DcCkxuqWEIaADREj5-hVy88lzZ9atzRvCcKm-bdabHOZBCEUKSssQvC99w1zZgYlCfs6Ennqve6QE32rtRS962PiXGnVw1ez-QX2vIb7ZmhZUcOXIos6n_6fUFKbgU1q0ed6N9tY9gEwSoXtbTi1ANhgzneg5bxI1BaC3QgJ90F7VBl2dWwJF_-N1ezX_FwYMGTYrx_tLS0tybQd-M_EEBM_nwWH6MaOZMKV-37rP9ZCTjHBz6SAIadGdsDfHqAjHyXlh10YhWbz7hWHWYySmvca0TYJ6TwD5PiKRf6lgY0N3Ql1OHifd3_tG1f1Pi7BPOFzQYraslspV814QRGooEYF9jMB4OYkODv-OVsfAvBWPlAgjBcl-P5MC1NjAHmZqyIKuBTutZEhgl02b0LNmCgDUMW5DRRfsMf7-BL-FUEoyxRMC2cPr5gW7VYrGet9f7kAIK0St-_4DtZhOM-FXtHH9HK9i29oX7WvYWOatVcqzj722bVIaZ32OUByaSGPQlveWg2zIYClpSpwN5TGJZwLxMnpaK-ZkigC-xJcGDyBsYzDdHoKt_q1iH0pD9GYxT_7yselI2skVi6tKws4pltwsCv-NBEpwTY0deiCM4RWAn0ZQ7fegz3NZpam-Z9Z2v7iDKcnNcLlst7M237LUuo2EwRv695XTpS36CIjA0MlGujrjFKx6SWdMIZHmkxPdh97-cLwVIi6rLOej-MUKcIS2-aq1LSYrn_YwYHEKtDxVPU562Wgzxrwjo64MvvVw5zi6wEZwZjGDFPm5bCivfsF_bcVZqvBxVWSbmBGLgbtOncIt_SQf1jN7MmqQnVtSSEfcgHPV4lXGvqCsFf_3bUfxUjJvE9tfNeeO4h2IYiJcKb1m42Yz4OMNebsBYMRL5RTx8dsnCAUxNl-62fJcqiklx4poLqW4eN0Ky-aYperJbARWhPqPEb26T76KuXE-xXVUFCFUsl2orShoj7KnJCJvS1cHhQNoOISwO8N2iPlteHcOs_NrObyDJ6GkbvKwYHjtdZSfq-4UsIayC0mGHMIerwxQ5ZsIbnjW3_Jz3FBxkNIqU-U2NfZ6QEYGCxgYSu7dMeLM2DO0dcEpRTeVqbVqVsdv5O68KnOzkr_KtpcUcpBTGIRy2ziy6CqSQ2zIQZMpDbFGEob13AO90A8Zm3JPfAommo-zg8XnPv28qv-33hIDEseuaDJq0bLeFlj4IdBtaVzq_l9CUIFZHCsXu2PFQ0xfrPrcu0OQ84gNwFx_gR5GhFK1i46pwcvOCUqiqzMIC7FO-AhMJKqn5dCWoZYloHdpGa7kYz2-Ix4aCelXPVLaC1qQoPZwGO9E6oE7n53ZeinTj77sNY0qZyxGYdsL4GhLW-EO_BwOS8oMehb7sIXItw-dWjb-ohp5HDTOwH0DqEt3T-X6C3o6GeraMdAzGXnxKB0et0Kvpq65bPAcW0yA8jkrjxoYUIW-wnWqo5SMCzpHE4auEj5Ral7-4WID66jkHVhiz7i8qv5YW4foctwPEuDBxeXN_I-tMSNmmkwOCeDzNHA8wx4u9qVJTvhZC0c7MeIqGVC04jIhyjMabrusKCjEjokjtiDa5znICgC4N-lD6xORUhPSqEW5T-IfQg8loAYENpMoTD5V6LGbnu7aKwkbFCTI48w79iRSZjk0RMVhSFWZud1XbJpVB-F7Nr4ItGmNgmU4BSTJgU5bFmQa14YET2xaVBbDBpmqhhNpHvL0wnBKLFMbNJcUyi8hHE64X7rULLIycM6YQdEVlmhT_hJCfTDjg60QSsyY-om-N5H9yUeq0-RceNNdWvij81xghkCBkZnPoT7YxcynAAOdVRHb7HsUMkmit3NcWDum7CE7NNm5-8OrK-V3u4TpvJXORcec7NjHpe8ixUN1S2_ATme5lQz49-TQaDVDETS8EiLa2r2FleIrdKVTAFtF0CspA-_IKarPRFBrc2EUX6nUlxqNdmLeRbtnavVQYmT-nL-cFjtYea5cPSIIj5jB5UWW-gzGv2If45byj10-wxhsoqe6fADs9t16ianPC6BLAANWTyi5oW75FSI4DVmFvDQ98Znmhp0NTS7DzRFDXmrqs-Q1zd52wyOs3tA2zkDRZgcXg6lt0tfF9DAauGQEPNACrkRk-LRsBIP5vV8F0m0myYMB7dpPE_Agt8c0oBx07QAVjUhmWeqbfnd40YEuI91Xnru4AHVHhjRQqZeBcO1Dqz3h7000Wkr0xViAIE4QUxjkXW32fXKI0XqRI9HKLA9O75ISMcEWWBWD_swHDhVM_0P7o3jOgb1I6LpBitoAFopHRaeP0O--iFC__IhkGP_WVCmHjsuik7fFmUoVMFesaP03291NJ9EJviLL8fJ3WUKc4iOYrUqR6M_LE1ykErV3AAMznSjRApp-D4kx0zA_s3vbOyFu2M5P6FW7u9jJ4d2kz0OUUVZUDmlAyombeQSCdYqfnFiNketLFI3YGhaeyxc6V-ZoWJlDAUhVn4WxX2PPLib80iZPqIvKBbnZ1-vGpexcV7JQ9kukfC6gIAS6fnB-qbqDvz4FHvX1ZHLzUeFloS6-xL3KlzwOYug2M4R-JAP9--2eXVzJy3eStkgMUiWixYDIxmoaiSqVaU5kWSdttNs_dmDQK2SOqv_ljDQkwHAdwSJco4McXR_gOYMJnbEmBsTPAkPei1lmqwY8N53c9JgbtGDnD3qq624yeOpy9zZ4nCcjm6EK2_XYe_2Z6ptb_3YtDHzKOgt0DINlWvW4OZlsmJz1pEevOAD9h6c6_Ikro4pePyXVcGOtuP7KpzjzK3JnjWGRc_VfvTmh709PQxdq8wtUdILwWIXHombihk5YOGYlh8i3Xg5O1UOfNSWgsSKgf8VBPYZk3eosEnXKozG_aV2KTnSIc1VWy43kwMYsoRHEYXkIgSgac0iar0GBOFdPhUstu7abp7eBATkIy6bDVyE3Wvfr7zvTV2B6RwBP0o_Cr8HbWL5ZyXu3-FQ_3Bt4PiFKMNqTG1jF0I6xbGSRrXrQXUvwnZxnPQabgkrPReM3gm1BpM7vurX2LYH7eya7dCaKcb9DR9delcYmmLY0Qc_ZPvc1EaTDbGaU5huY_nP1ufCFCPKZA8LIdQygiD8Z37gt9wUVswieszRaiRWGcnRG558bvjk5HOW_vPCcq8KIHsdIGA9Twrv1DLtohTpHHKHaai--juIB94Wyjq3PSe5K6_e5vXUV8KdlB0gsVAHPBM4HnXxJ1rBSbUT6xPLbnHfhRn3aX-Q7VOXQjHCSi_sQN1dxiPf9WqNaub437XHMRlE-IuwWM1aVEJikEagIOJQvQybXmeD9YYOGmOpszLVjrm8R9x6os5JfuQu9PN18PoY9Avpat7Eqr19T4lWzwqVgUwDZZA3hOG8k0nq3fWhLbgkxF9O-PFT3V2eo8TcwQDIHq3IU_CqcjhSzMieAkIjsVdb4AMaf0Lq3XSo3BReNSyjEAjHpGbVb9AlCNKWoTK2xfUrlN8zAunccHz1BxlmlxiO8mZ9zxFeCRZXZxw-lluR1gw9HwypGnWZJLfS9_1Q_oHBquVLEj8zAu43MwYRV--G-nqD0pTY7a0BB3_xXTCc-PRk_UAVbpUVDuGzWmzJN6kXgUVrMCFxA_SX7Oma2NSkXXpzmOofLCtsf_ET2_lfkNYJlG7QTnfowbFbMpxebkSxAWIeUrS8BeGn7KabStxetdxyOVtpY4bUcUwFc9ItlUxyOmzeSOzLm_HOtrsnaH-T4wc3P9X5TFj_uCEw7WqSfTqssl5xhW2Db-9Up8u2rjjg_NwfERHS2eUPYbtT7qCszCV6jVUW5xZkPOMODx-ht7IyqKOyyagb6wbFY6CZhScX_hmMuqnqSY5EzSfF-AyZGlAdc9Kb8ogzoZxGox2coutJHD0VLsX3rDEOofUE1_OKO3T1y288FRd7Df-f4YSztjn8wFIoudq-VaUpmVC498svNmWkh9Q9cOZ3hBxMCI8FzzUdARpfC2aW84bfLaM7IErduNBIvUOVXj3iIxUGb4bJ5xBjKuvrhKowXtA216iSw_iB4ZdUzd26XEUXvoQX5LXQMaKA5hvtf22Sgc3oF0LceW4JAo-cxafLrH1UD_QxH3I8NGlTf2C0Xp8wEDQYvhWlQwHCqHNbIOvRCccpu-KtFqTvnoRddgCasAkH-pYgq94Ne0L028DlVy022H5mtsYr3v9XSGeSQfEXwcZSGXIgorsAjkgN8O8fySi2GzLr00b6lzdIjRXEppzwn0UR6RXKA== diff --git a/notebooks/quick_intro.ipynb b/notebooks/quick_intro.ipynb index e1e2241..6d060b3 100644 --- a/notebooks/quick_intro.ipynb +++ b/notebooks/quick_intro.ipynb @@ -12,7 +12,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [], "source": [ "import io\n", @@ -32,7 +34,7 @@ " Path(\"CTD-spiked-unfiltered.cnv.bz2\").write_bytes(data.read())\n", "\n", "\n", - "url = \"https://github.com/pyoceans/python-ctd/raw/master/tests/data/CTD-spiked-unfiltered.cnv.bz2\"\n", + "url = \"https://github.com/pyoceans/python-ctd/raw/main/tests/data/CTD-spiked-unfiltered.cnv.bz2\"\n", "download_demo_file(url)" ] }, @@ -117,7 +119,7 @@ "ax1.grid(False)\n", "\n", "ax0.legend(loc=\"lower left\")\n", - "ax1.legend(loc=\"lower right\")" + "ax1.legend(loc=\"lower right\");" ] }, { @@ -173,7 +175,7 @@ "ax.plot(down.index, label=\"unfiltered\")\n", "ax.plot(down.lp_filter().index, label=\"filtered\")\n", "ax.axis([20870, 20930, 557.5, 559])\n", - "fig.legend()" + "fig.legend();" ] }, { @@ -234,11 +236,11 @@ "\n", "p = proc.index\n", "\n", - "SP = gsw.SP_from_C(proc[\"c0S/m\"] * 10.0, proc[\"t090C\"], p)\n", + "SP = gsw.SP_from_C(proc[\"c0S/m\"].to_numpy() * 10.0, proc[\"t090C\"].to_numpy(), p)\n", "SA = gsw.SA_from_SP(SP, p, lon, lat)\n", "SR = gsw.SR_from_SP(SP)\n", "\n", - "CT = gsw.CT_from_t(SA, proc[\"t090C\"].values, p)\n", + "CT = gsw.CT_from_t(SA, proc[\"t090C\"].to_numpy(), p)\n", "z = -gsw.z_from_p(p, lat)\n", "sigma0_CT = gsw.sigma0(SA, CT)\n", "\n", @@ -272,7 +274,7 @@ "ax.set_ylabel(\"Pressure (dbar)\")\n", "ax.grid(True)\n", "ax.legend()\n", - "ax.set_title(\"Salinities\")" + "ax.set_title(\"Salinities\");" ] }, { @@ -344,7 +346,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -358,7 +360,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.6" + "version": "3.10.4" } }, "nbformat": 4, diff --git a/requirements.txt b/requirements.txt index fd7d03f..e722ff8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +chardet gsw>=3.3.0 matplotlib numpy diff --git a/setup.cfg b/setup.cfg index 1cf732c..d4ce1de 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,13 +1,13 @@ [metadata] name = ctd -description = Python interface for ERDDAP +description = Tools to load hydrographic data into pandas DataFrame author = Filipe Fernandes author_email = ocefpaf@gmail.com url = https://github.com/pyoceans/python-ctd long_description_content_type = text/markdown long_description = file: README.md license = BSD-3-Clause -license_file = LICENSE.txt +license_files = LICENSE.txt classifiers = Development Status :: 5 - Production/Stable Intended Audience :: Science/Research @@ -15,10 +15,6 @@ classifiers = License :: OSI Approved :: BSD License Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 Topic :: Scientific/Engineering [options] @@ -42,7 +38,6 @@ ignore = .coveragerc docs docs/* - *.enc notebooks notebooks/* tests diff --git a/setup.py b/setup.py index d55eec5..b451531 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,9 @@ from setuptools import setup setup( + # The package metadata is specified in setup.cfg but GitHub's downstream dependency graph + # does not work unless we put the name this here too. + name="ctd", use_scm_version={ "write_to": "ctd/_version.py", "write_to_template": '__version__ = "{version}"', diff --git a/tests/data/btl/alt_bottletest.BTL b/tests/data/btl/alt_bottletest.BTL new file mode 100644 index 0000000..14ab445 --- /dev/null +++ b/tests/data/btl/alt_bottletest.BTL @@ -0,0 +1,335 @@ +* Sea-Bird SBE 9 Data File: +* FileName = C:\CTD_ACQUISITION\2021185HUD\ctddata\185A007.hdr +* Software Version Seasave V 7.26.7.121 +* Temperature SN = 5083 +* Conductivity SN = 3562 +* Number of Bytes Per Scan = 40 +* Number of Voltage Words = 5 +* Number of Scans Averaged by the Deck Unit = 1 +* System UpLoad Time = Sep 17 2021 03:38:32 +* NMEA Latitude = 44 16.00 N +* NMEA Longitude = 063 18.99 W +* NMEA UTC (Time) = Sep 17 2021 03:38:31 +* Store Lat/Lon Data = Append to Every Scan +* SBE 11plus V 5.2 +* number of scans to average = 1 +* pressure baud rate = 9600 +* NMEA baud rate = 9600 +* surface PAR voltage added to scan +* A/D offset = 2 +* Latitude/Longitude added to scan +* GPIB address = 1 +* advance primary conductivity 0.073 seconds +* advance secondary conductivity 0.073 seconds +* autorun on power up is disabled +* S> +** Ship: HUDSON +** Cruise: HUD2021185 +** Chief_Scientist: CHANTELLE LAYTON +** Organization: BIO +** Area_of_Operation: SCOTIAN SHELF +** Cruise_Description: ATLANTIC ZONE MONITORING PROGRAM (AZMP) +** Station_Name: 007 +** Event_Number: 007 +** Sounding: 148 +** Event_Comments: HL_02,488275 +* System UTC = Sep 17 2021 03:38:32 +* Surface_Pressure_Offset = 0.774, Temperature = 18.5245, Scan = 5463.0 +* Near_Surface_Scan = 5487.0, P = 0.878, T = 18.5461, S = 22.1669 +# interval = seconds: 0.0416667 +# start_time = Sep 17 2021 03:38:31 [NMEA time, header] +# +# +# +# +# 5083 +# 06-Nov-2020 +# +# 0 +# 3.68121154e-003 +# 5.97282277e-004 +# 1.50761235e-005 +# 2.04865165e-006 +# 2984.665 +# 4.35484023e-003 +# 6.37393260e-004 +# 2.17650031e-005 +# 2.04721418e-006 +# 1000.000 +# 1.00000000 +# 0.0000 +# +# +# +# +# +# 3562 +# 10-Nov-2020 +# 1 +# +# 0.0000 +# 2000.0000 +# 0 +# +# 0.00000000e+000 +# 0.00000000e+000 +# 0.00000000e+000 +# 0.00000000e+000 +# 0.0 +# -9.57000000e-008 +# +# +# -9.86623986e+000 +# 1.20518686e+000 +# -1.87726883e-003 +# 1.83608664e-004 +# -9.57000000e-008 +# 3.2500e-006 +# +# 0.00000000e+000 +# +# 1.00024000 +# -0.00124 +# +# +# +# +# +# 50601-0370 +# 20-Jan-2021 +# -4.274542e+004 +# 1.040996e+000 +# 1.266000e-002 +# 4.087300e-002 +# 0.000000e+000 +# 3.009606e+001 +# -6.521164e-005 +# 4.345040e-006 +# 2.428830e-009 +# 0.99996291 +# -0.35347 +# 0.000000e+000 +# 1.289670e-002 +# -8.390788e+000 +# +# +# +# +# +# 5081 +# 02-Dec-2020 +# +# 0 +# 3.68050257e-003 +# 6.01375916e-004 +# 1.56245547e-005 +# 2.05531937e-006 +# 3246.733 +# 4.41352149e-003 +# 6.46511212e-004 +# 2.28532525e-005 +# 2.05383168e-006 +# 1000.000 +# 1.00000000 +# 0.0000 +# +# +# +# +# +# 3561 +# 10-Nov-2020 +# 1 +# +# 0.0000 +# 2000.0000 +# 0 +# +# 0.00000000e+000 +# 0.00000000e+000 +# 0.00000000e+000 +# 0.00000000e+000 +# 0.0 +# -9.57000000e-008 +# +# +# -1.03650868e+001 +# 1.25241992e+000 +# -2.47281184e-003 +# 2.28722834e-004 +# -9.57000000e-008 +# 3.2500e-006 +# +# 0.00000000e+000 +# +# 1.00008000 +# -0.00073 +# +# +# +# +# +# 59017 +# 01-Mar-2017 +# 15.000 +# 0.000 +# +# +# +# +# +# 1043 +# 01-Dec-2015 +# 1.03320000 +# 0.80737000 +# 1.35890000 +# 1.00000000 +# +# +# +# +# +# 2524 +# 23-Dec-2020 +# 1 +# +# +# 0.0000 +# 0.0000e+000 +# 0.0000 +# 0.00e+000 +# 0.0000 +# 0.0 +# +# +# +# 5.0830e-001 +# -0.5100 +# -3.9982e-003 +# 1.9704e-004 +# -3.0858e-006 +# 2.5826e+000 +# 1.92634e-004 +# -4.64803e-002 +# 3.6000e-002 +# 1.0200 +#

-3.3000e-002

+#

5.0000e+003

+#

1.4500e+003

+# +# +# +# +# +# +# 3026 +# 23-Dec-2020 +# 1 +# +# +# 0.0000 +# 0.0000e+000 +# 0.0000 +# 0.00e+000 +# 0.0000 +# 0.0 +# +# +# +# 5.1700e-001 +# -0.5144 +# -4.6229e-003 +# 2.2368e-004 +# -3.3916e-006 +# 2.5826e+000 +# 1.92634e-004 +# -4.64803e-002 +# 3.6000e-002 +# 1.6800 +#

-3.3000e-002

+#

5.0000e+003

+#

1.4500e+003

+# +# +# +# +# +# +# 3668 +# 1-Jan-2015 +# 50.0 +# 0.0000 +# +# +# +# +# +# 6210 +# 1-Jan-2015 +# +# 2 +# 0.000 +# +# +# +# +# +# 1221 +# 16-Nov-2020 +# 4.6226 +# 2.4930 +# +# +# +# +# +# 1490 +# 9-Aug-2016 +# 2.983e-003 +# +# 4.800e-002 +# +# +# +# +# +# +# +# +# 1069 +# 24-Jun-2016 +# 1.00000000 +# 1.00000000 +# +# +# +# datcnv_date = Jan 13 2022 08:44:01, 7.26.6.28 +# datcnv_in = C:\DEV\Data\2021\HUD2021185\CTD\DATASHOP_PROCESSING\Step_0_Recalibrated_Data\CTDDATA\185a007.hex C:\DEV\Data\2021\HUD2021185\CTD\DATASHOP_PROCESSING\Step_0_Recalibrated_Data\HUD2021185_Events_001_to_117_Corrected.xmlcon +# datcnv_ox_hysteresis_correction = yes +# datcnv_bottle_scan_range_source = BL file +# datcnv_scans_per_bottle = 49 +# bottlesum_date = Jan 13 2022 08:51:21, 7.26.6.28 +# bottlesum_in = C:\DEV\Data\2021\HUD2021185\CTD\DATASHOP_PROCESSING\Step_0_Recalibrated_Data\CTDDATA\185a007.ros C:\DEV\Data\2021\HUD2021185\CTD\DATASHOP_PROCESSING\Step_0_Recalibrated_Data\HUD2021185_Events_001_to_117_Corrected.xmlcon C:\DEV\Data\2021\HUD2021185\CTD\DATASHOP_PROCESSING\Step_0_Recalibrated_Data\CTDDATA\185a007.BL +# bottlesum_ox_tau_correction = yes + Bottle Bottle Date Sbeox0ML/L Sbeox1ML/L Sal00 Sal11 Potemp068C Potemp168C Sigma-é00 Sigma-é11 Scan TimeS PrDM T068C C0S/m T168C C1S/m AltM Par/log Sbeox0V Sbeox1V FlSPuv0 FlSP Ph TurbWETbb0 Spar Latitude Longitude + Position S/N Time + 1 488275 Sep 17 2021 3.6759 3.6754 34.4673 34.4748 9.6059 9.6130 26.6026 26.6073 17690 737.042 144.655 9.6221 3.727925 9.6292 3.729318 6.26 1.0039e-01 1.6505 1.6371 2.5043 3.6630e-02 7.910 9.4191e-04 1.2454e-01 44.26730 -63.31760 (avg) + 03:50:48 14 0.595 0.079 0.0085 0.001062 0.0031 0.000408 0.12 0.0000e+00 0.0012 0.0002 0.0195 0.0000e+00 0.000 7.2788e-06 0.0000e+00 0.00001 0.00000 (sdev) + 2 488276 Sep 17 2021 5.0755 5.0691 33.2387 33.2423 5.6803 5.6778 26.2002 26.2033 21827 909.417 99.550 5.6883 3.257440 5.6858 3.257537 52.24 1.0039e-01 1.9333 1.9181 2.6738 5.3076e-02 7.880 3.9156e-04 1.2454e-01 44.26736 -63.31802 (avg) + 03:53:41 14 0.595 0.087 0.0009 0.000153 0.0069 0.000574 0.13 0.0000e+00 0.0006 0.0005 0.0156 1.3317e-02 0.000 5.4324e-05 0.0000e+00 0.00000 0.00000 (sdev) + 3 488277 Sep 17 2021 5.3913 5.4081 32.9032 32.9035 4.6559 4.6443 26.0508 26.0523 24106 1004.375 79.923 4.6616 3.138637 4.6500 3.137670 72.04 1.0039e-01 1.9841 1.9672 2.7567 6.2047e-02 7.864 3.3767e-04 1.2454e-01 44.26739 -63.31814 (avg) + 03:55:16 14 0.595 0.043 0.0036 0.000278 0.0053 0.000398 0.07 0.0000e+00 0.0005 0.0008 0.0272 7.8169e-03 0.000 4.4882e-05 0.0000e+00 0.00001 0.00000 (sdev) + 4 488278 Sep 17 2021 5.7698 5.7694 32.6090 32.6111 4.4102 4.4091 25.8432 25.8450 25970 1082.042 60.305 4.4144 3.091459 4.4133 3.091541 92.03 1.0039e-01 2.0769 2.0589 2.7874 9.1949e-02 7.879 2.9210e-04 1.2454e-01 44.26744 -63.31828 (avg) + 03:56:33 14 0.595 0.049 0.0023 0.000140 0.0023 0.000148 0.17 0.0000e+00 0.0005 0.0002 0.0141 9.6856e-03 0.000 6.1380e-06 0.0000e+00 0.00000 0.00000 (sdev) + 5 488279 Sep 17 2021 6.0206 6.0367 32.4310 32.4342 4.6414 4.6477 25.6778 25.6796 27678 1153.208 49.913 4.6449 3.095198 4.6512 3.096004 99.85 1.0039e-01 2.1551 2.1376 2.8394 1.7219e-01 7.902 2.9783e-04 1.2454e-01 44.26750 -63.31842 (avg) + 03:57:45 14 0.595 0.056 0.0008 0.000077 0.0028 0.000162 0.00 0.0000e+00 0.0005 0.0006 0.0180 1.3246e-02 0.000 2.0868e-05 0.0000e+00 0.00000 0.00000 (sdev) + 6 488280 Sep 17 2021 6.3072 6.3435 32.3181 32.3333 5.6782 5.7907 25.4725 25.4712 29213 1217.167 39.880 5.6813 3.172728 5.7938 3.183652 84.00 1.0039e-01 2.2814 2.2714 2.8621 3.7502e-01 7.951 2.4341e-04 1.2454e-01 44.26752 -63.31854 (avg) + 03:58:49 14 0.595 0.064 0.0377 0.002520 0.0327 0.003665 0.26 0.0000e+00 0.0004 0.0003 0.0185 1.9625e-02 0.000 3.3274e-06 0.0000e+00 0.00000 0.00000 (sdev) + 7 488281 Sep 17 2021 6.7603 6.7842 32.0851 32.0996 8.7174 8.7422 24.8799 24.8875 30896 1287.292 30.241 8.7205 3.412039 8.7453 3.415584 66.55 1.0039e-01 2.5552 2.5399 2.8983 2.4445e+00 8.035 3.2280e-04 1.2454e-01 44.26758 -63.31868 (avg) + 03:59:59 14 0.595 0.051 0.0255 0.003327 0.0049 0.000342 0.41 0.0000e+00 0.0014 0.0004 0.0179 7.8550e-02 0.000 7.1470e-06 0.0000e+00 0.00000 0.00000 (sdev) + 8 488282 Sep 17 2021 6.0264 6.0183 31.1459 31.1437 15.9320 15.9964 22.8012 22.7852 32452 1352.125 19.796 15.9350 3.949537 15.9994 3.955038 50.57 1.0039e-01 2.6323 2.6101 2.3874 1.2626e+00 8.115 2.9396e-04 1.2454e-01 44.26764 -63.31880 (avg) + 04:01:03 14 0.595 0.113 0.0398 0.002443 0.0365 0.001873 0.42 0.0000e+00 0.0034 0.0018 0.0185 6.3196e-02 0.000 1.1737e-06 0.0000e+00 0.00000 0.00000 (sdev) + 9 488283 Sep 17 2021 5.5867 5.6037 30.9038 30.9052 18.4210 18.4651 22.0370 22.0274 34129 1422.000 10.032 18.4227 4.144277 18.4669 4.148441 33.49 1.0039e-01 2.5785 2.5570 2.1243 9.1425e-01 8.174 2.4044e-04 1.2454e-01 44.26764 -63.31886 (avg) + 04:02:13 14 0.595 0.052 0.0032 0.000211 0.0044 0.000253 0.23 0.0000e+00 0.0030 0.0011 0.0179 4.1454e-02 0.000 6.3239e-06 0.0000e+00 0.00000 0.00000 (sdev) + 10 488284 Sep 17 2021 5.6037 5.6054 30.8985 30.9007 18.5010 18.5023 22.0135 22.0149 37066 1544.375 2.806 18.5015 4.150469 18.5028 4.150851 99.85 1.2766e-01 2.5834 2.5645 2.0742 8.1981e-01 8.202 2.3450e-04 1.2454e-01 44.26768 -63.31892 (avg) + 04:04:16 14 0.595 0.082 0.0004 0.000052 0.0012 0.000096 0.00 3.3264e-02 0.0020 0.0000 0.0212 4.1331e-02 0.000 5.7686e-06 0.0000e+00 0.00000 0.00000 (sdev) diff --git a/tests/test_read.py b/tests/test_read.py index fab8f83..3a8b331 100644 --- a/tests/test_read.py +++ b/tests/test_read.py @@ -53,6 +53,13 @@ def btl(): yield ctd.from_btl(data_path.joinpath("btl", "bottletest.btl")) +@pytest.fixture +def btl_as_stream(): + file = open(mode="rb", file=data_path.joinpath("btl", "alt_bottletest.BTL")) + stream = StringIO(file.read().decode("cp1252")) + yield ctd.from_btl(stream) + + @pytest.fixture def ros(): yield ctd.rosette_summary(data_path.joinpath("CTD", "g01l03s01m-m2.ros")) @@ -78,6 +85,15 @@ def test_btl_is_dataframe(btl): assert not btl.empty +def test_btl_with_dup_cols(btl_as_stream): + assert all(col in btl_as_stream.columns for col in ["Bottle", "Bottle_"]) + + +def test_btl_as_stringio(btl_as_stream): + assert isinstance(btl_as_stream, pd.DataFrame) + assert not btl_as_stream.empty + + def test_ros_is_dataframe(ros): assert isinstance(ros, pd.DataFrame) assert not ros.empty