diff --git a/.github/workflows/regression_tests_cpu_binaries.yml b/.github/workflows/regression_tests_cpu_binaries.yml new file mode 100644 index 0000000000..9f58fcf341 --- /dev/null +++ b/.github/workflows/regression_tests_cpu_binaries.yml @@ -0,0 +1,45 @@ +name: Run Regression Tests for CPU nightly binaries + +on: + # run every day at 6:15am + schedule: + - cron: '15 6 * * *' + merge_group: + +concurrency: + group: ci-cpu-${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} + cancel-in-progress: true + +jobs: + regression-cpu-nightly-binaries: + # creates workflows for OS: ubuntu, macOS + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, macOS-latest] + python-version: ["3.8", "3.9", "3.10"] + binaries: ["pypi", "conda"] + steps: + - uses: actions/checkout@v3 + - name: Setup conda with Python ${{ matrix.python-version }} + uses: s-weigand/setup-conda@v1 + with: + update-conda: true + python-version: ${{ matrix.python-version }} + conda-channels: anaconda, conda-forge + - run: conda --version + - run: python --version + - name: Setup Java 17 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '17' + - name: Checkout TorchServe + uses: actions/checkout@v3 + - name: Install dependencies + run: | + python ts_scripts/install_dependencies.py --environment=dev + - name: Torchserve Regression Tests + run: | + python test/regression_tests.py --binaries --${{ matrix.binaries }} --nightly diff --git a/.github/workflows/regression_tests_gpu_binaries.yml b/.github/workflows/regression_tests_gpu_binaries.yml new file mode 100644 index 0000000000..a0fc8c0083 --- /dev/null +++ b/.github/workflows/regression_tests_gpu_binaries.yml @@ -0,0 +1,56 @@ +name: Run Regression Tests for GPU nightly binaries + +on: + # run every day at 6:15am + schedule: + - cron: '15 6 * * *' + merge_group: + +concurrency: + group: ci-cpu-${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} + cancel-in-progress: true + +jobs: + regression-gpu-nightly-binaries: + # creates workflows for OS: ubuntu, macOS + runs-on: [self-hosted, regression-test-gpu] + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10"] + binaries: ["pypi", "conda"] + steps: + - name: Clean up previous run + run: | + echo "Cleaning up previous run" + ls -la ./ + sudo rm -rf ./* || true + sudo rm -rf ./.??* || true + ls -la ./ + - name: Setup Java 17 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '17' + - name: Checkout TorchServe + uses: actions/checkout@v3 + - uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + - name: Setup Conda + uses: s-weigand/setup-conda@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Setup Python ${{ matrix.python_version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - run: conda --version + - run: python --version + - name: Install dependencies + run: | + python ts_scripts/install_dependencies.py --environment=dev --cuda=cu118 + - name: Torchserve Regression Tests + run: | + python test/regression_tests.py --binaries --${{ matrix.binaries }} --nightly diff --git a/test/regression_tests.py b/test/regression_tests.py index a4f0bdd468..afb29748da 100644 --- a/test/regression_tests.py +++ b/test/regression_tests.py @@ -1,3 +1,4 @@ +import argparse import os import sys @@ -11,26 +12,77 @@ from ts_scripts.api_utils import test_api from ts_scripts.install_from_src import install_from_src from ts_scripts.regression_utils import test_regression -from ts_scripts.utils import check_python_version +from ts_scripts.utils import check_python_version, try_and_handle -now = datetime.datetime.now() -print("Current date and time : " + now.strftime("%Y-%m-%d %H:%M:%S")) -check_python_version() +def regression_tests(binaries, pypi, conda, nightly): + now = datetime.datetime.now() + print("Current date and time : " + now.strftime("%Y-%m-%d %H:%M:%S")) -# Install from source -install_from_src() + check_python_version() -# Generate mar file -mg.generate_mars() + if binaries: + if pypi: + if nightly: + cmd = f"pip install torchserve-nightly torch-model-archiver-nightly torch-workflow-archiver-nightly" + else: + cmd = f"pip install torchserve torch-model-archiver torch-workflow-archiver" + elif conda: + if nightly: + cmd = f"conda install -c pytorch-nightly torchserve torch-model-archiver torch-workflow-archiver" + else: + cmd = f"conda install -c pytorch torchserve torch-model-archiver torch-workflow-archiver" + try_and_handle(cmd, False) + print(f"## In directory: {os.getcwd()}; Executing command: {cmd}") + else: + # Install from source + install_from_src() -# Run newman api tests -test_api( - "all" -) # "all" > management, inference, increased_timeout_inference, https collections + # Generate mar file + mg.generate_mars() -# Run regression tests -test_regression() + # Run newman api tests + test_api( + "all" + ) # "all" > management, inference, increased_timeout_inference, https collections -# delete mar_gen_dir -mg.delete_model_store_gen_dir() + # Run regression tests + test_regression() + + # delete mar_gen_dir + mg.delete_model_store_gen_dir() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Regression tests for TorchServe") + parser.add_argument( + "--nightly", + action="store_true", + required=False, + help="Run regression tests using nightly binaries", + ) + + parser.add_argument( + "--binaries", + action="store_true", + required=False, + help="Run regression tests using binaries", + ) + + parser.add_argument( + "--pypi", + action="store_true", + required=False, + help="Run regression tests using pypi", + ) + + parser.add_argument( + "--conda", + action="store_true", + required=False, + help="Run regression tests using conda", + ) + + args = parser.parse_args() + + regression_tests(args.binaries, args.pypi, args.conda, args.nightly)