From 3d36094d6b3dc93d6e8da80903de54599381b888 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 14 Jun 2021 14:06:58 +0200 Subject: [PATCH 1/7] Add Windows to Github Action CI matrix --- .../workflows/{ubuntu-test.yml => test.yml} | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) rename .github/workflows/{ubuntu-test.yml => test.yml} (76%) diff --git a/.github/workflows/ubuntu-test.yml b/.github/workflows/test.yml similarity index 76% rename from .github/workflows/ubuntu-test.yml rename to .github/workflows/test.yml index 41cc155ac..aeaf8f53a 100644 --- a/.github/workflows/ubuntu-test.yml +++ b/.github/workflows/test.yml @@ -3,13 +3,14 @@ name: Tests on: [push, pull_request] jobs: - ubuntu: - - runs-on: ubuntu-latest + test: + name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}) + runs-on: ${{ matrix.os }} strategy: matrix: python-version: [3.6, 3.7, 3.8] scikit-learn: [0.21.2, 0.22.2, 0.23.1, 0.24] + os: [ubuntu-latest] exclude: # no scikit-learn 0.21.2 release for Python 3.8 - python-version: 3.8 scikit-learn: 0.21.2 @@ -24,6 +25,8 @@ jobs: - python-version: 3.8 scikit-learn: 0.23.1 code-cov: true + - os: windows-latest + scikit-learn: 0.24.* fail-fast: false max-parallel: 4 @@ -32,6 +35,7 @@ jobs: with: fetch-depth: 2 - name: Setup Python ${{ matrix.python-version }} + if: {{ matrix.os != windows-latest }} # windows-latest only uses preinstalled Python (3.7.9) uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -50,10 +54,15 @@ jobs: id: status-before run: | echo "::set-output name=BEFORE::$(git status --porcelain -b)" - - name: Run tests + - name: Run tests on Ubuntu + if: ${{ matrix.os == ubuntu-latest }} run: | if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov --reruns 5 --reruns-delay 1 + - name: Run tests on Windows + if: ${{ matrix.os == windows-latest }} + run: | # we need a separate step because of the bash-specific if-statement in the previous one. + pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv --reruns 5 --reruns-delay 1 - name: Check for files left behind by test if: ${{ always() }} run: | @@ -71,4 +80,4 @@ jobs: with: files: coverage.xml fail_ci_if_error: true - verbose: true \ No newline at end of file + verbose: true From 0f393a81166a9543a7b8fc25864910ceba38e547 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 14 Jun 2021 16:25:52 +0200 Subject: [PATCH 2/7] Fix syntax, disable Ubuntu tests Ubuntu tests only temporarily disabled for this PR, to avoid unnecessary computational costs/time. --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aeaf8f53a..dba6148fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,12 +55,12 @@ jobs: run: | echo "::set-output name=BEFORE::$(git status --porcelain -b)" - name: Run tests on Ubuntu - if: ${{ matrix.os == ubuntu-latest }} + if: false # matrix.os == 'ubuntu-latest' temporarily disable ubuntu tests to free capacity for windows tests during development run: | if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov --reruns 5 --reruns-delay 1 - name: Run tests on Windows - if: ${{ matrix.os == windows-latest }} + if: matrix.os == 'windows-latest' run: | # we need a separate step because of the bash-specific if-statement in the previous one. pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv --reruns 5 --reruns-delay 1 - name: Check for files left behind by test From 6f8f6adf740a2f11a183f7445a92add6d5b4f43d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 14 Jun 2021 16:53:29 +0200 Subject: [PATCH 3/7] Fix syntax for skip on install Python step --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dba6148fa..415f0f8e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: with: fetch-depth: 2 - name: Setup Python ${{ matrix.python-version }} - if: {{ matrix.os != windows-latest }} # windows-latest only uses preinstalled Python (3.7.9) + if: matrix.os != 'windows-latest' # windows-latest only uses preinstalled Python (3.7.9) uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From 3df7f0d75f3624bbd8fcbe21e1c318f87693e83b Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 14 Jun 2021 16:58:12 +0200 Subject: [PATCH 4/7] Explicitly add the OS to includes --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 415f0f8e9..c6fdf01f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,13 +18,17 @@ jobs: - python-version: 3.6 scikit-learn: 0.18.2 scipy: 1.2.0 + os: ubuntu-latest - python-version: 3.6 scikit-learn: 0.19.2 + os: ubuntu-latest - python-version: 3.6 scikit-learn: 0.20.2 + os: ubuntu-latest - python-version: 3.8 scikit-learn: 0.23.1 code-cov: true + os: ubuntu-latest - os: windows-latest scikit-learn: 0.24.* fail-fast: false From 77bfec56c48ffed629461f7ff4cbef1b6a859a8c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 15 Jun 2021 08:56:01 +0200 Subject: [PATCH 5/7] Disable check for files left behind for Windows The check is bash script, which means it fails on a Windows machine. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c6fdf01f2..62fc98d8c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: run: | # we need a separate step because of the bash-specific if-statement in the previous one. pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv --reruns 5 --reruns-delay 1 - name: Check for files left behind by test - if: ${{ always() }} + if: matrix.os != 'windows-latest' && always() run: | before="${{ steps.status-before.outputs.BEFORE }}" after="$(git status --porcelain -b)" From 6ac31b858b94a4ddecc680882282aeee83a05587 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 15 Jun 2021 09:14:29 +0200 Subject: [PATCH 6/7] Re-enable Ubuntu tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 62fc98d8c..059aec58d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,7 +59,7 @@ jobs: run: | echo "::set-output name=BEFORE::$(git status --porcelain -b)" - name: Run tests on Ubuntu - if: false # matrix.os == 'ubuntu-latest' temporarily disable ubuntu tests to free capacity for windows tests during development + if: matrix.os == 'ubuntu-latest' run: | if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov --reruns 5 --reruns-delay 1 From 9532d8cce29f14579a207eb7d9f0e9f99a2a33fe Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 15 Jun 2021 09:21:38 +0200 Subject: [PATCH 7/7] Replace Appveyor with Github Actions for WindowsCI --- appveyor.yml | 48 --------------- appveyor/run_with_env.cmd | 88 --------------------------- doc/progress.rst | 6 ++ tests/test_runs/test_run_functions.py | 5 +- 4 files changed, 7 insertions(+), 140 deletions(-) delete mode 100644 appveyor.yml delete mode 100644 appveyor/run_with_env.cmd diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index e3fa74aaf..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,48 +0,0 @@ -clone_folder: C:\\projects\\openml-python - -environment: -# global: -# CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\scikit-learn-contrib\\run_with_env.cmd" - - matrix: - - PYTHON: "C:\\Python3-x64" - PYTHON_VERSION: "3.6" - PYTHON_ARCH: "64" - MINICONDA: "C:\\Miniconda36-x64" - -matrix: - fast_finish: true - - -install: - # Miniconda is pre-installed in the worker build - - "SET PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" - - "python -m pip install -U pip" - - # Check that we have the expected version and architecture for Python - - "python --version" - - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" - - "pip --version" - - # Remove cygwin because it clashes with conda - # see http://help.appveyor.com/discussions/problems/3712-git-remote-https-seems-to-be-broken - - rmdir C:\\cygwin /s /q - - # Update previous packages and install the build and runtime dependencies of the project. - - conda update conda --yes - - conda update --all --yes - - # Install the build and runtime dependencies of the project. - - "cd C:\\projects\\openml-python" - - "pip install .[examples,test]" - - "pip install scikit-learn==0.21" - # Uninstall coverage, as it leads to an error on appveyor - - "pip uninstall -y pytest-cov" - - -# Not a .NET project, we build scikit-learn in the install step instead -build: false - -test_script: - - "cd C:\\projects\\openml-python" - - "%CMD_IN_ENV% pytest -n 4 --timeout=600 --timeout-method=thread --dist load -sv" diff --git a/appveyor/run_with_env.cmd b/appveyor/run_with_env.cmd deleted file mode 100644 index 5da547c49..000000000 --- a/appveyor/run_with_env.cmd +++ /dev/null @@ -1,88 +0,0 @@ -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) -:: -:: To build extensions for 64 bit Python 2, we need to configure environment -:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) -:: -:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific -:: environment configurations. -:: -:: Note: this script needs to be run with the /E:ON and /V:ON flags for the -:: cmd interpreter, at least for (SDK v7.0) -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows -:: http://stackoverflow.com/a/13751649/163740 -:: -:: Author: Olivier Grisel -:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ -:: -:: Notes about batch files for Python people: -:: -:: Quotes in values are literally part of the values: -:: SET FOO="bar" -:: FOO is now five characters long: " b a r " -:: If you don't want quotes, don't include them on the right-hand side. -:: -:: The CALL lines at the end of this file look redundant, but if you move them -:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y -:: case, I don't know why. -@ECHO OFF - -SET COMMAND_TO_RUN=%* -SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows -SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf - -:: Extract the major and minor versions, and allow for the minor version to be -:: more than 9. This requires the version number to have two dots in it. -SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% -IF "%PYTHON_VERSION:~3,1%" == "." ( - SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% -) ELSE ( - SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% -) - -:: Based on the Python version, determine what SDK version to use, and whether -:: to set the SDK for 64-bit. -IF %MAJOR_PYTHON_VERSION% == 2 ( - SET WINDOWS_SDK_VERSION="v7.0" - SET SET_SDK_64=Y -) ELSE ( - IF %MAJOR_PYTHON_VERSION% == 3 ( - SET WINDOWS_SDK_VERSION="v7.1" - IF %MINOR_PYTHON_VERSION% LEQ 4 ( - SET SET_SDK_64=Y - ) ELSE ( - SET SET_SDK_64=N - IF EXIST "%WIN_WDK%" ( - :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ - REN "%WIN_WDK%" 0wdf - ) - ) - ) ELSE ( - ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" - EXIT 1 - ) -) - -IF %PYTHON_ARCH% == 64 ( - IF %SET_SDK_64% == Y ( - ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture - SET DISTUTILS_USE_SDK=1 - SET MSSdk=1 - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 - ) ELSE ( - ECHO Using default MSVC build environment for 64 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 - ) -) ELSE ( - ECHO Using default MSVC build environment for 32 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) diff --git a/doc/progress.rst b/doc/progress.rst index b0c182e05..937c60eb2 100644 --- a/doc/progress.rst +++ b/doc/progress.rst @@ -6,6 +6,12 @@ Changelog ========= +0.13.0 +~~~~~~ + + * MAIN#1088: Do CI for Windows on Github Actions instead of Appveyor. + + 0.12.2 ~~~~~~ diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index c8f1729b7..b02b18880 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -366,10 +366,7 @@ def _check_sample_evaluations( evaluation = sample_evaluations[measure][rep][fold][sample] self.assertIsInstance(evaluation, float) if not (os.environ.get("CI_WINDOWS") or os.name == "nt"): - # Either Appveyor is much faster than Travis - # and/or measurements are not as accurate. - # Either way, windows seems to get an eval-time - # of 0 sometimes. + # Windows seems to get an eval-time of 0 sometimes. self.assertGreater(evaluation, 0) self.assertLess(evaluation, max_time_allowed)