From 436b3147e48ea8919dcd431c3a887279a41aab73 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Tue, 6 Apr 2021 15:18:44 +0200
Subject: [PATCH 01/13] Add a non-functional entry point

---
 .flake8       |  1 +
 openml/cli.py | 33 +++++++++++++++++++++++++++++++++
 setup.py      |  1 +
 3 files changed, 35 insertions(+)
 create mode 100644 openml/cli.py

diff --git a/.flake8 b/.flake8
index 08bb8ea10..211234f22 100644
--- a/.flake8
+++ b/.flake8
@@ -5,6 +5,7 @@ select = C,E,F,W,B,T
 ignore = E203, E402, W503
 per-file-ignores =
     *__init__.py:F401
+    *cli.py:T001
 exclude =
     venv
     examples
diff --git a/openml/cli.py b/openml/cli.py
new file mode 100644
index 000000000..8f20b3beb
--- /dev/null
+++ b/openml/cli.py
@@ -0,0 +1,33 @@
+"""" Command Line Interface for `openml` to configure its settings. """
+
+import argparse
+
+# from openml import config
+
+
+def configure(args: argparse.Namespace):
+    """ Configures the openml configuration file. """
+    print("Configuring", args.file)
+
+    # check if API key exists, if so ask to overwrite.
+
+
+def main() -> None:
+    subroutines = {"configure": configure}
+
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(dest="subroutine")
+
+    parser_configure = subparsers.add_parser(
+        "configure", description="Set or read variables in your configuration file.",
+    )
+    parser_configure.add_argument(
+        "file", default="~/.openml/config", help="The configuration file to edit or read."
+    )
+
+    args = parser.parse_args()
+    subroutines.get(args.subroutine, lambda _: parser.print_help())(args)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/setup.py b/setup.py
index dc1a58863..6983bad40 100644
--- a/setup.py
+++ b/setup.py
@@ -102,4 +102,5 @@
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
     ],
+    entry_points={"console_scripts": ["openml=openml.cli:main"]},
 )

From 0f812f2f869127d69cbbd4806dcaaed706c6dfd3 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Thu, 8 Apr 2021 10:42:48 +0200
Subject: [PATCH 02/13] Allow setting of API key through CLI

 - Add function to set any field in the configuration file
 - Add function to read out the configuration file
 - Towards full configurability from CLI
---
 openml/cli.py    | 108 ++++++++++++++++++++++++++++++++++++++++++++---
 openml/config.py |  47 +++++++++++++++++----
 2 files changed, 142 insertions(+), 13 deletions(-)

diff --git a/openml/cli.py b/openml/cli.py
index 8f20b3beb..496344d35 100644
--- a/openml/cli.py
+++ b/openml/cli.py
@@ -1,15 +1,95 @@
 """" Command Line Interface for `openml` to configure its settings. """
 
 import argparse
+import string
+from typing import Union, Callable
 
-# from openml import config
+
+from openml import config
+
+
+def is_hex(string_: str) -> bool:
+    return all(c in string.hexdigits for c in string_)
+
+
+def looks_like_api_key(apikey: str) -> bool:
+    return len(apikey) == 32 and is_hex(apikey)
+
+
+def wait_until_valid_input(
+    prompt: str,
+    check: Callable[[str], bool],
+    error_message: Union[str, Callable[[str], str]] = "That is not a valid response.",
+) -> str:
+    """  Asks `prompt` until an input is received which returns True for `check`.
+
+    Parameters
+    ----------
+    prompt: str
+        message to display
+    check: Callable[[str], bool]
+        function to call with the given input, should return true only if the input is valid.
+    error_message: Union[str, Callable[[str], str]
+        a message to display on invalid input, or a `str`->`str` function that can give feedback
+        specific to the error.
+
+    Returns
+    -------
+
+    """
+
+    response = input(prompt)
+    while not check(response):
+        if isinstance(error_message, str):
+            print(error_message)
+        else:
+            print(error_message(response), end="\n\n")
+        response = input(prompt)
+
+    return response
+
+
+def print_configuration():
+    file = config.determine_config_file_path()
+    header = f"File '{file}' contains (or defaults to):"
+    print(header)
+
+    max_key_length = max(map(len, config.get_config_as_dict()))
+    for field, value in config.get_config_as_dict().items():
+        print(f"{field.ljust(max_key_length)}: {value}")
+
+
+def configure_apikey() -> None:
+    print(f"\nYour current API key is set to: '{config.apikey}'")
+    print("You can get an API key at https://new.openml.org")
+    print("You must create an account if you don't have one yet.")
+    print("  1. Log in with the account.")
+    print("  2. Navigate to the profile page (top right circle > Your Profile). ")
+    print("  3. Click the API Key button to reach the page with your API key.")
+    print("If you have any difficulty following these instructions, please let us know on Github.")
+
+    def apikey_error(apikey: str) -> str:
+        if len(apikey) != 32:
+            return f"The key should contain 32 characters but contains {len(apikey)}."
+        if not is_hex(apikey):
+            return "Some characters are not hexadecimal."
+        return "This does not look like an API key."
+
+    response = wait_until_valid_input(
+        prompt="Please enter your API key:", check=looks_like_api_key, error_message=apikey_error,
+    )
+
+    config.set_field_in_config_file("apikey", response)
+    print("Key set.")
 
 
 def configure(args: argparse.Namespace):
     """ Configures the openml configuration file. """
-    print("Configuring", args.file)
-
-    # check if API key exists, if so ask to overwrite.
+    print_configuration()
+    set_functions = {
+        "apikey": configure_apikey,
+    }
+    set_functions.get(args.field, quit)()
 
 
 def main() -> None:
@@ -21,8 +101,26 @@ def main() -> None:
     parser_configure = subparsers.add_parser(
         "configure", description="Set or read variables in your configuration file.",
     )
+
     parser_configure.add_argument(
-        "file", default="~/.openml/config", help="The configuration file to edit or read."
+        "field",
+        type=str,
+        choices=[
+            "apikey",
+            "server",
+            "cachedir",
+            "avoid_duplicate_runs",
+            "connection_n_retries",
+            "verbosity",
+            "all",
+            "none",
+        ],
+        default="all",
+        nargs="?",
+        help="The field you wish to edit, auto-completes the field name, "
+        "e.g. `openml configure cache` is equivalent to `openml configure cachedir`."
+        "Choosing 'all' lets you configure all fields one by one."
+        "Choosing 'none' will print out the current configuration.",
     )
 
     args = parser.parse_args()
diff --git a/openml/config.py b/openml/config.py
index 9e2e697d5..973276295 100644
--- a/openml/config.py
+++ b/openml/config.py
@@ -9,7 +9,7 @@
 import os
 from pathlib import Path
 import platform
-from typing import Tuple, cast
+from typing import Tuple, cast, Any
 
 from io import StringIO
 import configparser
@@ -177,6 +177,16 @@ def stop_using_configuration_for_example(cls):
         cls._start_last_called = False
 
 
+def determine_config_file_path() -> Path:
+    if platform.system() == "Linux":
+        config_dir = Path(os.environ.get("XDG_CONFIG_HOME", Path("~") / ".config" / "openml"))
+    else:
+        config_dir = Path("~") / ".openml"
+    # Still use os.path.expanduser to trigger the mock in the unit test
+    config_dir = Path(os.path.expanduser(config_dir))
+    return config_dir / "config"
+
+
 def _setup(config=None):
     """Setup openml package. Called on first import.
 
@@ -193,13 +203,8 @@ def _setup(config=None):
     global connection_n_retries
     global max_retries
 
-    if platform.system() == "Linux":
-        config_dir = Path(os.environ.get("XDG_CONFIG_HOME", Path("~") / ".config" / "openml"))
-    else:
-        config_dir = Path("~") / ".openml"
-    # Still use os.path.expanduser to trigger the mock in the unit test
-    config_dir = Path(os.path.expanduser(config_dir))
-    config_file = config_dir / "config"
+    config_file = determine_config_file_path()
+    config_dir = config_file.parent
 
     # read config file, create directory for config file
     if not os.path.exists(config_dir):
@@ -258,6 +263,32 @@ def _get(config, key):
         )
 
 
+def set_field_in_config_file(field: str, value: Any):
+    """ Overwrites the `field` in the configuration file with the new `value`. """
+    fields = [
+        "apikey",
+        "server",
+        "cache_directory",
+        "avoid_duplicate_runs",
+        "connection_n_retries",
+        "max_retries",
+    ]
+    if field not in fields:
+        return ValueError(f"Field '{field}' is not valid and must be one of '{fields}'.")
+
+    globals()[field] = value
+    config_file = determine_config_file_path()
+    config = _parse_config(str(config_file))
+    with open(config_file, "w") as fh:
+        for f in fields:
+            # We can't blindly set all values based on globals() because the user when the user
+            # sets it through config.FIELD it should not be stored to file.
+            value = config.get("FAKE_SECTION", f)
+            if f == field:
+                value = globals()[f]
+            fh.write(f"{f} = {value}\n")
+
+
 def _parse_config(config_file: str):
     """ Parse the config file, set up defaults. """
     config = configparser.RawConfigParser(defaults=_defaults)

From ef8434b88861cb305542157d2ce8f81700b4b2c8 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Thu, 8 Apr 2021 10:48:24 +0200
Subject: [PATCH 03/13] Remove autocomplete promise, use _defaults

Autocomplete seems to be incompatible with `choices`, so I'll ignore
that for now. We also use `config._defaults` instead of an explicit
list to avoid duplication.
---
 openml/cli.py    | 14 ++------------
 openml/config.py | 14 +++-----------
 2 files changed, 5 insertions(+), 23 deletions(-)

diff --git a/openml/cli.py b/openml/cli.py
index 496344d35..3bff36d17 100644
--- a/openml/cli.py
+++ b/openml/cli.py
@@ -105,20 +105,10 @@ def main() -> None:
     parser_configure.add_argument(
         "field",
         type=str,
-        choices=[
-            "apikey",
-            "server",
-            "cachedir",
-            "avoid_duplicate_runs",
-            "connection_n_retries",
-            "verbosity",
-            "all",
-            "none",
-        ],
+        choices=[*config._defaults.keys(), "all", "none"],
         default="all",
         nargs="?",
-        help="The field you wish to edit, auto-completes the field name, "
-        "e.g. `openml configure cache` is equivalent to `openml configure cachedir`."
+        help="The field you wish to edit."
         "Choosing 'all' lets you configure all fields one by one."
         "Choosing 'none' will print out the current configuration.",
     )
diff --git a/openml/config.py b/openml/config.py
index 973276295..a112a6613 100644
--- a/openml/config.py
+++ b/openml/config.py
@@ -265,22 +265,14 @@ def _get(config, key):
 
 def set_field_in_config_file(field: str, value: Any):
     """ Overwrites the `field` in the configuration file with the new `value`. """
-    fields = [
-        "apikey",
-        "server",
-        "cache_directory",
-        "avoid_duplicate_runs",
-        "connection_n_retries",
-        "max_retries",
-    ]
-    if field not in fields:
-        return ValueError(f"Field '{field}' is not valid and must be one of '{fields}'.")
+    if field not in _defaults:
+        return ValueError(f"Field '{field}' is not valid and must be one of '{_defaults.keys()}'.")
 
     globals()[field] = value
     config_file = determine_config_file_path()
     config = _parse_config(str(config_file))
     with open(config_file, "w") as fh:
-        for f in fields:
+        for f in _defaults.keys():
             # We can't blindly set all values based on globals() because the user when the user
             # sets it through config.FIELD it should not be stored to file.
             value = config.get("FAKE_SECTION", f)

From a64b3e9c9ae15281b406dce5dd0128466c90783a Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Thu, 8 Apr 2021 11:21:50 +0200
Subject: [PATCH 04/13] Add server configuration

---
 openml/cli.py    | 47 ++++++++++++++++++++++++++++++++++++++++++++---
 openml/config.py |  5 ++++-
 2 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/openml/cli.py b/openml/cli.py
index 3bff36d17..1a1e9da47 100644
--- a/openml/cli.py
+++ b/openml/cli.py
@@ -3,6 +3,7 @@
 import argparse
 import string
 from typing import Union, Callable
+from urllib.parse import urlparse
 
 
 from openml import config
@@ -16,6 +17,14 @@ def looks_like_api_key(apikey: str) -> bool:
     return len(apikey) == 32 and is_hex(apikey)
 
 
+def looks_like_url(url: str) -> bool:
+    # There's no thorough url parser, but we only seem to use netloc.
+    try:
+        return bool(urlparse(url).netloc)
+    except Exception:
+        return False
+
+
 def wait_until_valid_input(
     prompt: str,
     check: Callable[[str], bool],
@@ -83,13 +92,43 @@ def apikey_error(apikey: str) -> str:
     print("Key set.")
 
 
+def configure_server():
+    print("\nSpecify which server you wish to connect to.")
+
+    def is_valid_server(server: str) -> bool:
+        is_shorthand = server in ["test", "production"]
+        return is_shorthand or looks_like_url(server)
+
+    response = wait_until_valid_input(
+        prompt="Specify a url or use 'test' or 'production' as a shorthand:",
+        check=is_valid_server,
+        error_message="Must be 'test', 'production' or a url.",
+    )
+
+    if response == "test":
+        response = "https://test.openml.org/api/v1/xml"
+    elif response == "production":
+        response = "https://www.openml.org/api/v1/xml"
+
+    config.set_field_in_config_file("server", response)
+
+
 def configure(args: argparse.Namespace):
-    """ Configures the openml configuration file. """
+    """ Calls the right submenu(s) to edit `args.field` in the configuration file. """
     print_configuration()
     set_functions = {
         "apikey": configure_apikey,
+        "server": configure_server,
     }
-    set_functions.get(args.field, quit)()
+
+    def not_supported_yet():
+        print(f"Setting '{args.field}' is not supported yet.")
+
+    if args.field not in ["all", "none"]:
+        set_functions.get(args.field, not_supported_yet)()
+    elif args.field == "all":
+        for set_field_function in set_functions.values():
+            set_field_function()
 
 
 def main() -> None:
@@ -99,7 +138,9 @@ def main() -> None:
     subparsers = parser.add_subparsers(dest="subroutine")
 
     parser_configure = subparsers.add_parser(
-        "configure", description="Set or read variables in your configuration file.",
+        "configure",
+        description="Set or read variables in your configuration file. For more help also see "
+        "'https://openml.github.io/openml-python/master/usage.html#configuration'.",
     )
 
     parser_configure.add_argument(
diff --git a/openml/config.py b/openml/config.py
index a112a6613..9db8a27c3 100644
--- a/openml/config.py
+++ b/openml/config.py
@@ -273,8 +273,11 @@ def set_field_in_config_file(field: str, value: Any):
     config = _parse_config(str(config_file))
     with open(config_file, "w") as fh:
         for f in _defaults.keys():
-            # We can't blindly set all values based on globals() because the user when the user
+            # We can't blindly set all values based on globals() because when the user
             # sets it through config.FIELD it should not be stored to file.
+            # There doesn't seem to be a way to avoid writing defaults to file with configparser,
+            # because it is impossible to distinguish from an explicitly set value that matches
+            # the default value, to one that was set to its default because it was omitted.
             value = config.get("FAKE_SECTION", f)
             if f == field:
                 value = globals()[f]

From 2ff55d9d37a12fd410b57f81bdcb69bc0480f5b7 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 9 Apr 2021 10:23:50 +0200
Subject: [PATCH 05/13] Allow fields to be set directly non-interactively

With the `openml configure FIELD VALUE` command.
---
 openml/cli.py | 93 ++++++++++++++++++++++++++++++++-------------------
 1 file changed, 59 insertions(+), 34 deletions(-)

diff --git a/openml/cli.py b/openml/cli.py
index 1a1e9da47..201367b91 100644
--- a/openml/cli.py
+++ b/openml/cli.py
@@ -68,15 +68,12 @@ def print_configuration():
         print(f"{field.ljust(max_key_length)}: {value}")
 
 
-def configure_apikey() -> None:
-    print(f"\nYour current API key is set to: '{config.apikey}'")
-    print("You can get an API key at https://new.openml.org")
-    print("You must create an account if you don't have one yet.")
-    print("  1. Log in with the account.")
-    print("  2. Navigate to the profile page (top right circle > Your Profile). ")
-    print("  3. Click the API Key button to reach the page with your API key.")
-    print("If you have any difficulty following these instructions, please let us know on Github.")
+def verbose_set(field, value):
+    config.set_field_in_config_file(field, value)
+    print(f"{field} set to '{value}'.")
 
+
+def configure_apikey(value: str) -> None:
     def apikey_error(apikey: str) -> str:
         if len(apikey) != 32:
             return f"The key should contain 32 characters but contains {len(apikey)}."
@@ -84,51 +81,75 @@ def apikey_error(apikey: str) -> str:
             return "Some characters are not hexadecimal."
         return "This does not look like an API key."
 
-    response = wait_until_valid_input(
-        prompt="Please enter your API key:", check=looks_like_api_key, error_message=apikey_error,
-    )
-
-    config.set_field_in_config_file("apikey", response)
-    print("Key set.")
-
-
-def configure_server():
-    print("\nSpecify which server you wish to connect to.")
-
+    if value is not None:
+        if not looks_like_api_key(value):
+            print(apikey_error(value))
+            quit(-1)
+    else:
+        print(f"\nYour current API key is set to: '{config.apikey}'")
+        print("You can get an API key at https://new.openml.org")
+        print("You must create an account if you don't have one yet.")
+        print("  1. Log in with the account.")
+        print("  2. Navigate to the profile page (top right circle > Your Profile). ")
+        print("  3. Click the API Key button to reach the page with your API key.")
+        print("If you have any difficulty following these instructions, let us know on Github.")
+
+        value = wait_until_valid_input(
+            prompt="Please enter your API key:",
+            check=looks_like_api_key,
+            error_message=apikey_error,
+        )
+    verbose_set("apikey", value)
+
+
+def configure_server(value: str):
     def is_valid_server(server: str) -> bool:
         is_shorthand = server in ["test", "production"]
         return is_shorthand or looks_like_url(server)
 
-    response = wait_until_valid_input(
-        prompt="Specify a url or use 'test' or 'production' as a shorthand:",
-        check=is_valid_server,
-        error_message="Must be 'test', 'production' or a url.",
-    )
+    error_message = "Must be 'test', 'production' or a url."
+
+    if value is not None:
+        if not is_valid_server(value):
+            print(error_message)
+            quit(-1)
+    else:
+        print("\nSpecify which server you wish to connect to.")
+        value = wait_until_valid_input(
+            prompt="Specify a url or use 'test' or 'production' as a shorthand:",
+            check=is_valid_server,
+            error_message=error_message,
+        )
 
-    if response == "test":
-        response = "https://test.openml.org/api/v1/xml"
-    elif response == "production":
-        response = "https://www.openml.org/api/v1/xml"
+    if value == "test":
+        value = "https://test.openml.org/api/v1/xml"
+    elif value == "production":
+        value = "https://www.openml.org/api/v1/xml"
 
-    config.set_field_in_config_file("server", response)
+    verbose_set("server", value)
 
 
 def configure(args: argparse.Namespace):
     """ Calls the right submenu(s) to edit `args.field` in the configuration file. """
-    print_configuration()
     set_functions = {
         "apikey": configure_apikey,
         "server": configure_server,
     }
 
-    def not_supported_yet():
+    def not_supported_yet(_):
         print(f"Setting '{args.field}' is not supported yet.")
 
     if args.field not in ["all", "none"]:
-        set_functions.get(args.field, not_supported_yet)()
-    elif args.field == "all":
+        set_functions.get(args.field, not_supported_yet)(args.value)
+    else:
+        if args.value is not None:
+            print(f"Can not set value ('{args.value}') when field is specified as '{args.field}'.")
+            quit()
+        print_configuration()
+
+    if args.field == "all":
         for set_field_function in set_functions.values():
-            set_field_function()
+            set_field_function(args.value)
 
 
 def main() -> None:
@@ -154,6 +175,10 @@ def main() -> None:
         "Choosing 'none' will print out the current configuration.",
     )
 
+    parser_configure.add_argument(
+        "value", type=str, default=None, nargs="?", help="The value to set the FIELD to.",
+    )
+
     args = parser.parse_args()
     subroutines.get(args.subroutine, lambda _: parser.print_help())(args)
 

From 8c00fc05d8179154915c7f57e648a829b3a91b84 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 9 Apr 2021 11:10:48 +0200
Subject: [PATCH 06/13] Combine error and check functionalities

Otherwise you have to duplicate all checks in the error message
function.
---
 openml/cli.py | 99 +++++++++++++++++++++++++++++++--------------------
 1 file changed, 61 insertions(+), 38 deletions(-)

diff --git a/openml/cli.py b/openml/cli.py
index 201367b91..900d8fdfc 100644
--- a/openml/cli.py
+++ b/openml/cli.py
@@ -1,8 +1,11 @@
 """" Command Line Interface for `openml` to configure its settings. """
 
 import argparse
+
+# import os
+# import pathlib
 import string
-from typing import Union, Callable
+from typing import Callable  # Union, Callable
 from urllib.parse import urlparse
 
 
@@ -13,10 +16,6 @@ def is_hex(string_: str) -> bool:
     return all(c in string.hexdigits for c in string_)
 
 
-def looks_like_api_key(apikey: str) -> bool:
-    return len(apikey) == 32 and is_hex(apikey)
-
-
 def looks_like_url(url: str) -> bool:
     # There's no thorough url parser, but we only seem to use netloc.
     try:
@@ -25,35 +24,29 @@ def looks_like_url(url: str) -> bool:
         return False
 
 
-def wait_until_valid_input(
-    prompt: str,
-    check: Callable[[str], bool],
-    error_message: Union[str, Callable[[str], str]] = "That is not a valid response.",
-) -> str:
+def wait_until_valid_input(prompt: str, check: Callable[[str], str],) -> str:
     """  Asks `prompt` until an input is received which returns True for `check`.
 
     Parameters
     ----------
     prompt: str
         message to display
-    check: Callable[[str], bool]
-        function to call with the given input, should return true only if the input is valid.
-    error_message: Union[str, Callable[[str], str]
-        a message to display on invalid input, or a `str`->`str` function that can give feedback
-        specific to the error.
+    check: Callable[[str], str]
+        function to call with the given input, that provides an error message if the input is not
+        valid otherwise, and False-like otherwise.
 
     Returns
     -------
+    valid input
 
     """
 
     response = input(prompt)
-    while not check(response):
-        if isinstance(error_message, str):
-            print(error_message)
-        else:
-            print(error_message(response), end="\n\n")
+    error_message = check(response)
+    while error_message:
+        print(error_message, end="\n\n")
         response = input(prompt)
+        error_message = check(response)
 
     return response
 
@@ -74,16 +67,17 @@ def verbose_set(field, value):
 
 
 def configure_apikey(value: str) -> None:
-    def apikey_error(apikey: str) -> str:
+    def check_apikey(apikey: str) -> str:
         if len(apikey) != 32:
             return f"The key should contain 32 characters but contains {len(apikey)}."
         if not is_hex(apikey):
             return "Some characters are not hexadecimal."
-        return "This does not look like an API key."
+        return ""
 
     if value is not None:
-        if not looks_like_api_key(value):
-            print(apikey_error(value))
+        apikey_malformed = check_apikey(value)
+        if apikey_malformed:
+            print(apikey_malformed)
             quit(-1)
     else:
         print(f"\nYour current API key is set to: '{config.apikey}'")
@@ -94,31 +88,27 @@ def apikey_error(apikey: str) -> str:
         print("  3. Click the API Key button to reach the page with your API key.")
         print("If you have any difficulty following these instructions, let us know on Github.")
 
-        value = wait_until_valid_input(
-            prompt="Please enter your API key:",
-            check=looks_like_api_key,
-            error_message=apikey_error,
-        )
+        value = wait_until_valid_input(prompt="Please enter your API key:", check=check_apikey,)
     verbose_set("apikey", value)
 
 
-def configure_server(value: str):
-    def is_valid_server(server: str) -> bool:
+def configure_server(value: str) -> None:
+    def check_server(server: str) -> str:
         is_shorthand = server in ["test", "production"]
-        return is_shorthand or looks_like_url(server)
-
-    error_message = "Must be 'test', 'production' or a url."
+        if is_shorthand or looks_like_url(server):
+            return ""
+        return "Must be 'test', 'production' or a url."
 
     if value is not None:
-        if not is_valid_server(value):
-            print(error_message)
+        server_malformed = check_server(value)
+        if server_malformed:
+            print(server_malformed)
             quit(-1)
     else:
         print("\nSpecify which server you wish to connect to.")
         value = wait_until_valid_input(
             prompt="Specify a url or use 'test' or 'production' as a shorthand:",
-            check=is_valid_server,
-            error_message=error_message,
+            check=check_server,
         )
 
     if value == "test":
@@ -129,6 +119,39 @@ def is_valid_server(server: str) -> bool:
     verbose_set("server", value)
 
 
+# def configure_cachedir(value: str) -> None:
+#     def is_valid_cachedir(path: str) -> bool:
+#         return True
+#
+#     def cache_dir_error(path: str) -> str:
+#         p = pathlib.Path(path)
+#         if p.is_file():
+#             return f"'{path}' is a file, not a directory."
+#         expanded = p.expanduser()
+#         if not expanded.is_absolute():
+#             return f"'{path}' is not absolute (even after expanding '~')."
+#         if expanded.exists():
+#             try:
+#                 os.mkdir(expanded)
+#             except PermissionError:
+#               return f"'{path}' does not exist and there are not enough permissions to create it."
+#         return "is not valid"
+#
+#     if value is not None:
+#         if not is_valid_cachedir(value):
+#             print(cache_dir_error(value))
+#             quit(-1)
+#     else:
+#         print("\nConfiguring the cache directory. It can not be a relative path.")
+#         value = wait_until_valid_input(
+#             prompt="Specify the directory to use (or create) as cache directory:",
+#             check=is_valid_cachedir,
+#         )
+#
+#     print("NOTE: Data from your old cache directory is not moved over.")
+#     verbose_set("cachedir", value)
+
+
 def configure(args: argparse.Namespace):
     """ Calls the right submenu(s) to edit `args.field` in the configuration file. """
     set_functions = {

From a41c41f47070a4160e8f49ff2b4795a3b740938e Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 9 Apr 2021 14:11:40 +0200
Subject: [PATCH 07/13] Share logic about setting/collecting the value

---
 openml/cli.py | 161 +++++++++++++++++++++++++++++---------------------
 1 file changed, 93 insertions(+), 68 deletions(-)

diff --git a/openml/cli.py b/openml/cli.py
index 900d8fdfc..32a62ccf6 100644
--- a/openml/cli.py
+++ b/openml/cli.py
@@ -1,11 +1,10 @@
 """" Command Line Interface for `openml` to configure its settings. """
 
 import argparse
-
-# import os
-# import pathlib
+import os
+import pathlib
 import string
-from typing import Callable  # Union, Callable
+from typing import Union, Callable
 from urllib.parse import urlparse
 
 
@@ -74,22 +73,23 @@ def check_apikey(apikey: str) -> str:
             return "Some characters are not hexadecimal."
         return ""
 
-    if value is not None:
-        apikey_malformed = check_apikey(value)
-        if apikey_malformed:
-            print(apikey_malformed)
-            quit(-1)
-    else:
-        print(f"\nYour current API key is set to: '{config.apikey}'")
-        print("You can get an API key at https://new.openml.org")
-        print("You must create an account if you don't have one yet.")
-        print("  1. Log in with the account.")
-        print("  2. Navigate to the profile page (top right circle > Your Profile). ")
-        print("  3. Click the API Key button to reach the page with your API key.")
-        print("If you have any difficulty following these instructions, let us know on Github.")
+    instructions = (
+        f"Your current API key is set to: '{config.apikey}'"
+        "You can get an API key at https://new.openml.org"
+        "You must create an account if you don't have one yet."
+        "  1. Log in with the account."
+        "  2. Navigate to the profile page (top right circle > Your Profile). "
+        "  3. Click the API Key button to reach the page with your API key."
+        "If you have any difficulty following these instructions, let us know on Github."
+    )
 
-        value = wait_until_valid_input(prompt="Please enter your API key:", check=check_apikey,)
-    verbose_set("apikey", value)
+    configure_field(
+        field="apikey",
+        value=value,
+        check_with_message=check_apikey,
+        intro_message=instructions,
+        input_message="Please enter your API key:",
+    )
 
 
 def configure_server(value: str) -> None:
@@ -99,57 +99,81 @@ def check_server(server: str) -> str:
             return ""
         return "Must be 'test', 'production' or a url."
 
+    configure_field(
+        field="server",
+        value=value,
+        check_with_message=check_server,
+        intro_message="Specify which server you wish to connect to.",
+        input_message="Specify a url or use 'test' or 'production' as a shorthand:",
+    )
+
+
+def configure_cachedir(value: str) -> None:
+    def check_cache_dir(path: str) -> str:
+        p = pathlib.Path(path)
+        if p.is_file():
+            return f"'{path}' is a file, not a directory."
+        expanded = p.expanduser()
+        if not expanded.is_absolute():
+            return f"'{path}' is not absolute (even after expanding '~')."
+        if not expanded.exists():
+            try:
+                os.mkdir(expanded)
+            except PermissionError:
+                return f"'{path}' does not exist and there are not enough permissions to create it."
+        return ""
+
+    configure_field(
+        field="cachedir",
+        value=value,
+        check_with_message=check_cache_dir,
+        intro_message="Configuring the cache directory. It can not be a relative path.",
+        input_message="Specify the directory to use (or create) as cache directory:",
+    )
+    print("NOTE: Data from your old cache directory is not moved over.")
+
+
+def configure_field(
+    field: str,
+    value: Union[None, str],
+    check_with_message: Callable[[str], str],
+    intro_message: str,
+    input_message: str,
+) -> None:
+    """ Configure `field` with `value`. If `value` is None ask the user for input.
+
+    `value` and user input are validated with `check_with_message` function, and
+    in the case of user input the user gets to input a new value.
+    The change is saved in the openml configuration file.
+    In case an invalid `value` is supplied, no changes are made.
+
+    Parameters
+    ----------
+    field: str
+        Field to set.
+    value: str, None
+        Value to field to. If `None` will ask user for input.
+    check_with_message: Callable[[str], str]
+        Function which validates `value` or user input, and returns either an error message if it
+        is invalid, or a False-like value if `value` is valid.
+    intro_message: str
+        Message that is printed once if user input is requested (e.g. instructions).
+    input_message: str
+        Message that comes with the input prompt.
+
+    Returns
+    -------
+
+    """
     if value is not None:
-        server_malformed = check_server(value)
-        if server_malformed:
-            print(server_malformed)
-            quit(-1)
+        malformed_input = check_with_message(value)
+        if malformed_input:
+            print(malformed_input)
+            quit()
     else:
-        print("\nSpecify which server you wish to connect to.")
-        value = wait_until_valid_input(
-            prompt="Specify a url or use 'test' or 'production' as a shorthand:",
-            check=check_server,
-        )
-
-    if value == "test":
-        value = "https://test.openml.org/api/v1/xml"
-    elif value == "production":
-        value = "https://www.openml.org/api/v1/xml"
-
-    verbose_set("server", value)
-
-
-# def configure_cachedir(value: str) -> None:
-#     def is_valid_cachedir(path: str) -> bool:
-#         return True
-#
-#     def cache_dir_error(path: str) -> str:
-#         p = pathlib.Path(path)
-#         if p.is_file():
-#             return f"'{path}' is a file, not a directory."
-#         expanded = p.expanduser()
-#         if not expanded.is_absolute():
-#             return f"'{path}' is not absolute (even after expanding '~')."
-#         if expanded.exists():
-#             try:
-#                 os.mkdir(expanded)
-#             except PermissionError:
-#               return f"'{path}' does not exist and there are not enough permissions to create it."
-#         return "is not valid"
-#
-#     if value is not None:
-#         if not is_valid_cachedir(value):
-#             print(cache_dir_error(value))
-#             quit(-1)
-#     else:
-#         print("\nConfiguring the cache directory. It can not be a relative path.")
-#         value = wait_until_valid_input(
-#             prompt="Specify the directory to use (or create) as cache directory:",
-#             check=is_valid_cachedir,
-#         )
-#
-#     print("NOTE: Data from your old cache directory is not moved over.")
-#     verbose_set("cachedir", value)
+        print(intro_message)
+        value = wait_until_valid_input(prompt=input_message, check=check_with_message,)
+    verbose_set(field, value)
 
 
 def configure(args: argparse.Namespace):
@@ -157,6 +181,7 @@ def configure(args: argparse.Namespace):
     set_functions = {
         "apikey": configure_apikey,
         "server": configure_server,
+        "cachedir": configure_cachedir,
     }
 
     def not_supported_yet(_):

From 7703d3b4794239ba320943e7b898e75039f50980 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 9 Apr 2021 14:58:52 +0200
Subject: [PATCH 08/13] Complete CLI for other fields.

Max_retries is excluded because it should not be user configurable, and
will most likely be removed.
Verbosity is configurable but is currently not actually used.
---
 openml/cli.py | 95 +++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 80 insertions(+), 15 deletions(-)

diff --git a/openml/cli.py b/openml/cli.py
index 32a62ccf6..9f726ed2f 100644
--- a/openml/cli.py
+++ b/openml/cli.py
@@ -74,12 +74,12 @@ def check_apikey(apikey: str) -> str:
         return ""
 
     instructions = (
-        f"Your current API key is set to: '{config.apikey}'"
-        "You can get an API key at https://new.openml.org"
-        "You must create an account if you don't have one yet."
-        "  1. Log in with the account."
-        "  2. Navigate to the profile page (top right circle > Your Profile). "
-        "  3. Click the API Key button to reach the page with your API key."
+        f"Your current API key is set to: '{config.apikey}'. "
+        "You can get an API key at https://new.openml.org. "
+        "You must create an account if you don't have one yet:\n"
+        "  1. Log in with the account.\n"
+        "  2. Navigate to the profile page (top right circle > Your Profile). \n"
+        "  3. Click the API Key button to reach the page with your API key.\n"
         "If you have any difficulty following these instructions, let us know on Github."
     )
 
@@ -104,7 +104,7 @@ def check_server(server: str) -> str:
         value=value,
         check_with_message=check_server,
         intro_message="Specify which server you wish to connect to.",
-        input_message="Specify a url or use 'test' or 'production' as a shorthand:",
+        input_message="Specify a url or use 'test' or 'production' as a shorthand: ",
     )
 
 
@@ -128,11 +128,74 @@ def check_cache_dir(path: str) -> str:
         value=value,
         check_with_message=check_cache_dir,
         intro_message="Configuring the cache directory. It can not be a relative path.",
-        input_message="Specify the directory to use (or create) as cache directory:",
+        input_message="Specify the directory to use (or create) as cache directory: ",
     )
     print("NOTE: Data from your old cache directory is not moved over.")
 
 
+def configure_connection_n_retries(value: str) -> None:
+    def valid_connection_retries(value: str) -> str:
+        if not value.isdigit():
+            return f"Must be an integer number (smaller than {config.max_retries})."
+        if int(value) > config.max_retries:
+            return f"connection_n_retries may not exceed {config.max_retries}."
+        if int(value) == 0:
+            return "connection_n_retries must be non-zero."
+        return ""
+
+    configure_field(
+        field="connection_n_retries",
+        value=value,
+        check_with_message=valid_connection_retries,
+        intro_message="Configuring the number of times to attempt to connect to the OpenML Server",
+        input_message=f"Enter an integer between 0 and {config.max_retries}: ",
+    )
+
+
+def configure_avoid_duplicate_runs(value: str) -> None:
+    def is_python_bool(value: str) -> str:
+        if value in ["True", "False"]:
+            return ""
+        return "Must be 'True' or 'False' (mind the capital)."
+
+    intro_message = (
+        "If set to True, when `run_flow_on_task` or similar methods are called a lookup is "
+        "performed to see if there already exists such a run on the server. "
+        "If so, download those results instead. "
+        "If set to False, runs will always be executed."
+    )
+
+    configure_field(
+        field="avoid_duplicate_runs",
+        value=value,
+        check_with_message=is_python_bool,
+        intro_message=intro_message,
+        input_message="Enter 'True' or 'False': ",
+    )
+
+
+def configure_verbosity(value: str) -> None:
+    def is_zero_through_two(value: str) -> str:
+        if value in ["0", "1", "2"]:
+            return ""
+        return "Must be '0', '1' or '2'."
+
+    intro_message = (
+        "Set the verbosity of log messages which should be shown by openml-python."
+        " 0: normal output (warnings and errors)"
+        " 1: info output (some high-level progress output)"
+        " 2: debug output (detailed information (for developers))"
+    )
+
+    configure_field(
+        field="verbosity",
+        value=value,
+        check_with_message=is_zero_through_two,
+        intro_message=intro_message,
+        input_message="Enter '0', '1' or '2': ",
+    )
+
+
 def configure_field(
     field: str,
     value: Union[None, str],
@@ -160,10 +223,6 @@ def configure_field(
         Message that is printed once if user input is requested (e.g. instructions).
     input_message: str
         Message that comes with the input prompt.
-
-    Returns
-    -------
-
     """
     if value is not None:
         malformed_input = check_with_message(value)
@@ -182,6 +241,9 @@ def configure(args: argparse.Namespace):
         "apikey": configure_apikey,
         "server": configure_server,
         "cachedir": configure_cachedir,
+        "connection_n_retries": configure_connection_n_retries,
+        "avoid_duplicate_runs": configure_avoid_duplicate_runs,
+        "verbosity": configure_verbosity,
     }
 
     def not_supported_yet(_):
@@ -197,6 +259,7 @@ def not_supported_yet(_):
 
     if args.field == "all":
         for set_field_function in set_functions.values():
+            print()  # Visually separating the output by field.
             set_field_function(args.value)
 
 
@@ -212,14 +275,16 @@ def main() -> None:
         "'https://openml.github.io/openml-python/master/usage.html#configuration'.",
     )
 
+    configurable_fields = [f for f in config._defaults if f not in ["max_retries"]]
+
     parser_configure.add_argument(
         "field",
         type=str,
-        choices=[*config._defaults.keys(), "all", "none"],
+        choices=[*configurable_fields, "all", "none"],
         default="all",
         nargs="?",
-        help="The field you wish to edit."
-        "Choosing 'all' lets you configure all fields one by one."
+        help="The field you wish to edit. "
+        "Choosing 'all' lets you configure all fields one by one. "
         "Choosing 'none' will print out the current configuration.",
     )
 

From 75e37833b295b84fcee74028553214a96c74b61b Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 9 Apr 2021 15:27:02 +0200
Subject: [PATCH 09/13] Bring back sanitizing user input

And extend it to the bool inputs.
---
 openml/cli.py | 63 +++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 48 insertions(+), 15 deletions(-)

diff --git a/openml/cli.py b/openml/cli.py
index 9f726ed2f..0faa34d45 100644
--- a/openml/cli.py
+++ b/openml/cli.py
@@ -23,7 +23,9 @@ def looks_like_url(url: str) -> bool:
         return False
 
 
-def wait_until_valid_input(prompt: str, check: Callable[[str], str],) -> str:
+def wait_until_valid_input(
+    prompt: str, check: Callable[[str], str], sanitize: Union[Callable[[str], str], None]
+) -> str:
     """  Asks `prompt` until an input is received which returns True for `check`.
 
     Parameters
@@ -33,18 +35,23 @@ def wait_until_valid_input(prompt: str, check: Callable[[str], str],) -> str:
     check: Callable[[str], str]
         function to call with the given input, that provides an error message if the input is not
         valid otherwise, and False-like otherwise.
+    sanitize: Callable[[str], str], optional
+        A function which attempts to sanitize the user input (e.g. auto-complete).
 
     Returns
     -------
     valid input
 
     """
-
     response = input(prompt)
+    if sanitize:
+        response = sanitize(response)
     error_message = check(response)
     while error_message:
         print(error_message, end="\n\n")
         response = input(prompt)
+        if sanitize:
+            response = sanitize(response)
         error_message = check(response)
 
     return response
@@ -99,12 +106,20 @@ def check_server(server: str) -> str:
             return ""
         return "Must be 'test', 'production' or a url."
 
+    def replace_shorthand(server: str) -> str:
+        if server == "test":
+            return "https://test.openml.org/api/v1/xml"
+        if server == "production":
+            return "https://www.openml.org/api/v1/xml"
+        return server
+
     configure_field(
         field="server",
         value=value,
         check_with_message=check_server,
         intro_message="Specify which server you wish to connect to.",
         input_message="Specify a url or use 'test' or 'production' as a shorthand: ",
+        sanitize=replace_shorthand,
     )
 
 
@@ -134,12 +149,12 @@ def check_cache_dir(path: str) -> str:
 
 
 def configure_connection_n_retries(value: str) -> None:
-    def valid_connection_retries(value: str) -> str:
-        if not value.isdigit():
+    def valid_connection_retries(n: str) -> str:
+        if not n.isdigit():
             return f"Must be an integer number (smaller than {config.max_retries})."
-        if int(value) > config.max_retries:
+        if int(n) > config.max_retries:
             return f"connection_n_retries may not exceed {config.max_retries}."
-        if int(value) == 0:
+        if int(n) == 0:
             return "connection_n_retries must be non-zero."
         return ""
 
@@ -153,11 +168,18 @@ def valid_connection_retries(value: str) -> str:
 
 
 def configure_avoid_duplicate_runs(value: str) -> None:
-    def is_python_bool(value: str) -> str:
-        if value in ["True", "False"]:
+    def is_python_bool(bool_: str) -> str:
+        if bool_ in ["True", "False"]:
             return ""
         return "Must be 'True' or 'False' (mind the capital)."
 
+    def autocomplete_bool(bool_: str) -> str:
+        if bool_.lower() in ["n", "no", "f", "false", "0"]:
+            return "False"
+        if bool_.lower() in ["y", "yes", "t", "true", "1"]:
+            return "True"
+        return bool_
+
     intro_message = (
         "If set to True, when `run_flow_on_task` or similar methods are called a lookup is "
         "performed to see if there already exists such a run on the server. "
@@ -171,12 +193,13 @@ def is_python_bool(value: str) -> str:
         check_with_message=is_python_bool,
         intro_message=intro_message,
         input_message="Enter 'True' or 'False': ",
+        sanitize=autocomplete_bool,
     )
 
 
 def configure_verbosity(value: str) -> None:
-    def is_zero_through_two(value: str) -> str:
-        if value in ["0", "1", "2"]:
+    def is_zero_through_two(verbosity: str) -> str:
+        if verbosity in ["0", "1", "2"]:
             return ""
         return "Must be '0', '1' or '2'."
 
@@ -202,13 +225,15 @@ def configure_field(
     check_with_message: Callable[[str], str],
     intro_message: str,
     input_message: str,
+    sanitize: Union[Callable[[str], str], None] = None,
 ) -> None:
     """ Configure `field` with `value`. If `value` is None ask the user for input.
 
-    `value` and user input are validated with `check_with_message` function, and
-    in the case of user input the user gets to input a new value.
-    The change is saved in the openml configuration file.
-    In case an invalid `value` is supplied, no changes are made.
+    `value` and user input are first corrected/auto-completed with `convert_value` if provided,
+    then validated with `check_with_message` function.
+    If the user input a wrong value in interactive mode, the user gets to input a new value.
+    The new valid value is saved in the openml configuration file.
+    In case an invalid `value` is supplied directly (non-interactive), no changes are made.
 
     Parameters
     ----------
@@ -223,15 +248,23 @@ def configure_field(
         Message that is printed once if user input is requested (e.g. instructions).
     input_message: str
         Message that comes with the input prompt.
+    sanitize: Union[Callable[[str], str], None]
+        A function to convert user input to 'more acceptable' input, e.g. for auto-complete.
+        If no correction of user input is possible, return the original value.
+        If no function is provided, don't attempt to correct/auto-complete input.
     """
     if value is not None:
+        if sanitize:
+            value = sanitize(value)
         malformed_input = check_with_message(value)
         if malformed_input:
             print(malformed_input)
             quit()
     else:
         print(intro_message)
-        value = wait_until_valid_input(prompt=input_message, check=check_with_message,)
+        value = wait_until_valid_input(
+            prompt=input_message, check=check_with_message, sanitize=sanitize,
+        )
     verbose_set(field, value)
 
 

From a41b144b8cbc6303cfdb30cd59b297c98486ed3d Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 9 Apr 2021 15:30:32 +0200
Subject: [PATCH 10/13] Add small bit of info about the command line tool

---
 doc/usage.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/usage.rst b/doc/usage.rst
index 1d54baa62..49f656cc3 100644
--- a/doc/usage.rst
+++ b/doc/usage.rst
@@ -59,6 +59,10 @@ which are separated by newlines. The following keys are defined:
     * 1: info output
     * 2: debug output
 
+This file is easily configurable by the ``openml`` command line interface.
+To see where the file is stored, and what its values are, use `openml configure none`.
+Set any field with ``openml configure FIELD`` or even all fields with just ``openml configure``.
+
 ~~~~~~~~~~~~
 Key concepts
 ~~~~~~~~~~~~

From fb2c56b0c0db16867a88cdc2b30a71c8d82fa385 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 9 Apr 2021 15:37:40 +0200
Subject: [PATCH 11/13] Add API key configuration note in the introduction

---
 examples/20_basic/introduction_tutorial.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/examples/20_basic/introduction_tutorial.py b/examples/20_basic/introduction_tutorial.py
index 151692fdc..737362e49 100644
--- a/examples/20_basic/introduction_tutorial.py
+++ b/examples/20_basic/introduction_tutorial.py
@@ -42,13 +42,17 @@
 # * After logging in, open your account page (avatar on the top right)
 # * Open 'Account Settings', then 'API authentication' to find your API key.
 #
-# There are two ways to authenticate:
+# There are two ways to permanently authenticate:
 #
+# * Use the ``openml`` CLI tool with ``openml configure apikey MYKEY``,
+#   replacing **MYKEY** with your API key.
 # * Create a plain text file **~/.openml/config** with the line
 #   **'apikey=MYKEY'**, replacing **MYKEY** with your API key. The config
 #   file must be in the directory ~/.openml/config and exist prior to
 #   importing the openml module.
-# * Run the code below, replacing 'YOURKEY' with your API key.
+#
+# Alternatively, by running the code below and replacing 'YOURKEY' with your API key,
+# you authenticate for the duration of the python process.
 #
 # .. warning:: This example uploads data. For that reason, this example
 #   connects to the test server instead. This prevents the live server from

From 149065dc88b0dc22aa03b4c036a96bc1aba7b025 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 9 Apr 2021 15:39:52 +0200
Subject: [PATCH 12/13] Add to progress log

---
 doc/progress.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/progress.rst b/doc/progress.rst
index 05446d61b..1acc7ed68 100644
--- a/doc/progress.rst
+++ b/doc/progress.rst
@@ -10,6 +10,7 @@ Changelog
 ~~~~~~
 
 * FIX #1035: Render class attributes and methods again.
+* ADD #1049: Add a command line tool for configuration openml-python.
 
 0.12.0
 ~~~~~~

From 935ee250df6242721bd3fdadb1fcea16906f3818 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 9 Apr 2021 15:45:49 +0200
Subject: [PATCH 13/13] Refactor flow of wait_until_valid_input

---
 openml/cli.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/openml/cli.py b/openml/cli.py
index 0faa34d45..b26e67d2e 100644
--- a/openml/cli.py
+++ b/openml/cli.py
@@ -43,18 +43,16 @@ def wait_until_valid_input(
     valid input
 
     """
-    response = input(prompt)
-    if sanitize:
-        response = sanitize(response)
-    error_message = check(response)
-    while error_message:
-        print(error_message, end="\n\n")
+
+    while True:
         response = input(prompt)
         if sanitize:
             response = sanitize(response)
         error_message = check(response)
-
-    return response
+        if error_message:
+            print(error_message, end="\n\n")
+        else:
+            return response
 
 
 def print_configuration():