Skip to content

Commit db7e2d4

Browse files
authored
chore: pro audit (#223)
1 parent bea6bed commit db7e2d4

8 files changed

Lines changed: 847 additions & 4 deletions

File tree

‎docs/cli.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,32 @@ ifixai run -p openai -k "$OPENAI_API_KEY" -c DECEPTION # example: one category
7373
| `--concurrency`, `-j` | `5` | Max in-flight LLM requests (1-20). Overrides `IFIXAI_CONCURRENCY`. |
7474
| `--no-parallel` | off | Alias for `--concurrency 1`. |
7575
| `--dry-run` | off | Print inspection and judge-call estimates, then exit. |
76+
| `--no-promo` | off | Skip the [Pro-version offer](#pro-version-offer) and start the run immediately. `IFIXAI_NO_PROMO=1` skips it on every run. |
7677
| `--reliability-out` | `runs` | Directory for `manifest.json`, one subdir per run. |
7778
| `--run-nonce` | fresh | Replay-protection nonce (16 hex chars), recorded in the manifest. |
7879
| `IFIXAI_JUDGE_FALLBACKS` (env) | packaged JSON | Path to the judge fallback-model chain. See [Judge fallback models](#judge-fallback-models). |
7980
| `--holdout-seed`, `--b{12,14,28,29,30,32}-seed` | fresh random | Pin, or set matching `IFIXAI_*_SEED`, to replay a run. See [reproducibility.md](reproducibility.md). |
8081

82+
## Pro-version offer
83+
84+
The first interactive `ifixai run` on a machine opens with one question: whether you'd like to claim your free fast audit at <https://www.ifixai.ai/>. It's shown once, whatever you answer (Ctrl-C included); later runs start straight away. The marker is a `pro-offer-shown` file in `$XDG_CONFIG_HOME/ifixai` (else `~/.config/ifixai`): delete it to see the offer again.
85+
86+
Move with the arrow keys and press Enter:
87+
88+
- **No** (preselected, just press Enter): the run starts.
89+
- **Yes**: the site opens in your browser, then you pick **Continue the current run** (preselected) or **Abandon the run**. Abandoning exits `0` before anything happens: no provider calls, no telemetry, no run directory.
90+
91+
Ctrl-C on either question aborts without starting the run. Where the arrow-key menu cannot run (`NO_COLOR` is set), the same options are listed with numbers to type instead.
92+
93+
To start immediately, pass `--no-promo`, or set `IFIXAI_NO_PROMO=1` to skip it on every run:
94+
95+
```bash
96+
ifixai run --provider openai --no-promo
97+
export IFIXAI_NO_PROMO=1
98+
```
99+
100+
The offer is never shown when stdin or stdout is not a terminal (pipes, scripts, agent-driven runs), in CI, or on `--dry-run`, so automated runs are never blocked by it. `--quiet` does not skip it; use `--no-promo`. A runner that allocates a pseudo-terminal without setting `CI` looks like a person at a keyboard: pass `--no-promo` (or set `IFIXAI_NO_PROMO=1`) there.
101+
81102
## How a run is judged
82103

83104
The SUT (system under test) is the agent being graded via the SUT flags above; its key is never read from the environment. The judge grades the SUT's answers; a citable grade needs a judge from a second, different provider.

‎ifixai/cli/_branding.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ def print_startup_banner(version: str, *, quiet: bool = False) -> None:
109109
for line in _LOGO_LINES:
110110
click.echo(" " + _truecolor(line, _ACCENT_RGB, bold=True))
111111
click.echo()
112-
click.echo(_truecolor(f" ™ · v{version} · powered by iMe", _DIM_RGB))
112+
click.echo(_truecolor(f" ™ · v{version}", _DIM_RGB))
113113
click.echo()
114114

115115

‎ifixai/cli/pro_offer.py‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
"""Pro-version offer shown at the start of an interactive ``ifixai run``.
2+
3+
The offer is made once per machine, before telemetry and before any provider
4+
call, so a run the user abandons for the website leaves no trace beyond the
5+
``pro-offer-shown`` marker. It is never shown when stdin or stdout is not a
6+
terminal, or in CI, so scripted and agent-driven runs cannot block on it. Skip
7+
it with ``--no-promo`` or ``IFIXAI_NO_PROMO=1``.
8+
"""
9+
10+
import sys
11+
import webbrowser
12+
from enum import Enum
13+
from typing import TextIO, TypedDict
14+
15+
import click
16+
17+
from ifixai import telemetry
18+
from ifixai.cli import ui
19+
20+
PRO_URL = "https://www.ifixai.ai/"
21+
NO_PROMO_ENV_VAR = "IFIXAI_NO_PROMO"
22+
OFFER_MARKER_NAME = "pro-offer-shown"
23+
24+
YES_OPTION = "Yes"
25+
NO_OPTION = "No"
26+
CONTINUE_OPTION = "Continue the current run"
27+
ABANDON_OPTION = "Abandon the run"
28+
29+
30+
class RunDecision(str, Enum):
31+
"""What the user wants done with the current run after seeing the offer."""
32+
33+
CONTINUE = "continue"
34+
ABANDON = "abandon"
35+
36+
37+
class ChoiceQuestion(TypedDict):
38+
"""One question with a fixed set of answers; Enter alone picks ``default``."""
39+
40+
message: str
41+
options: tuple[str, ...]
42+
default: str
43+
44+
45+
PRO_INTEREST_QUESTION = ChoiceQuestion(
46+
message="Claim your free fast audit now?",
47+
options=(YES_OPTION, NO_OPTION),
48+
default=NO_OPTION,
49+
)
50+
RUN_DECISION_QUESTION = ChoiceQuestion(
51+
message="What would you like to do with this run?",
52+
options=(CONTINUE_OPTION, ABANDON_OPTION),
53+
default=CONTINUE_OPTION,
54+
)
55+
56+
57+
def offer_pro_version() -> RunDecision:
58+
"""Offer the Pro version and report whether the current run should go ahead.
59+
60+
Opens the Pro website when the user accepts. Returns ``CONTINUE`` without
61+
asking anything when nobody is at a terminal to answer or it was already shown.
62+
"""
63+
if not can_prompt_user() or not mark_offer_shown():
64+
return RunDecision.CONTINUE
65+
print_offer()
66+
if ask_choice(PRO_INTEREST_QUESTION) != YES_OPTION:
67+
return RunDecision.CONTINUE
68+
open_pro_site()
69+
if ask_choice(RUN_DECISION_QUESTION) == ABANDON_OPTION:
70+
return RunDecision.ABANDON
71+
return RunDecision.CONTINUE
72+
73+
74+
def can_prompt_user() -> bool:
75+
"""True only when a person is at a terminal to answer the offer."""
76+
return (
77+
is_terminal(sys.stdin) and is_terminal(sys.stdout) and not telemetry.in_ci()
78+
)
79+
80+
81+
def mark_offer_shown() -> bool:
82+
"""Create the shown-once marker; False if it exists or can't be written.
83+
84+
Marked before asking, so any answer (Ctrl-C included) counts. When it can't
85+
be written the offer is skipped, so a looping script never stops on it twice.
86+
"""
87+
try:
88+
(telemetry._ensure_config_dir() / OFFER_MARKER_NAME).touch(exist_ok=False)
89+
except OSError:
90+
return False
91+
return True
92+
93+
94+
def is_terminal(stream: TextIO | None) -> bool:
95+
"""True when the stream exists and is attached to a terminal.
96+
97+
Python sets a standard stream to ``None`` when the process has no handle
98+
for it (a closed descriptor, ``pythonw``, a detached Windows process).
99+
"""
100+
return stream is not None and stream.isatty()
101+
102+
103+
def print_offer() -> None:
104+
"""Print the Pro-version pitch and its link."""
105+
click.echo(click.style(" iFixAi Pro", bold=True))
106+
click.echo(" Claim your free fast audit")
107+
click.echo(click.style(f" {PRO_URL}", fg="cyan"))
108+
click.echo()
109+
110+
111+
def open_pro_site() -> None:
112+
"""Open the Pro website in the default browser; print the link when that fails."""
113+
if webbrowser.open(PRO_URL):
114+
click.echo(click.style(f"Opened {PRO_URL} in your browser.", fg="green"))
115+
return
116+
click.echo(
117+
click.style(
118+
f"Could not open a browser. Visit {PRO_URL} to claim your free fast audit.",
119+
fg="yellow",
120+
)
121+
)
122+
123+
124+
def ask_choice(question: ChoiceQuestion) -> str:
125+
"""Ask with an arrow-key menu, or a numbered list where menus are unavailable."""
126+
if ui.is_interactive():
127+
return ui.select_or_abort(
128+
question["message"], question["options"], question["default"]
129+
)
130+
return ask_numbered_choice(question)
131+
132+
133+
def ask_numbered_choice(question: ChoiceQuestion) -> str:
134+
"""List the options with numbers and read the number the user types.
135+
136+
The plain path for terminals where the arrow-key menu cannot run: ``NO_COLOR``
137+
is set, or ``questionary`` is not installed.
138+
"""
139+
options = question["options"]
140+
for number, option in enumerate(options, start=1):
141+
click.echo(f" [{number}] {option}")
142+
typed_number = click.prompt(
143+
question["message"],
144+
type=click.IntRange(1, len(options)),
145+
default=options.index(question["default"]) + 1,
146+
)
147+
return options[typed_number - 1]

‎ifixai/cli/run.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
_resolve_standard_eval_mode,
3636
execute_tests,
3737
)
38+
from ifixai.cli.pro_offer import NO_PROMO_ENV_VAR, RunDecision, offer_pro_version
3839
from ifixai.cli.reports import save_reports
3940
from ifixai.cli.schemas import InteractiveConfig
4041
from ifixai.core.concurrency import (
@@ -734,6 +735,16 @@ def _validate_min_score(ctx: click.Context, param: click.Parameter, value: float
734735
help="Disable pseudonymous run telemetry for this run. Set IFIXAI_TELEMETRY=0 or "
735736
"DO_NOT_TRACK=1 to disable it permanently.",
736737
)
738+
@click.option(
739+
"--no-promo",
740+
"no_promo",
741+
is_flag=True,
742+
default=False,
743+
envvar=NO_PROMO_ENV_VAR,
744+
help="Skip the Pro-version offer and start the run immediately. Set "
745+
f"{NO_PROMO_ENV_VAR}=1 to skip it on every run. The offer is never shown "
746+
"in CI or when input/output is not a terminal.",
747+
)
737748
@click.option(
738749
"--print-telemetry",
739750
"print_telemetry",
@@ -796,6 +807,7 @@ def run(
796807
grounding: str,
797808
quiet: bool,
798809
no_telemetry: bool,
810+
no_promo: bool,
799811
print_telemetry: bool,
800812
show_install_id: bool,
801813
) -> None:
@@ -938,6 +950,10 @@ def run(
938950
holdout_seed = resume_manifest.holdout_seed
939951

940952
print_startup_banner(IFIXAI_VERSION, quiet=quiet)
953+
# Asked before telemetry and any provider call, so abandoning leaves no trace.
954+
if not (no_promo or dry_run) and offer_pro_version() is RunDecision.ABANDON:
955+
click.echo("Run abandoned -- no inspections were run.")
956+
return
941957
if no_telemetry:
942958
telemetry.disable()
943959
telemetry.show_disclosure()

‎ifixai/cli/ui.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
_ACCENT = "rgb(232,99,42)"
1818
_DIM = "grey62"
19+
SELECT_INSTRUCTION = "(↑/↓ to move, enter to select)"
1920

2021

2122
@lru_cache(maxsize=1)
@@ -109,14 +110,33 @@ def select(
109110
default=default or choices[0],
110111
style=_qstyle(),
111112
use_shortcuts=False,
112-
instruction="(↑/↓ to move, enter to select)",
113+
instruction=SELECT_INSTRUCTION,
113114
)
114115
if descriptions:
115116
kwargs["show_description"] = True
116117
answer = questionary.select(message, **kwargs).ask()
117118
return answer if answer is not None else (default or choices[0])
118119

119120

121+
def select_or_abort(message: str, choices: Sequence[str], default: str) -> str:
122+
"""Arrow-key single choice where Ctrl-C aborts instead of picking the default.
123+
124+
``select`` answers a cancelled prompt with its default, which is wrong when
125+
the default starts work the user just tried to stop. Raises
126+
``KeyboardInterrupt`` on Ctrl-C. Callers must check ``is_interactive()`` first.
127+
"""
128+
import questionary
129+
130+
return questionary.select(
131+
message,
132+
choices=_choices(list(choices), None),
133+
default=default,
134+
style=_qstyle(),
135+
use_shortcuts=False,
136+
instruction=SELECT_INSTRUCTION,
137+
).unsafe_ask()
138+
139+
120140
def multiselect(
121141
message: str,
122142
choices: Sequence[str],

‎ifixai/telemetry.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,8 @@ def disable() -> None:
111111
_force_disabled = True
112112

113113

114-
def _in_ci() -> bool:
114+
def in_ci() -> bool:
115+
"""True when the process is running under an automated CI system."""
115116
ci = os.environ.get("CI")
116117
if ci is not None and ci.strip().lower() not in {"", "0", "false", "no"}:
117118
return True
@@ -127,7 +128,7 @@ def is_enabled() -> bool:
127128
return False
128129
if "DO_NOT_TRACK" in os.environ: # presence-based, any value incl. "0"
129130
return False
130-
if _in_ci():
131+
if in_ci():
131132
return False
132133
try:
133134
if _optout_path().exists():
Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
"""Real keystrokes through the arrow-key menu (needs ``questionary`` installed)."""
2+
3+
from collections.abc import Iterator
4+
from pathlib import Path
5+
from typing import Any
6+
7+
import pytest
8+
9+
from ifixai.cli import pro_offer, ui
10+
from ifixai.cli.pro_offer import PRO_URL, RunDecision
11+
12+
pytest.importorskip("questionary")
13+
application = pytest.importorskip("prompt_toolkit.application")
14+
keyboard = pytest.importorskip("prompt_toolkit.input")
15+
screen = pytest.importorskip("prompt_toolkit.output")
16+
17+
UP_ARROW = "\x1b[A"
18+
DOWN_ARROW = "\x1b[B"
19+
ENTER = "\r"
20+
CTRL_C = "\x03"
21+
22+
23+
class BrowserRecorder:
24+
"""Stands in for ``webbrowser.open`` and records the URLs it was asked for."""
25+
26+
def __init__(self) -> None:
27+
self.opened_urls: list[str] = []
28+
29+
def __call__(self, url: str) -> bool:
30+
self.opened_urls.append(url)
31+
return True
32+
33+
34+
def report_terminal_attached() -> bool:
35+
return True
36+
37+
38+
@pytest.fixture(autouse=True)
39+
def config_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
40+
"""Keeps the offer's shown-once marker in ``tmp_path``, never the real home."""
41+
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
42+
43+
44+
@pytest.fixture
45+
def keys() -> Iterator[Any]:
46+
"""A keyboard the test types on; the menus read from it instead of a console."""
47+
with keyboard.create_pipe_input() as pipe:
48+
with application.create_app_session(input=pipe, output=screen.DummyOutput()):
49+
yield pipe
50+
51+
52+
def test_enter_alone_picks_the_default(keys: Any) -> None:
53+
keys.send_text(ENTER)
54+
55+
assert ui.select_or_abort("Pick one", ["first", "second"], "second") == "second"
56+
57+
58+
def test_arrows_move_the_pointer_before_enter(keys: Any) -> None:
59+
keys.send_text(UP_ARROW + ENTER)
60+
61+
assert ui.select_or_abort("Pick one", ["first", "second"], "second") == "first"
62+
63+
64+
def test_ctrl_c_raises_instead_of_answering_with_the_default(keys: Any) -> None:
65+
keys.send_text(CTRL_C)
66+
67+
with pytest.raises(KeyboardInterrupt):
68+
ui.select_or_abort("Pick one", ["first", "second"], "second")
69+
70+
71+
def test_the_existing_select_still_answers_a_cancel_with_its_default(
72+
keys: Any, monkeypatch: pytest.MonkeyPatch
73+
) -> None:
74+
monkeypatch.setattr(ui, "is_interactive", report_terminal_attached)
75+
keys.send_text(CTRL_C)
76+
77+
assert ui.select("Pick one", ["first", "second"], default="second") == "second"
78+
79+
80+
@pytest.mark.parametrize(
81+
"typed,expected,expected_urls",
82+
[
83+
(ENTER, RunDecision.CONTINUE, []),
84+
(UP_ARROW + ENTER + ENTER, RunDecision.CONTINUE, [PRO_URL]),
85+
(UP_ARROW + ENTER + DOWN_ARROW + ENTER, RunDecision.ABANDON, [PRO_URL]),
86+
],
87+
)
88+
def test_the_offer_end_to_end_with_real_keystrokes(
89+
keys: Any,
90+
monkeypatch: pytest.MonkeyPatch,
91+
typed: str,
92+
expected: RunDecision,
93+
expected_urls: list[str],
94+
) -> None:
95+
browser = BrowserRecorder()
96+
monkeypatch.setattr(pro_offer.webbrowser, "open", browser)
97+
monkeypatch.setattr(pro_offer, "can_prompt_user", report_terminal_attached)
98+
monkeypatch.setattr(ui, "is_interactive", report_terminal_attached)
99+
keys.send_text(typed)
100+
101+
assert pro_offer.offer_pro_version() is expected
102+
assert browser.opened_urls == expected_urls

0 commit comments

Comments
 (0)