Skip to content

profile

ipw.cli.profile

Run profiling against an inference client.

profile(dataset_id, client_id, client_base_url, model, dataset_param, client_param, output_dir, max_queries, eval_client, eval_base_url, eval_model)

Execute profiling run with the execution pipeline.

Source code in intelligence-per-watt/src/ipw/cli/profile.py 
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
@click.command(help="Run profiling against an inference client.")
@click.option("--client", "client_id", required=True, help="Client identifier")
@click.option("--model", required=True, help="Model name to invoke")
@click.option("--dataset", "dataset_id", default="ipw", help="Dataset identifier")
@click.option("--client-base-url", help="Client base URL")
@click.option(
    "--eval-client",
    help="Evaluation client identifier (judge)",
    default="openai",
    show_default=True,
)
@click.option(
    "--eval-base-url",
    help="Evaluation client base URL",
    default="https://api.openai.com/v1",
    show_default=True,
)
@click.option(
    "--eval-model",
    help="Evaluation model to use for scoring",
    default="gpt-5-nano-2025-08-07",
    show_default=True,
)
@click.option(
    "--dataset-param",
    multiple=True,
    callback=_collect_params,
    help="Dataset params key=value",
)
@click.option(
    "--client-param",
    multiple=True,
    callback=_collect_params,
    help="Client params key=value",
)
@click.option("--output-dir", type=click.Path())
@click.option("--max-queries", type=int)
def profile(
    dataset_id: str,
    client_id: str,
    client_base_url: str | None,
    model: str,
    dataset_param,
    client_param,
    output_dir: str | None,
    max_queries: int | None,
    eval_client: str | None,
    eval_base_url: str | None,
    eval_model: str | None,
) -> None:
    """Execute profiling run with the execution pipeline."""
    import ipw.analysis
    import ipw.clients
    import ipw.datasets

    ipw.clients.ensure_registered()
    missing_reason = getattr(ipw.clients, "MISSING_CLIENTS", {}).get(client_id)
    if missing_reason:
        raise click.ClickException(
            f"Inference client '{client_id}' is unavailable: {missing_reason}"
        )

    ipw.datasets.ensure_registered()
    ipw.analysis.ensure_registered()

    from ipw.analysis.base import AnalysisContext
    from ipw.core.registry import AnalysisRegistry, DatasetRegistry
    from ipw.execution import ProfilerRunner  # Deferred import for heavy dependencies

    config = ProfilerConfig(
        dataset_id=dataset_id,
        client_id=client_id,
        client_base_url=client_base_url,
        dataset_params=dataset_param,
        client_params=client_param,
        model=model,
        max_queries=max_queries,
        output_dir=Path(output_dir) if output_dir else None,
    )

    # Preflight: dataset requirements (api keys, etc)
    try:
        dataset_cls = DatasetRegistry.get(dataset_id)
        dataset_instance = dataset_cls(**dataset_param)
        issues = dataset_instance.verify_requirements()
        if issues:
            raise click.ClickException(
                "Dataset requirements not satisfied:\n- " + "\n- ".join(issues)
            )
        _warn_on_custom_eval(dataset_instance, eval_client, eval_base_url, eval_model)
    except Exception as exc:
        raise click.ClickException(str(exc)) from exc

    runner = ProfilerRunner(config)
    runner.run()
    success("Profiling run completed")

    # Post-run analysis
    results_dir = runner._output_path
    if results_dir and results_dir.exists():
        info("Running post-profile analysis...")
        context = AnalysisContext(
            results_dir=results_dir,
            options={
                "model": model,
                "eval_client": eval_client,
                "eval_base_url": eval_base_url,
                "eval_model": eval_model,
            },
        )
        try:
            analysis = AnalysisRegistry.create("accuracy")
            result = analysis.run(context)
            _print_result(result, verbose=False)
        except Exception as e:
            warning(f"Warning: Analysis failed: {e}")