.. DO NOT EDIT.
.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
.. "tutorials/finetune_classifier.py"
.. LINE NUMBERS ARE GIVEN BELOW.

.. only:: html

    .. note::
        :class: sphx-glr-download-link-note

        :ref:`Go to the end <sphx_glr_download_tutorials_finetune_classifier.py>`
        to download the full example code.

.. rst-class:: sphx-glr-example-title

.. _sphx_glr_tutorials_finetune_classifier.py:


Fine-tuning TabICL for classification
=====================================

Adapt a pretrained TabICL classifier to a single dataset with
:class:`tabicl.FinetunedTabICLClassifier` (cross-entropy on raw logits,
same objective the pretrained head was fit with).

.. note::

    A CUDA GPU is recommended for large-scale fine-tuning. Multi-GPU via
    ``torchrun --nproc-per-node=N`` (auto-detected).

.. GENERATED FROM PYTHON SOURCE LINES 16-26

.. code-block:: Python

    import os

    import matplotlib.pyplot as plt
    import numpy as np
    from sklearn.metrics import accuracy_score, log_loss, roc_auc_score
    from sklearn.model_selection import train_test_split

    from tabicl import FinetunedTabICLClassifier, TabICLClassifier









.. GENERATED FROM PYTHON SOURCE LINES 27-29

Target: one moderate feature (curved split), one hard feature (disc)
--------------------------------------------------------------------

.. GENERATED FROM PYTHON SOURCE LINES 29-73

.. code-block:: Python

    ISLAND_CENTER = np.array([-1.5, 0.5], dtype=np.float32)
    ISLAND_RADIUS = 0.9
    MAIN_AMP = 0.7  # sine amplitude of the main boundary
    MAIN_FREQ = 1.2  # sine frequency of the main boundary


    def target_fn(X: np.ndarray) -> np.ndarray:
        y = (X[:, 0] + MAIN_AMP * np.sin(MAIN_FREQ * X[:, 1]) > 0).astype(np.int64)
        inside = np.sum((X - ISLAND_CENTER) ** 2, axis=1) < ISLAND_RADIUS**2
        return np.where(inside, 1, y)


    def make_dataset(n_samples: int = 1_500, random_state: int = 0):
        rng = np.random.RandomState(random_state)
        X = rng.uniform(-3.0, 3.0, size=(n_samples, 2)).astype(np.float32)
        y = target_fn(X)
        return X, y


    X, y = make_dataset(n_samples=1_500, random_state=0)

    # Split: 80 train (sparse at the disc) / 200 val (early stopping) / rest test.
    # Stratify on the joint (class, inside-disc) key so the training set
    # reliably captures ~5 disc points.
    in_island_all = np.sum((X - ISLAND_CENTER) ** 2, axis=1) < ISLAND_RADIUS**2
    strat_key = y.astype(int) * 2 + in_island_all.astype(int)

    X_train, X_rest, y_train, y_rest, _, strat_rest = train_test_split(
        X, y, strat_key, train_size=80, random_state=0, stratify=strat_key
    )
    X_val, X_test, y_val, y_test = train_test_split(X_rest, y_rest, train_size=200, random_state=0, stratify=strat_rest)

    is_main_process = int(os.environ.get("LOCAL_RANK", "0")) == 0


    def _metrics(proba: np.ndarray, y_true: np.ndarray) -> tuple[float, float, float]:
        preds = np.argmax(proba, axis=1)
        return (
            float(roc_auc_score(y_true, proba[:, 1])),
            float(log_loss(y_true, proba, labels=[0, 1])),
            float(accuracy_score(y_true, preds)),
        )









.. GENERATED FROM PYTHON SOURCE LINES 74-78

Baseline — zero-shot TabICL
---------------------------

Expected: draws the vertical split, smears the island.

.. GENERATED FROM PYTHON SOURCE LINES 78-86

.. code-block:: Python

    base = TabICLClassifier(n_estimators=4, random_state=0)
    base.fit(X_train, y_train)
    base_proba = base.predict_proba(X_test)
    base_auc, base_ll, base_acc = _metrics(base_proba, y_test)
    # Captured for the training-curve reference line in Figure 2.
    base_val_auc = float(roc_auc_score(y_val, base.predict_proba(X_val)[:, 1]))









.. GENERATED FROM PYTHON SOURCE LINES 87-93

Fine-tune
---------

``_HistoryLogger`` below is installed via the same
``_make_experiment_logger`` hook ``wandb_kwargs`` uses, to capture per-epoch
val metrics for Figure 2 without pulling in W&B.

.. GENERATED FROM PYTHON SOURCE LINES 93-138

.. code-block:: Python

    history: dict[str, list[float]] = {
        "epoch": [],
        "val_roc_auc": [],
        "val_log_loss": [],
        "val_accuracy": [],
        "train_loss": [],
    }


    class _HistoryLogger:
        """Record per-epoch validation metrics into ``history``."""

        def setup(self, config):
            del config

        def log_step(self, metrics, step):
            del metrics, step

        def log_epoch(self, metrics, step):
            del step
            history["epoch"].append(int(metrics.get("train/epoch", len(history["epoch"]))) + 1)
            history["val_roc_auc"].append(float(metrics.get("val/roc_auc", np.nan)))
            history["val_log_loss"].append(float(metrics.get("val/log_loss", np.nan)))
            history["val_accuracy"].append(float(metrics.get("val/accuracy", np.nan)))
            history["train_loss"].append(float(metrics.get("train/mean_loss", np.nan)))

        def finish(self):
            pass


    clf = FinetunedTabICLClassifier(
        epochs=60,
        learning_rate=1e-5,
        n_estimators_finetune=2,
        n_estimators_validation=2,
        n_estimators_inference=4,
        early_stopping=True,
        patience=10,
        eval_metric="roc_auc",
        random_state=0,
        verbose=True,
    )
    clf._make_experiment_logger = lambda: _HistoryLogger()
    clf.fit(X_train, y_train, X_val=X_val, y_val=y_val)





.. rst-class:: sphx-glr-script-out

 .. code-block:: none

    /home/docs/checkouts/readthedocs.org/user_builds/tabicl/checkouts/stable/tutorials/finetune_classifier.py:136: UserWarning: `output_dir` is not set; no checkpoints will be saved and all fine-tuning progress is lost if the run is interrupted.
      clf.fit(X_train, y_train, X_val=X_val, y_val=y_val)
    Baseline val roc_auc: 0.9811

    Fine-tune:   0%|          | 0/60 [00:00<?, ?it/s]
    Fine-tune:   0%|          | 0/60 [00:00<?, ?it/s, train_loss=0.2665, val_roc_auc=0.9811, best=0.9811, s/epoch=0.4]
    Fine-tune:   2%|▏         | 1/60 [00:00<00:37,  1.59it/s, train_loss=0.2665, val_roc_auc=0.9811, best=0.9811, s/epoch=0.4]
    Fine-tune:   2%|▏         | 1/60 [00:01<00:37,  1.59it/s, train_loss=0.1482, val_roc_auc=0.9806, best=0.9811, s/epoch=0.4]
    Fine-tune:   3%|▎         | 2/60 [00:01<00:36,  1.60it/s, train_loss=0.1482, val_roc_auc=0.9806, best=0.9811, s/epoch=0.4]
    Fine-tune:   3%|▎         | 2/60 [00:01<00:36,  1.60it/s, train_loss=0.0947, val_roc_auc=0.9815, best=0.9815, s/epoch=0.3]
    Fine-tune:   5%|▌         | 3/60 [00:01<00:36,  1.58it/s, train_loss=0.0947, val_roc_auc=0.9815, best=0.9815, s/epoch=0.3]
    Fine-tune:   5%|▌         | 3/60 [00:02<00:36,  1.58it/s, train_loss=0.1667, val_roc_auc=0.9835, best=0.9835, s/epoch=0.3]
    Fine-tune:   7%|▋         | 4/60 [00:02<00:35,  1.59it/s, train_loss=0.1667, val_roc_auc=0.9835, best=0.9835, s/epoch=0.3]
    Fine-tune:   7%|▋         | 4/60 [00:03<00:35,  1.59it/s, train_loss=0.7136, val_roc_auc=0.9857, best=0.9857, s/epoch=0.4]
    Fine-tune:   8%|▊         | 5/60 [00:03<00:34,  1.60it/s, train_loss=0.7136, val_roc_auc=0.9857, best=0.9857, s/epoch=0.4]
    Fine-tune:   8%|▊         | 5/60 [00:03<00:34,  1.60it/s, train_loss=0.6135, val_roc_auc=0.9892, best=0.9892, s/epoch=0.3]
    Fine-tune:  10%|█         | 6/60 [00:03<00:33,  1.60it/s, train_loss=0.6135, val_roc_auc=0.9892, best=0.9892, s/epoch=0.3]
    Fine-tune:  10%|█         | 6/60 [00:04<00:33,  1.60it/s, train_loss=0.0148, val_roc_auc=0.9911, best=0.9911, s/epoch=0.4]
    Fine-tune:  12%|█▏        | 7/60 [00:04<00:33,  1.59it/s, train_loss=0.0148, val_roc_auc=0.9911, best=0.9911, s/epoch=0.4]
    Fine-tune:  12%|█▏        | 7/60 [00:05<00:33,  1.59it/s, train_loss=0.0391, val_roc_auc=0.9916, best=0.9916, s/epoch=0.4]
    Fine-tune:  13%|█▎        | 8/60 [00:05<00:32,  1.59it/s, train_loss=0.0391, val_roc_auc=0.9916, best=0.9916, s/epoch=0.4]
    Fine-tune:  13%|█▎        | 8/60 [00:05<00:32,  1.59it/s, train_loss=0.0748, val_roc_auc=0.9921, best=0.9921, s/epoch=0.3]
    Fine-tune:  15%|█▌        | 9/60 [00:05<00:31,  1.60it/s, train_loss=0.0748, val_roc_auc=0.9921, best=0.9921, s/epoch=0.3]
    Fine-tune:  15%|█▌        | 9/60 [00:06<00:31,  1.60it/s, train_loss=0.2246, val_roc_auc=0.9922, best=0.9922, s/epoch=0.4]
    Fine-tune:  17%|█▋        | 10/60 [00:06<00:31,  1.60it/s, train_loss=0.2246, val_roc_auc=0.9922, best=0.9922, s/epoch=0.4]
    Fine-tune:  17%|█▋        | 10/60 [00:06<00:31,  1.60it/s, train_loss=0.2194, val_roc_auc=0.9924, best=0.9924, s/epoch=0.3]
    Fine-tune:  18%|█▊        | 11/60 [00:06<00:30,  1.60it/s, train_loss=0.2194, val_roc_auc=0.9924, best=0.9924, s/epoch=0.3]
    Fine-tune:  18%|█▊        | 11/60 [00:07<00:30,  1.60it/s, train_loss=0.1049, val_roc_auc=0.9920, best=0.9924, s/epoch=0.3]
    Fine-tune:  20%|██        | 12/60 [00:07<00:29,  1.61it/s, train_loss=0.1049, val_roc_auc=0.9920, best=0.9924, s/epoch=0.3]
    Fine-tune:  20%|██        | 12/60 [00:08<00:29,  1.61it/s, train_loss=0.0780, val_roc_auc=0.9921, best=0.9924, s/epoch=0.3]
    Fine-tune:  22%|██▏       | 13/60 [00:08<00:29,  1.62it/s, train_loss=0.0780, val_roc_auc=0.9921, best=0.9924, s/epoch=0.3]
    Fine-tune:  22%|██▏       | 13/60 [00:08<00:29,  1.62it/s, train_loss=0.1736, val_roc_auc=0.9925, best=0.9925, s/epoch=0.3]
    Fine-tune:  23%|██▎       | 14/60 [00:08<00:28,  1.62it/s, train_loss=0.1736, val_roc_auc=0.9925, best=0.9925, s/epoch=0.3]
    Fine-tune:  23%|██▎       | 14/60 [00:09<00:28,  1.62it/s, train_loss=0.2008, val_roc_auc=0.9922, best=0.9925, s/epoch=0.3]
    Fine-tune:  25%|██▌       | 15/60 [00:09<00:27,  1.62it/s, train_loss=0.2008, val_roc_auc=0.9922, best=0.9925, s/epoch=0.3]
    Fine-tune:  25%|██▌       | 15/60 [00:09<00:27,  1.62it/s, train_loss=0.2372, val_roc_auc=0.9920, best=0.9925, s/epoch=0.3]
    Fine-tune:  27%|██▋       | 16/60 [00:09<00:27,  1.63it/s, train_loss=0.2372, val_roc_auc=0.9920, best=0.9925, s/epoch=0.3]
    Fine-tune:  27%|██▋       | 16/60 [00:10<00:27,  1.63it/s, train_loss=0.1123, val_roc_auc=0.9919, best=0.9925, s/epoch=0.3]
    Fine-tune:  28%|██▊       | 17/60 [00:10<00:26,  1.63it/s, train_loss=0.1123, val_roc_auc=0.9919, best=0.9925, s/epoch=0.3]
    Fine-tune:  28%|██▊       | 17/60 [00:11<00:26,  1.63it/s, train_loss=0.2490, val_roc_auc=0.9920, best=0.9925, s/epoch=0.4]
    Fine-tune:  30%|███       | 18/60 [00:11<00:25,  1.63it/s, train_loss=0.2490, val_roc_auc=0.9920, best=0.9925, s/epoch=0.4]
    Fine-tune:  30%|███       | 18/60 [00:11<00:25,  1.63it/s, train_loss=0.0581, val_roc_auc=0.9924, best=0.9925, s/epoch=0.3]
    Fine-tune:  32%|███▏      | 19/60 [00:11<00:25,  1.63it/s, train_loss=0.0581, val_roc_auc=0.9924, best=0.9925, s/epoch=0.3]
    Fine-tune:  32%|███▏      | 19/60 [00:12<00:25,  1.63it/s, train_loss=0.1895, val_roc_auc=0.9924, best=0.9925, s/epoch=0.4]
    Fine-tune:  33%|███▎      | 20/60 [00:12<00:24,  1.63it/s, train_loss=0.1895, val_roc_auc=0.9924, best=0.9925, s/epoch=0.4]
    Fine-tune:  33%|███▎      | 20/60 [00:13<00:24,  1.63it/s, train_loss=0.0116, val_roc_auc=0.9924, best=0.9925, s/epoch=0.3]
    Fine-tune:  35%|███▌      | 21/60 [00:13<00:23,  1.63it/s, train_loss=0.0116, val_roc_auc=0.9924, best=0.9925, s/epoch=0.3]
    Fine-tune:  35%|███▌      | 21/60 [00:13<00:23,  1.63it/s, train_loss=0.2290, val_roc_auc=0.9919, best=0.9925, s/epoch=0.3]
    Fine-tune:  37%|███▋      | 22/60 [00:13<00:23,  1.64it/s, train_loss=0.2290, val_roc_auc=0.9919, best=0.9925, s/epoch=0.3]
    Fine-tune:  37%|███▋      | 22/60 [00:14<00:23,  1.64it/s, train_loss=0.1353, val_roc_auc=0.9917, best=0.9925, s/epoch=0.3]
    Fine-tune:  38%|███▊      | 23/60 [00:14<00:22,  1.63it/s, train_loss=0.1353, val_roc_auc=0.9917, best=0.9925, s/epoch=0.3]
    Fine-tune:  38%|███▊      | 23/60 [00:14<00:22,  1.63it/s, train_loss=0.1583, val_roc_auc=0.9915, best=0.9925, s/epoch=0.3]
    Fine-tune:  38%|███▊      | 23/60 [00:14<00:23,  1.55it/s, train_loss=0.1583, val_roc_auc=0.9915, best=0.9925, s/epoch=0.3]


.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <style>#sk-container-id-3 {
      /* Definition of color scheme common for light and dark mode */
      --sklearn-color-text: #000;
      --sklearn-color-text-muted: #666;
      --sklearn-color-line: gray;
      /* Definition of color scheme for unfitted estimators */
      --sklearn-color-unfitted-level-0: #fff5e6;
      --sklearn-color-unfitted-level-1: #f6e4d2;
      --sklearn-color-unfitted-level-2: #ffe0b3;
      --sklearn-color-unfitted-level-3: chocolate;
      /* Definition of color scheme for fitted estimators */
      --sklearn-color-fitted-level-0: #f0f8ff;
      --sklearn-color-fitted-level-1: #d4ebff;
      --sklearn-color-fitted-level-2: #b3dbfd;
      --sklearn-color-fitted-level-3: cornflowerblue;
    }

    #sk-container-id-3.light {
      /* Specific color for light theme */
      --sklearn-color-text-on-default-background: black;
      --sklearn-color-background: white;
      --sklearn-color-border-box: black;
      --sklearn-color-icon: #696969;
    }

    #sk-container-id-3.dark {
      --sklearn-color-text-on-default-background: white;
      --sklearn-color-background: #111;
      --sklearn-color-border-box: white;
      --sklearn-color-icon: #878787;
    }

    #sk-container-id-3 {
      color: var(--sklearn-color-text);
    }

    #sk-container-id-3 pre {
      padding: 0;
    }

    #sk-container-id-3 input.sk-hidden--visually {
      border: 0;
      clip: rect(1px 1px 1px 1px);
      clip: rect(1px, 1px, 1px, 1px);
      height: 1px;
      margin: -1px;
      overflow: hidden;
      padding: 0;
      position: absolute;
      width: 1px;
    }

    #sk-container-id-3 div.sk-dashed-wrapped {
      border: 1px dashed var(--sklearn-color-line);
      margin: 0 0.4em 0.5em 0.4em;
      box-sizing: border-box;
      padding-bottom: 0.4em;
      background-color: var(--sklearn-color-background);
    }

    #sk-container-id-3 div.sk-container {
      /* jupyter's `normalize.less` sets `[hidden] { display: none; }`
         but bootstrap.min.css set `[hidden] { display: none !important; }`
         so we also need the `!important` here to be able to override the
         default hidden behavior on the sphinx rendered scikit-learn.org.
         See: https://github.com/scikit-learn/scikit-learn/issues/21755 */
      display: inline-block !important;
      position: relative;
    }

    #sk-container-id-3 div.sk-text-repr-fallback {
      display: none;
    }

    div.sk-parallel-item,
    div.sk-serial,
    div.sk-item {
      /* draw centered vertical line to link estimators */
      background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));
      background-size: 2px 100%;
      background-repeat: no-repeat;
      background-position: center center;
    }

    /* Parallel-specific style estimator block */

    #sk-container-id-3 div.sk-parallel-item::after {
      content: "";
      width: 100%;
      border-bottom: 2px solid var(--sklearn-color-text-on-default-background);
      flex-grow: 1;
    }

    #sk-container-id-3 div.sk-parallel {
      display: flex;
      align-items: stretch;
      justify-content: center;
      background-color: var(--sklearn-color-background);
      position: relative;
    }

    #sk-container-id-3 div.sk-parallel-item {
      display: flex;
      flex-direction: column;
    }

    #sk-container-id-3 div.sk-parallel-item:first-child::after {
      align-self: flex-end;
      width: 50%;
    }

    #sk-container-id-3 div.sk-parallel-item:last-child::after {
      align-self: flex-start;
      width: 50%;
    }

    #sk-container-id-3 div.sk-parallel-item:only-child::after {
      width: 0;
    }

    /* Serial-specific style estimator block */

    #sk-container-id-3 div.sk-serial {
      display: flex;
      flex-direction: column;
      align-items: center;
      background-color: var(--sklearn-color-background);
      padding-right: 1em;
      padding-left: 1em;
    }


    /* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is
    clickable and can be expanded/collapsed.
    - Pipeline and ColumnTransformer use this feature and define the default style
    - Estimators will overwrite some part of the style using the `sk-estimator` class
    */

    /* Pipeline and ColumnTransformer style (default) */

    #sk-container-id-3 div.sk-toggleable {
      /* Default theme specific background. It is overwritten whether we have a
      specific estimator or a Pipeline/ColumnTransformer */
      background-color: var(--sklearn-color-background);
    }

    /* Toggleable label */
    #sk-container-id-3 label.sk-toggleable__label {
      cursor: pointer;
      display: flex;
      width: 100%;
      margin-bottom: 0;
      padding: 0.5em;
      box-sizing: border-box;
      text-align: center;
      align-items: center;
      justify-content: center;
      gap: 0.5em;
    }

    #sk-container-id-3 label.sk-toggleable__label .caption {
      font-size: 0.6rem;
      font-weight: lighter;
      color: var(--sklearn-color-text-muted);
    }

    #sk-container-id-3 label.sk-toggleable__label-arrow:before {
      /* Arrow on the left of the label */
      content: "▸";
      float: left;
      margin-right: 0.25em;
      color: var(--sklearn-color-icon);
    }

    #sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {
      color: var(--sklearn-color-text);
    }

    /* Toggleable content - dropdown */

    #sk-container-id-3 div.sk-toggleable__content {
      display: none;
      text-align: left;
      /* unfitted */
      background-color: var(--sklearn-color-unfitted-level-0);
    }

    #sk-container-id-3 div.sk-toggleable__content.fitted {
      /* fitted */
      background-color: var(--sklearn-color-fitted-level-0);
    }

    #sk-container-id-3 div.sk-toggleable__content pre {
      margin: 0.2em;
      border-radius: 0.25em;
      color: var(--sklearn-color-text);
      /* unfitted */
      background-color: var(--sklearn-color-unfitted-level-0);
    }

    #sk-container-id-3 div.sk-toggleable__content.fitted pre {
      /* unfitted */
      background-color: var(--sklearn-color-fitted-level-0);
    }

    #sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {
      /* Expand drop-down */
      display: block;
      width: 100%;
      overflow: visible;
    }

    #sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {
      content: "▾";
    }

    /* Pipeline/ColumnTransformer-specific style */

    #sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {
      color: var(--sklearn-color-text);
      background-color: var(--sklearn-color-unfitted-level-2);
    }

    #sk-container-id-3 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {
      background-color: var(--sklearn-color-fitted-level-2);
    }

    /* Estimator-specific style */

    /* Colorize estimator box */
    #sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {
      /* unfitted */
      background-color: var(--sklearn-color-unfitted-level-2);
    }

    #sk-container-id-3 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {
      /* fitted */
      background-color: var(--sklearn-color-fitted-level-2);
    }

    #sk-container-id-3 div.sk-label label.sk-toggleable__label,
    #sk-container-id-3 div.sk-label label {
      /* The background is the default theme color */
      color: var(--sklearn-color-text-on-default-background);
    }

    /* On hover, darken the color of the background */
    #sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {
      color: var(--sklearn-color-text);
      background-color: var(--sklearn-color-unfitted-level-2);
    }

    /* Label box, darken color on hover, fitted */
    #sk-container-id-3 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {
      color: var(--sklearn-color-text);
      background-color: var(--sklearn-color-fitted-level-2);
    }

    /* Estimator label */

    #sk-container-id-3 div.sk-label label {
      font-family: monospace;
      font-weight: bold;
      line-height: 1.2em;
    }

    #sk-container-id-3 div.sk-label-container {
      text-align: center;
    }

    /* Estimator-specific */
    #sk-container-id-3 div.sk-estimator {
      font-family: monospace;
      border: 1px dotted var(--sklearn-color-border-box);
      border-radius: 0.25em;
      box-sizing: border-box;
      margin-bottom: 0.5em;
      /* unfitted */
      background-color: var(--sklearn-color-unfitted-level-0);
    }

    #sk-container-id-3 div.sk-estimator.fitted {
      /* fitted */
      background-color: var(--sklearn-color-fitted-level-0);
    }

    /* on hover */
    #sk-container-id-3 div.sk-estimator:hover {
      /* unfitted */
      background-color: var(--sklearn-color-unfitted-level-2);
    }

    #sk-container-id-3 div.sk-estimator.fitted:hover {
      /* fitted */
      background-color: var(--sklearn-color-fitted-level-2);
    }

    /* Specification for estimator info (e.g. "i" and "?") */

    /* Common style for "i" and "?" */

    .sk-estimator-doc-link,
    a:link.sk-estimator-doc-link,
    a:visited.sk-estimator-doc-link {
      float: right;
      font-size: smaller;
      line-height: 1em;
      font-family: monospace;
      background-color: var(--sklearn-color-unfitted-level-0);
      border-radius: 1em;
      height: 1em;
      width: 1em;
      text-decoration: none !important;
      margin-left: 0.5em;
      text-align: center;
      /* unfitted */
      border: var(--sklearn-color-unfitted-level-3) 1pt solid;
      color: var(--sklearn-color-unfitted-level-3);
    }

    .sk-estimator-doc-link.fitted,
    a:link.sk-estimator-doc-link.fitted,
    a:visited.sk-estimator-doc-link.fitted {
      /* fitted */
      background-color: var(--sklearn-color-fitted-level-0);
      border: var(--sklearn-color-fitted-level-3) 1pt solid;
      color: var(--sklearn-color-fitted-level-3);
    }

    /* On hover */
    div.sk-estimator:hover .sk-estimator-doc-link:hover,
    .sk-estimator-doc-link:hover,
    div.sk-label-container:hover .sk-estimator-doc-link:hover,
    .sk-estimator-doc-link:hover {
      /* unfitted */
      background-color: var(--sklearn-color-unfitted-level-3);
      border: var(--sklearn-color-fitted-level-0) 1pt solid;
      color: var(--sklearn-color-unfitted-level-0);
      text-decoration: none;
    }

    div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,
    .sk-estimator-doc-link.fitted:hover,
    div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,
    .sk-estimator-doc-link.fitted:hover {
      /* fitted */
      background-color: var(--sklearn-color-fitted-level-3);
      border: var(--sklearn-color-fitted-level-0) 1pt solid;
      color: var(--sklearn-color-fitted-level-0);
      text-decoration: none;
    }

    /* Span, style for the box shown on hovering the info icon */
    .sk-estimator-doc-link span {
      display: none;
      z-index: 9999;
      position: relative;
      font-weight: normal;
      right: .2ex;
      padding: .5ex;
      margin: .5ex;
      width: min-content;
      min-width: 20ex;
      max-width: 50ex;
      color: var(--sklearn-color-text);
      box-shadow: 2pt 2pt 4pt #999;
      /* unfitted */
      background: var(--sklearn-color-unfitted-level-0);
      border: .5pt solid var(--sklearn-color-unfitted-level-3);
    }

    .sk-estimator-doc-link.fitted span {
      /* fitted */
      background: var(--sklearn-color-fitted-level-0);
      border: var(--sklearn-color-fitted-level-3);
    }

    .sk-estimator-doc-link:hover span {
      display: block;
    }

    /* "?"-specific style due to the `<a>` HTML tag */

    #sk-container-id-3 a.estimator_doc_link {
      float: right;
      font-size: 1rem;
      line-height: 1em;
      font-family: monospace;
      background-color: var(--sklearn-color-unfitted-level-0);
      border-radius: 1rem;
      height: 1rem;
      width: 1rem;
      text-decoration: none;
      /* unfitted */
      color: var(--sklearn-color-unfitted-level-1);
      border: var(--sklearn-color-unfitted-level-1) 1pt solid;
    }

    #sk-container-id-3 a.estimator_doc_link.fitted {
      /* fitted */
      background-color: var(--sklearn-color-fitted-level-0);
      border: var(--sklearn-color-fitted-level-1) 1pt solid;
      color: var(--sklearn-color-fitted-level-1);
    }

    /* On hover */
    #sk-container-id-3 a.estimator_doc_link:hover {
      /* unfitted */
      background-color: var(--sklearn-color-unfitted-level-3);
      color: var(--sklearn-color-background);
      text-decoration: none;
    }

    #sk-container-id-3 a.estimator_doc_link.fitted:hover {
      /* fitted */
      background-color: var(--sklearn-color-fitted-level-3);
    }

    .estimator-table {
        font-family: monospace;
    }

    .estimator-table summary {
        padding: .5rem;
        cursor: pointer;
    }

    .estimator-table summary::marker {
        font-size: 0.7rem;
    }

    .estimator-table details[open] {
        padding-left: 0.1rem;
        padding-right: 0.1rem;
        padding-bottom: 0.3rem;
    }

    .estimator-table .parameters-table {
        margin-left: auto !important;
        margin-right: auto !important;
        margin-top: 0;
    }

    .estimator-table .parameters-table tr:nth-child(odd) {
        background-color: #fff;
    }

    .estimator-table .parameters-table tr:nth-child(even) {
        background-color: #f6f6f6;
    }

    .estimator-table .parameters-table tr:hover {
        background-color: #e0e0e0;
    }

    .estimator-table table td {
        border: 1px solid rgba(106, 105, 104, 0.232);
    }

    /*
        `table td`is set in notebook with right text-align.
        We need to overwrite it.
    */
    .estimator-table table td.param {
        text-align: left;
        position: relative;
        padding: 0;
    }

    .user-set td {
        color:rgb(255, 94, 0);
        text-align: left !important;
    }

    .user-set td.value {
        color:rgb(255, 94, 0);
        background-color: transparent;
    }

    .default td {
        color: black;
        text-align: left !important;
    }

    .user-set td i,
    .default td i {
        color: black;
    }

    /*
        Styles for parameter documentation links
        We need styling for visited so jupyter doesn't overwrite it
    */
    a.param-doc-link,
    a.param-doc-link:link,
    a.param-doc-link:visited {
        text-decoration: underline dashed;
        text-underline-offset: .3em;
        color: inherit;
        display: block;
        padding: .5em;
    }

    /* "hack" to make the entire area of the cell containing the link clickable */
    a.param-doc-link::before {
        position: absolute;
        content: "";
        inset: 0;
    }

    .param-doc-description {
        display: none;
        position: absolute;
        z-index: 9999;
        left: 0;
        padding: .5ex;
        margin-left: 1.5em;
        color: var(--sklearn-color-text);
        box-shadow: .3em .3em .4em #999;
        width: max-content;
        text-align: left;
        max-height: 10em;
        overflow-y: auto;

        /* unfitted */
        background: var(--sklearn-color-unfitted-level-0);
        border: thin solid var(--sklearn-color-unfitted-level-3);
    }

    /* Fitted state for parameter tooltips */
    .fitted .param-doc-description {
        /* fitted */
        background: var(--sklearn-color-fitted-level-0);
        border: thin solid var(--sklearn-color-fitted-level-3);
    }

    .param-doc-link:hover .param-doc-description {
        display: block;
    }

    .copy-paste-icon {
        background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0NDggNTEyIj48IS0tIUZvbnQgQXdlc29tZSBGcmVlIDYuNy4yIGJ5IEBmb250YXdlc29tZSAtIGh0dHBzOi8vZm9udGF3ZXNvbWUuY29tIExpY2Vuc2UgLSBodHRwczovL2ZvbnRhd2Vzb21lLmNvbS9saWNlbnNlL2ZyZWUgQ29weXJpZ2h0IDIwMjUgRm9udGljb25zLCBJbmMuLS0+PHBhdGggZD0iTTIwOCAwTDMzMi4xIDBjMTIuNyAwIDI0LjkgNS4xIDMzLjkgMTQuMWw2Ny45IDY3LjljOSA5IDE0LjEgMjEuMiAxNC4xIDMzLjlMNDQ4IDMzNmMwIDI2LjUtMjEuNSA0OC00OCA0OGwtMTkyIDBjLTI2LjUgMC00OC0yMS41LTQ4LTQ4bDAtMjg4YzAtMjYuNSAyMS41LTQ4IDQ4LTQ4ek00OCAxMjhsODAgMCAwIDY0LTY0IDAgMCAyNTYgMTkyIDAgMC0zMiA2NCAwIDAgNDhjMCAyNi41LTIxLjUgNDgtNDggNDhMNDggNTEyYy0yNi41IDAtNDgtMjEuNS00OC00OEwwIDE3NmMwLTI2LjUgMjEuNS00OCA0OC00OHoiLz48L3N2Zz4=);
        background-repeat: no-repeat;
        background-size: 14px 14px;
        background-position: 0;
        display: inline-block;
        width: 14px;
        height: 14px;
        cursor: pointer;
    }
    </style><body><div id="sk-container-id-3" class="sk-top-container"><div class="sk-text-repr-fallback"><pre>FinetunedTabICLClassifier(epochs=60, n_estimators_inference=4, patience=10,
                              random_state=0, verbose=True)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class="sk-container" hidden><div class="sk-item"><div class="sk-estimator fitted sk-toggleable"><input class="sk-toggleable__control sk-hidden--visually" id="sk-estimator-id-3" type="checkbox" checked><label for="sk-estimator-id-3" class="sk-toggleable__label fitted sk-toggleable__label-arrow"><div><div>FinetunedTabICLClassifier</div></div><div><span class="sk-estimator-doc-link fitted">i<span>Fitted</span></span></div></label><div class="sk-toggleable__content fitted" data-param-prefix="">
            <div class="estimator-table">
                <details>
                    <summary>Parameters</summary>
                    <table class="parameters-table">
                      <tbody>
                    
            <tr class="user-set">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('epochs',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">epochs</td>
                <td class="value">60</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('learning_rate',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">learning_rate</td>
                <td class="value">1e-05</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('weight_decay',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">weight_decay</td>
                <td class="value">0.01</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('grad_clip',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">grad_clip</td>
                <td class="value">1.0</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('amp',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">amp</td>
                <td class="value">True</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('use_lr_scheduler',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">use_lr_scheduler</td>
                <td class="value">True</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('warmup_proportion',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">warmup_proportion</td>
                <td class="value">0.1</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('n_estimators_finetune',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">n_estimators_finetune</td>
                <td class="value">2</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('n_estimators_validation',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">n_estimators_validation</td>
                <td class="value">2</td>
            </tr>
    

            <tr class="user-set">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('n_estimators_inference',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">n_estimators_inference</td>
                <td class="value">4</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('max_data_size',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">max_data_size</td>
                <td class="value">10000</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('finetune_ctx_query_ratio',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">finetune_ctx_query_ratio</td>
                <td class="value">0.2</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('validation_split_ratio',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">validation_split_ratio</td>
                <td class="value">0.1</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('early_stopping',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">early_stopping</td>
                <td class="value">True</td>
            </tr>
    

            <tr class="user-set">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('patience',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">patience</td>
                <td class="value">10</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('min_delta',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">min_delta</td>
                <td class="value">0.0001</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('time_limit',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">time_limit</td>
                <td class="value">None</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('save_interval',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">save_interval</td>
                <td class="value">1</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('norm_methods',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">norm_methods</td>
                <td class="value">None</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('feat_shuffle_method',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">feat_shuffle_method</td>
                <td class="value">&#x27;latin&#x27;</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('outlier_threshold',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">outlier_threshold</td>
                <td class="value">4.0</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('model_path',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">model_path</td>
                <td class="value">None</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('allow_auto_download',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">allow_auto_download</td>
                <td class="value">True</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('checkpoint_version',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">checkpoint_version</td>
                <td class="value">&#x27;tabicl-classifier-v2-20260212.ckpt&#x27;</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('freeze_col',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">freeze_col</td>
                <td class="value">False</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('freeze_row',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">freeze_row</td>
                <td class="value">False</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('freeze_icl',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">freeze_icl</td>
                <td class="value">False</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('device',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">device</td>
                <td class="value">None</td>
            </tr>
    

            <tr class="user-set">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('random_state',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">random_state</td>
                <td class="value">0</td>
            </tr>
    

            <tr class="user-set">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('verbose',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">verbose</td>
                <td class="value">True</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('wandb_kwargs',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">wandb_kwargs</td>
                <td class="value">None</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('class_shuffle_method',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">class_shuffle_method</td>
                <td class="value">&#x27;shift&#x27;</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('softmax_temperature',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">softmax_temperature</td>
                <td class="value">0.9</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('average_logits',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">average_logits</td>
                <td class="value">True</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('support_many_classes',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">support_many_classes</td>
                <td class="value">True</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('eval_metric',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">eval_metric</td>
                <td class="value">&#x27;roc_auc&#x27;</td>
            </tr>
    

            <tr class="default">
                <td><i class="copy-paste-icon"
                     onclick="copyToClipboard('extra_classifier_kwargs',
                              this.parentElement.nextElementSibling)"
                ></i></td>
                <td class="param">extra_classifier_kwargs</td>
                <td class="value">None</td>
            </tr>
    
                      </tbody>
                    </table>
                </details>
            </div>
        </div></div></div></div></div><script>function copyToClipboard(text, element) {
        // Get the parameter prefix from the closest toggleable content
        const toggleableContent = element.closest('.sk-toggleable__content');
        const paramPrefix = toggleableContent ? toggleableContent.dataset.paramPrefix : '';
        const fullParamName = paramPrefix ? `${paramPrefix}${text}` : text;

        const originalStyle = element.style;
        const computedStyle = window.getComputedStyle(element);
        const originalWidth = computedStyle.width;
        const originalHTML = element.innerHTML.replace('Copied!', '');

        navigator.clipboard.writeText(fullParamName)
            .then(() => {
                element.style.width = originalWidth;
                element.style.color = 'green';
                element.innerHTML = "Copied!";

                setTimeout(() => {
                    element.innerHTML = originalHTML;
                    element.style = originalStyle;
                }, 2000);
            })
            .catch(err => {
                console.error('Failed to copy:', err);
                element.style.color = 'red';
                element.innerHTML = "Failed!";
                setTimeout(() => {
                    element.innerHTML = originalHTML;
                    element.style = originalStyle;
                }, 2000);
            });
        return false;
    }

    document.querySelectorAll('.copy-paste-icon').forEach(function(element) {
        const toggleableContent = element.closest('.sk-toggleable__content');
        const paramPrefix = toggleableContent ? toggleableContent.dataset.paramPrefix : '';
        const paramName = element.parentElement.nextElementSibling
            .textContent.trim().split(' ')[0];
        const fullParamName = paramPrefix ? `${paramPrefix}${paramName}` : paramName;

        element.setAttribute('title', fullParamName);
    });


    /**
     * Adapted from Skrub
     * https://github.com/skrub-data/skrub/blob/403466d1d5d4dc76a7ef569b3f8228db59a31dc3/skrub/_reporting/_data/templates/report.js#L789
     * @returns "light" or "dark"
     */
    function detectTheme(element) {
        const body = document.querySelector('body');

        // Check VSCode theme
        const themeKindAttr = body.getAttribute('data-vscode-theme-kind');
        const themeNameAttr = body.getAttribute('data-vscode-theme-name');

        if (themeKindAttr && themeNameAttr) {
            const themeKind = themeKindAttr.toLowerCase();
            const themeName = themeNameAttr.toLowerCase();

            if (themeKind.includes("dark") || themeName.includes("dark")) {
                return "dark";
            }
            if (themeKind.includes("light") || themeName.includes("light")) {
                return "light";
            }
        }

        // Check Jupyter theme
        if (body.getAttribute('data-jp-theme-light') === 'false') {
            return 'dark';
        } else if (body.getAttribute('data-jp-theme-light') === 'true') {
            return 'light';
        }

        // Guess based on a parent element's color
        const color = window.getComputedStyle(element.parentNode, null).getPropertyValue('color');
        const match = color.match(/^rgb\s*\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\)\s*$/i);
        if (match) {
            const [r, g, b] = [
                parseFloat(match[1]),
                parseFloat(match[2]),
                parseFloat(match[3])
            ];

            // https://en.wikipedia.org/wiki/HSL_and_HSV#Lightness
            const luma = 0.299 * r + 0.587 * g + 0.114 * b;

            if (luma > 180) {
                // If the text is very bright we have a dark theme
                return 'dark';
            }
            if (luma < 75) {
                // If the text is very dark we have a light theme
                return 'light';
            }
            // Otherwise fall back to the next heuristic.
        }

        // Fallback to system preference
        return window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
    }


    function forceTheme(elementId) {
        const estimatorElement = document.querySelector(`#${elementId}`);
        if (estimatorElement === null) {
            console.error(`Element with id ${elementId} not found.`);
        } else {
            const theme = detectTheme(estimatorElement);
            estimatorElement.classList.add(theme);
        }
    }

    forceTheme('sk-container-id-3');</script></body>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 139-141

Evaluate on the held-out test set
---------------------------------

.. GENERATED FROM PYTHON SOURCE LINES 141-158

.. code-block:: Python

    ft_proba = clf.predict_proba(X_test)
    ft_auc, ft_ll, ft_acc = _metrics(ft_proba, y_test)

    if is_main_process:
        header = f"{'metric':<12}{'pretrained':>14}{'fine-tuned':>14}{'Δ':>14}"
        rule = "=" * len(header)
        print()
        print(rule)
        print(f"Test-set metrics  (n_train={len(X_train)}, n_test={len(X_test)})")
        print(rule)
        print(header)
        print("-" * len(header))
        print(f"{'ROC-AUC ↑':<12}{base_auc:>14.4f}{ft_auc:>14.4f}{ft_auc - base_auc:>+14.4f}")
        print(f"{'log-loss ↓':<12}{base_ll:>14.4f}{ft_ll:>14.4f}{ft_ll - base_ll:>+14.4f}")
        print(f"{'accuracy ↑':<12}{base_acc:>14.4f}{ft_acc:>14.4f}{ft_acc - base_acc:>+14.4f}")
        print(rule)





.. rst-class:: sphx-glr-script-out

 .. code-block:: none


    ======================================================
    Test-set metrics  (n_train=80, n_test=1220)
    ======================================================
    metric          pretrained    fine-tuned             Δ
    ------------------------------------------------------
    ROC-AUC ↑           0.9790        0.9898       +0.0109
    log-loss ↓          0.1880        0.1278       -0.0602
    accuracy ↑          0.9057        0.9393       +0.0336
    ======================================================




.. GENERATED FROM PYTHON SOURCE LINES 159-165

Figure 1 — Decision boundaries + probability contours
-----------------------------------------------------

Dashed black curve = true main boundary. Dashed yellow ring = true
disc boundary. Panel titles split accuracy into inside-disc vs
outside-disc so the localized improvement is visible at a glance.

.. GENERATED FROM PYTHON SOURCE LINES 165-254

.. code-block:: Python

    if is_main_process:
        h = 0.15
        xx, yy = np.meshgrid(
            np.arange(-3.0, 3.0 + h, h),
            np.arange(-3.0, 3.0 + h, h),
        )
        grid = np.c_[xx.ravel(), yy.ravel()].astype(np.float32)
        p_base = base.predict_proba(grid)[:, 1].reshape(xx.shape)
        p_ft = clf.predict_proba(grid)[:, 1].reshape(xx.shape)

        base_pred = np.argmax(base_proba, axis=1)
        ft_pred = np.argmax(ft_proba, axis=1)
        in_island_test = np.sum((X_test - ISLAND_CENTER) ** 2, axis=1) < ISLAND_RADIUS**2

        # Precompute the true main boundary curve x₁ = −MAIN_AMP·sin(MAIN_FREQ·x₂)
        x2_curve = np.linspace(-3.0, 3.0, 400)
        x1_curve = -MAIN_AMP * np.sin(MAIN_FREQ * x2_curve)

        fig1, axes = plt.subplots(1, 2, figsize=(12.0, 5.4), sharex=True, sharey=True, constrained_layout=True)
        cf = None
        # Consistent emerald for every "ground truth" reference so they read
        # as a group, distinct from the solid-black model decision contour.
        TRUTH_COLOR = "#10b981"
        for ax, title, grid_p, preds, (auc, acc) in [
            (axes[0], "Pretrained TabICL", p_base, base_pred, (base_auc, base_acc)),
            (axes[1], "Fine-tuned TabICL", p_ft, ft_pred, (ft_auc, ft_acc)),
        ]:
            cf = ax.contourf(xx, yy, grid_p, levels=20, cmap="RdYlBu_r", alpha=0.85, vmin=0.0, vmax=1.0)
            # 0.5 decision contour, heavy black — the model's own boundary.
            ax.contour(xx, yy, grid_p, levels=[0.5], colors="black", linewidths=2.0)
            # True main boundary (sine curve) — dashed reference, shared across
            # panels so the comparison with the model boundary is direct.
            ax.plot(x1_curve, x2_curve, color=TRUTH_COLOR, lw=2.0, ls="--", label="true main boundary")
            # True disc boundary — both panels share this reference too.
            theta = np.linspace(0, 2 * np.pi, 200)
            ax.plot(
                ISLAND_CENTER[0] + ISLAND_RADIUS * np.cos(theta),
                ISLAND_CENTER[1] + ISLAND_RADIUS * np.sin(theta),
                color=TRUTH_COLOR,
                lw=2.4,
                ls="--",
                label="true disc boundary",
            )
            # Training data (shape-coded by true label).
            m0 = y_train == 0
            m1 = y_train == 1
            ax.scatter(
                X_train[m0, 0],
                X_train[m0, 1],
                marker="o",
                c="#1d4ed8",
                s=46,
                edgecolor="white",
                linewidths=1.0,
                label="train y=0",
            )
            ax.scatter(
                X_train[m1, 0],
                X_train[m1, 1],
                marker="s",
                c="#b91c1c",
                s=46,
                edgecolor="white",
                linewidths=1.0,
                label="train y=1",
            )
            # Split test-set accuracy into "inside the disc" vs "outside"
            # to quantify the localized improvement directly in the title.
            acc_in = (
                float((preds[in_island_test] == y_test[in_island_test]).mean()) if in_island_test.any() else float("nan")
            )
            acc_out = (
                float((preds[~in_island_test] == y_test[~in_island_test]).mean())
                if (~in_island_test).any()
                else float("nan")
            )
            ax.set_title(f"{title}\nROC-AUC={auc:.3f}  acc={acc:.3f}", fontsize=12)
            ax.set_xlabel("x₁", fontsize=11)
            ax.set_xlim(-3, 3)
            ax.set_ylim(-3, 3)
            ax.tick_params(labelsize=10)
            ax.grid(alpha=0.25)
        axes[0].set_ylabel("x₂", fontsize=11)
        axes[0].legend(loc="lower right", framealpha=0.92, fontsize=9)
        cbar = fig1.colorbar(cf, ax=axes, shrink=0.85)
        cbar.set_label("P(class 1)", fontsize=11)
        cbar.ax.tick_params(labelsize=9)
        fig1.suptitle("Decision boundaries: pretrained vs. fine-tuned", fontsize=14)




.. image-sg:: /tutorials/images/sphx_glr_finetune_classifier_001.png
   :alt: Decision boundaries: pretrained vs. fine-tuned, Pretrained TabICL ROC-AUC=0.979  acc=0.906, Fine-tuned TabICL ROC-AUC=0.990  acc=0.939
   :srcset: /tutorials/images/sphx_glr_finetune_classifier_001.png
   :class: sphx-glr-single-img





.. GENERATED FROM PYTHON SOURCE LINES 255-261

Figure 2 — Training dynamics + metric comparison
------------------------------------------------

Left: val ROC-AUC per epoch; dashed line = pretrained floor, star =
best epoch kept by the safety net. Right: test-set ROC-AUC / log-loss /
accuracy bars.

.. GENERATED FROM PYTHON SOURCE LINES 261-321

.. code-block:: Python

    if is_main_process and history["epoch"]:
        fig2, (ax_tr, ax_bar) = plt.subplots(1, 2, figsize=(12.8, 4.8), constrained_layout=True)

        ep = history["epoch"]
        val_auc = history["val_roc_auc"]
        ax_tr.plot(ep, val_auc, "o-", color="#0f766e", lw=2.0, markersize=5, label="fine-tuning: val ROC-AUC")
        ax_tr.axhline(
            base_val_auc,
            ls="--",
            color="#64748b",
            lw=1.5,
            label=f"pretrained baseline ({base_val_auc:.3f})",
        )
        best_idx = int(np.nanargmax(val_auc))
        ax_tr.scatter(
            [ep[best_idx]],
            [val_auc[best_idx]],
            marker="*",
            s=220,
            color="#f59e0b",
            edgecolor="black",
            linewidths=0.8,
            zorder=5,
            label=f"best epoch ({val_auc[best_idx]:.3f} @ epoch {ep[best_idx]})",
        )
        ax_tr.set_xlabel("epoch")
        ax_tr.set_ylabel("validation ROC-AUC (higher is better)")
        ax_tr.set_title("Validation metric across fine-tuning epochs")
        ax_tr.grid(alpha=0.3)
        ax_tr.legend(fontsize=9, loc="lower right")

        metric_names = ["ROC-AUC ↑", "log-loss ↓", "accuracy ↑"]
        base_vals = [base_auc, base_ll, base_acc]
        ft_vals = [ft_auc, ft_ll, ft_acc]
        x_pos = np.arange(len(metric_names))
        w = 0.38
        bars_b = ax_bar.bar(x_pos - w / 2, base_vals, w, color="#64748b", label="pretrained")
        bars_f = ax_bar.bar(x_pos + w / 2, ft_vals, w, color="#0f766e", label="fine-tuned")
        for bars, vals in [(bars_b, base_vals), (bars_f, ft_vals)]:
            for rect, v in zip(bars, vals):
                y_anchor = v + (0.02 if v >= 0 else -0.04)
                ax_bar.text(
                    rect.get_x() + rect.get_width() / 2,
                    y_anchor,
                    f"{v:.3f}",
                    ha="center",
                    va="bottom" if v >= 0 else "top",
                    fontsize=8,
                )
        ax_bar.set_xticks(x_pos)
        ax_bar.set_xticklabels(metric_names)
        ax_bar.set_title("Test-set metrics: pretrained vs. fine-tuned")
        ax_bar.set_ylabel("metric value")
        ax_bar.axhline(0, color="black", lw=0.5)
        ax_bar.grid(alpha=0.25, axis="y")
        ax_bar.legend(fontsize=9, loc="upper right")

        fig2.suptitle("Training dynamics & test-set gains", fontsize=13)
        plt.show()




.. image-sg:: /tutorials/images/sphx_glr_finetune_classifier_002.png
   :alt: Training dynamics & test-set gains, Validation metric across fine-tuning epochs, Test-set metrics: pretrained vs. fine-tuned
   :srcset: /tutorials/images/sphx_glr_finetune_classifier_002.png
   :class: sphx-glr-single-img






.. rst-class:: sphx-glr-timing

   **Total running time of the script:** (0 minutes 25.350 seconds)


.. _sphx_glr_download_tutorials_finetune_classifier.py:

.. only:: html

  .. container:: sphx-glr-footer sphx-glr-footer-example

    .. container:: sphx-glr-download sphx-glr-download-jupyter

      :download:`Download Jupyter notebook: finetune_classifier.ipynb <finetune_classifier.ipynb>`

    .. container:: sphx-glr-download sphx-glr-download-python

      :download:`Download Python source code: finetune_classifier.py <finetune_classifier.py>`

    .. container:: sphx-glr-download sphx-glr-download-zip

      :download:`Download zipped: finetune_classifier.zip <finetune_classifier.zip>`


.. only:: html

 .. rst-class:: sphx-glr-signature

    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.github.io>`_