Get a calibration fit with its map

curl --request GET \
  --url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit} \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "run": "<string>",
  "fitted_at": "2023-11-07T05:31:56Z",
  "cause": "nightly",
  "version": 2,
  "engine": "<string>",
  "engine_version": "<string>",
  "fit": "<string>",
  "applied": true,
  "stage": "early",
  "outcomes": 1,
  "pinned": true,
  "current": true,
  "model": {
    "form": "log_odds_linear",
    "a": 123,
    "b": 123
  },
  "range": {
    "lower": 0.5,
    "upper": 0.5
  },
  "label": "<string>",
  "filter": {
    "sources": [
      "posted"
    ],
    "since": "2023-11-07T05:31:56Z",
    "until": "2023-11-07T05:31:56Z"
  },
  "pinned_from": "<string>",
  "not_fitted": {
    "reason": "too_few_outcomes",
    "message": "<string>",
    "gain_standard_errors": 123
  },
  "outcomes_by_source": {
    "posted": 1,
    "rule": 1,
    "queue": 1,
    "implicit": 1
  },
  "fitted_on": "all",
  "source": "outcomes",
  "held_out": {
    "raw_log_loss": 1,
    "calibrated_log_loss": 1,
    "raw_ece": 1,
    "calibrated_ece": 1,
    "standard_error": 1,
    "raw_brier": 0.5,
    "calibrated_brier": 0.5,
    "holdout": "five_groups",
    "gain_standard_errors": 123,
    "raw_accuracy": 0.5,
    "calibrated_accuracy": 0.5,
    "raw_level_distance": 1,
    "calibrated_level_distance": 1,
    "ranking": {
      "base_rate": 0.5,
      "raw": {
        "auroc": 0.5,
        "precision_at_5": 0.5,
        "precision_at_10": 0.5
      },
      "calibrated": {
        "auroc": 0.5,
        "precision_at_5": 0.5,
        "precision_at_10": 0.5
      }
    }
  },
  "features_only": {
    "raw_log_loss": 1,
    "calibrated_log_loss": 1,
    "raw_ece": 1,
    "calibrated_ece": 1,
    "standard_error": 1,
    "raw_brier": 0.5,
    "calibrated_brier": 0.5,
    "holdout": "five_groups",
    "gain_standard_errors": 123,
    "raw_accuracy": 0.5,
    "calibrated_accuracy": 0.5,
    "raw_level_distance": 1,
    "calibrated_level_distance": 1
  },
  "role": "current",
  "promotion": "first",
  "held": "too_few_outcomes_since",
  "versus_current": {
    "outcomes": 1,
    "gain": 123,
    "standard_error": 1,
    "shift": 1,
    "moved_picks": 0.5
  },
  "features": {
    "engine_weight": 123,
    "per_feature": [
      {
        "name": "<string>",
        "weight": 123,
        "center": 123,
        "scale": 123,
        "missing": 0.5,
        "without": {
          "log_loss": 1
        },
        "alone": {
          "log_loss": 1
        },
        "warning": "reads_like_the_outcome",
        "by_option": {}
      }
    ],
    "penalty": 123
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

GET

namespaces

{ns}

judgments

{name}

calibration

fits

{fit}

Get a calibration fit with its map

curl --request GET \
  --url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit} \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "run": "<string>",
  "fitted_at": "2023-11-07T05:31:56Z",
  "cause": "nightly",
  "version": 2,
  "engine": "<string>",
  "engine_version": "<string>",
  "fit": "<string>",
  "applied": true,
  "stage": "early",
  "outcomes": 1,
  "pinned": true,
  "current": true,
  "model": {
    "form": "log_odds_linear",
    "a": 123,
    "b": 123
  },
  "range": {
    "lower": 0.5,
    "upper": 0.5
  },
  "label": "<string>",
  "filter": {
    "sources": [
      "posted"
    ],
    "since": "2023-11-07T05:31:56Z",
    "until": "2023-11-07T05:31:56Z"
  },
  "pinned_from": "<string>",
  "not_fitted": {
    "reason": "too_few_outcomes",
    "message": "<string>",
    "gain_standard_errors": 123
  },
  "outcomes_by_source": {
    "posted": 1,
    "rule": 1,
    "queue": 1,
    "implicit": 1
  },
  "fitted_on": "all",
  "source": "outcomes",
  "held_out": {
    "raw_log_loss": 1,
    "calibrated_log_loss": 1,
    "raw_ece": 1,
    "calibrated_ece": 1,
    "standard_error": 1,
    "raw_brier": 0.5,
    "calibrated_brier": 0.5,
    "holdout": "five_groups",
    "gain_standard_errors": 123,
    "raw_accuracy": 0.5,
    "calibrated_accuracy": 0.5,
    "raw_level_distance": 1,
    "calibrated_level_distance": 1,
    "ranking": {
      "base_rate": 0.5,
      "raw": {
        "auroc": 0.5,
        "precision_at_5": 0.5,
        "precision_at_10": 0.5
      },
      "calibrated": {
        "auroc": 0.5,
        "precision_at_5": 0.5,
        "precision_at_10": 0.5
      }
    }
  },
  "features_only": {
    "raw_log_loss": 1,
    "calibrated_log_loss": 1,
    "raw_ece": 1,
    "calibrated_ece": 1,
    "standard_error": 1,
    "raw_brier": 0.5,
    "calibrated_brier": 0.5,
    "holdout": "five_groups",
    "gain_standard_errors": 123,
    "raw_accuracy": 0.5,
    "calibrated_accuracy": 0.5,
    "raw_level_distance": 1,
    "calibrated_level_distance": 1
  },
  "role": "current",
  "promotion": "first",
  "held": "too_few_outcomes_since",
  "versus_current": {
    "outcomes": 1,
    "gain": 123,
    "standard_error": 1,
    "shift": 1,
    "moved_picks": 0.5
  },
  "features": {
    "engine_weight": 123,
    "per_feature": [
      {
        "name": "<string>",
        "weight": 123,
        "center": 123,
        "scale": 123,
        "missing": 0.5,
        "without": {
          "log_loss": 1
        },
        "alone": {
          "log_loss": 1
        },
        "warning": "reads_like_the_outcome",
        "by_option": {}
      }
    ],
    "penalty": 123
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

Authorizations

Authorization

string

header

required

An organization API key. Keys carry a role (read_write or read_only) and may be restricted to a namespace prefix such as acme/*, or to one namespace such as acme/prod/tenant_1. A prefix matches on a / boundary: acme/prod/tenant_1* covers acme/prod/tenant_1 and everything under acme/prod/tenant_1/, never acme/prod/tenant_12.

Path Parameters

string

required

A namespace name, or a template prefix ending in /*, with any / sent as %2F: acme%2Fprod%2Ftenant_123 or acme%2Fprod%2F*.

A namespace name, or a template prefix: a namespace path ending in /*, such as acme/prod/*, which every namespace under acme/prod/ inherits judgments from. Up to 256 bytes. Never exactly . or .., which a URL path can't carry.

Required string length: 1 - 256

Pattern: ^(?!\.\.?$)[A-Za-z0-9._:/-]+(/\*)?$

name

string

required

A judgment, attribute or threshold name. Names are path segments in field references, so they never contain ..

Required string length: 1 - 128

Pattern: ^[A-Za-z0-9_-]+$

fit

string

required

A fit's id, as answers and GET .../calibration/fits show it.

Pattern: ^fit_[0-9a-f]{16}$

Response

The fit.

One fit of one run, for one version and engine epoch.

run

string

required

The run, newest first; ids sort by when the run was made.

fitted_at

string<date-time>

required

RFC 3339, UTC.

cause

enum<string>

required

Why a run ran. nightly: the scheduled run, or one brought forward (after outcomes first arrive, an engine drift or switch, or a lifted pin). fit_now: POST .../calibration/fit. pin: a pin, which writes a run with the pinned fit. named: a named run. shadow: a composite's combiner fitted by its confirmed shadow job.

Available options:

nightly,

fit_now,

pin,

named,

shadow

version

integer<int32>

required

Required range: x >= 1

engine

string

required

engine_version

string

required

fit

string | null

required

The fit's id. Null when the fit did not beat the raw answers on held-out outcomes, or a named run's outcomes missed the minimums (not_fitted).

applied

boolean

required

Whether the fit beat the raw answers on held-out outcomes when it was made, so answers of its run read it.

stage

enum<string>

required

early and full as calibrated.stage means them; features for a judgment whose p reads its features and the engine's p together.

Available options:

early,

full,

features

outcomes

integer<int64>

required

Required range: x >= 0

pinned

boolean

required

True when the run holds this fit because it is pinned.

current

boolean

required

True on the fits answers of this version and epoch read now.

model

object

required

How the fit turns the engine's raw value into the calibrated one, given as the formula's terms. Write logit(p) = ln(p / (1 - p)), with p first clamped to [0.005, 0.995], and σ(z) = 1 / (1 + e^-z).

log_odds_linear, a bool: calibrated p = σ(a · logit(p) + b).
temperature: a choice or score's calibrated distribution is softmax(ln(p_i) / t) over its options or levels (each p_i first clamped as above); on a bool, σ(logit(p) / t).
points: linear interpolation of the raw value through the points (x, y), flat beyond the ends; a choice or score maps each option's or level's probability and divides by their sum.
features, a bool with features: σ(intercept + Σ wᵢ · (xᵢ - cᵢ) / sᵢ) over inputs, where the first input, named log_odds, is logit(p) and the rest are the features' raw values; a missing value contributes 0.
features_candidates, a choice among candidates: existence is the features form on 1 - escape_p, the chance a match exists; among the candidates with any probability, candidate j's share of it is softmax(log_probability_weight · ln(p_j) + Σ wᵢ · (x_jᵢ - cᵢ) / sᵢ) over its inputs; escape_p is 1 - the existence chance.
features_options, a fixed choice: softmax over the options of log_probability_weight · ln(p_k) + intercept_against_escape + Σ wₖᵢ · (xᵢ - cᵢ) / sᵢ, weights per option in options' order, and over none_of_the_above of log_probability_weight · ln(p_escape).
features_levels, a score: the chance of a level at or below level j is σ(cuts_j + cumulative_log_odds_weight · logit(C_j) - Σ wᵢ · (xᵢ - cᵢ) / sᵢ), C_j the engine's chance of a level at or below j; each level's chance is the difference. Null when the fit did not beat the raw answers.

Show child attributes

range

object | null

required

The lowest and highest raw value the fit was made on; outside it, a calibrated number is extrapolated.

Show child attributes

label

string

A named run's label.

filter

object

The outcomes a named run is fitted on. An outcome left out still counts as an answer's outcome, so it never makes that answer an implicit negative.

Show child attributes

pinned_from

string

For a pinned fit, the run it was pinned from.

not_fitted

object

Why there is no calibration or no threshold recommendation. message says what to post, such as "Only positive outcomes so far: post outcomes for documents where it did not happen." when every outcome is true.

Show child attributes

outcomes_by_source

object

Joined outcomes by where they came from, summing to outcomes.

Show child attributes

fitted_on

enum<string>

Available options:

all,

queue

source

enum<string>

outcomes: the epoch's own outcomes. replay: after an engine drift, until the new epoch's own outcomes meet the minimums, the fit also rests on outcomes of earlier epochs, each with the new engine's answer to the context the outcome labelled; after a version was activated on another engine, on the outcomes of the version it replaced, the same way.

Available options:

outcomes,

replay

held_out

object | null

The fit's held-out test as it was made, with its ranking on a bool.

Show child attributes

features_only

object | null

For a judgment with features, the same test of a fit on the features alone, without the engine's answer. Null elsewhere.

Show child attributes

role

enum<string>

current: the fit the run kept or made current. challenger: the run's new fit, kept beside it, listed directly under it. Absent on runs made before calibration kept the current fit.

Available options:

current,

challenger

promotion

enum<string> | null

On a current row, what the run did with the version and epoch's fit. Null on a challenger row.

Available options:

first,

promoted,

withdrawn,

held,

pinned

held

enum<string> | null

Why the current fit was kept, when it was.

Available options:

too_few_outcomes_since,

not_clearly_better,

too_small_a_move,

pinned,

too_few_outcomes

versus_current

object | null

The run's new fit against the current one, on the outcomes since the current fit's run.

Show child attributes

features

object | null

On a judgment with features, the fit that reads them beside the engine's answer, whether answers read it or not: engine_weight and each feature read on its own, as the report's held_out_metrics.features gives them, and penalty, how strongly that fit pulls every weight toward 0, recorded so a change of it shows per fit. Null on any other fit.

Show child attributes

Namespaces

Documents

Judgments

Outcomes and calibration

Groups

Subscriptions

Jobs

Engines

Organization

Webhooks

Get a calibration fit with its map

Authorizations

Path Parameters

Response