Compare two calibration fits on the same outcomes
curl --request GET \
--url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"version": 2,
"engine": "<string>",
"engine_version": "<string>",
"outcomes": 1,
"fitted_on": "all",
"a": {
"fit": "<string>",
"run": "<string>",
"fitted_at": "2023-11-07T05:31:56Z",
"held_out": {
"raw_log_loss": 1,
"calibrated_log_loss": 1,
"raw_ece": 1,
"calibrated_ece": 1,
"standard_error": 1,
"raw_brier": 0.5,
"calibrated_brier": 0.5,
"holdout": "five_groups",
"gain_standard_errors": 123,
"raw_accuracy": 0.5,
"calibrated_accuracy": 0.5,
"raw_level_distance": 1,
"calibrated_level_distance": 1,
"ranking": {
"base_rate": 0.5,
"raw": {
"auroc": 0.5,
"precision_at_5": 0.5,
"precision_at_10": 0.5
},
"calibrated": {
"auroc": 0.5,
"precision_at_5": 0.5,
"precision_at_10": 0.5
}
}
},
"reliability": [
{
"lower": 0.5,
"upper": 0.5,
"count": 1,
"mean_predicted": 0.5,
"observed": 0.5
}
]
},
"b": {
"fit": "<string>",
"run": "<string>",
"fitted_at": "2023-11-07T05:31:56Z",
"held_out": {
"raw_log_loss": 1,
"calibrated_log_loss": 1,
"raw_ece": 1,
"calibrated_ece": 1,
"standard_error": 1,
"raw_brier": 0.5,
"calibrated_brier": 0.5,
"holdout": "five_groups",
"gain_standard_errors": 123,
"raw_accuracy": 0.5,
"calibrated_accuracy": 0.5,
"raw_level_distance": 1,
"calibrated_level_distance": 1,
"ranking": {
"base_rate": 0.5,
"raw": {
"auroc": 0.5,
"precision_at_5": 0.5,
"precision_at_10": 0.5
},
"calibrated": {
"auroc": 0.5,
"precision_at_5": 0.5,
"precision_at_10": 0.5
}
}
},
"reliability": [
{
"lower": 0.5,
"upper": 0.5,
"count": 1,
"mean_predicted": 0.5,
"observed": 0.5
}
]
},
"difference": {
"log_loss": 123,
"standard_error": 1
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}Outcomes and calibration
Compare two calibration fits on the same outcomes
Scores two fits of one version and engine epoch on the same
outcomes: the epoch’s outcomes now, as the next run would fit them.
Each fit’s recorded held_out rests on the outcomes it had when it
was made, so two of them do not compare; this does. Each fit is read
as answers read it, beside the raw answers, with its reliability
curve, and difference is the paired difference in log loss (a
minus b: positive favours b) with its standard error. The fits’
stored maps are scored, never refitted, so an outcome either fit was
made on is scored too.
Two fits of different versions or epochs are invalid_request. One
compare per judgment a minute: within it, rate_limited with
Retry-After and the same seconds in details.retry_after_s. When reading the outcomes takes too long, as for the
report, unavailable with Retry-After: the retry gets the answer.
Not metered.
GET
/
namespaces
/
{ns}
/
judgments
/
{name}
/
calibration
/
fits
/
{fit}
/
compare
Compare two calibration fits on the same outcomes
curl --request GET \
--url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits/{fit}/compare")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"version": 2,
"engine": "<string>",
"engine_version": "<string>",
"outcomes": 1,
"fitted_on": "all",
"a": {
"fit": "<string>",
"run": "<string>",
"fitted_at": "2023-11-07T05:31:56Z",
"held_out": {
"raw_log_loss": 1,
"calibrated_log_loss": 1,
"raw_ece": 1,
"calibrated_ece": 1,
"standard_error": 1,
"raw_brier": 0.5,
"calibrated_brier": 0.5,
"holdout": "five_groups",
"gain_standard_errors": 123,
"raw_accuracy": 0.5,
"calibrated_accuracy": 0.5,
"raw_level_distance": 1,
"calibrated_level_distance": 1,
"ranking": {
"base_rate": 0.5,
"raw": {
"auroc": 0.5,
"precision_at_5": 0.5,
"precision_at_10": 0.5
},
"calibrated": {
"auroc": 0.5,
"precision_at_5": 0.5,
"precision_at_10": 0.5
}
}
},
"reliability": [
{
"lower": 0.5,
"upper": 0.5,
"count": 1,
"mean_predicted": 0.5,
"observed": 0.5
}
]
},
"b": {
"fit": "<string>",
"run": "<string>",
"fitted_at": "2023-11-07T05:31:56Z",
"held_out": {
"raw_log_loss": 1,
"calibrated_log_loss": 1,
"raw_ece": 1,
"calibrated_ece": 1,
"standard_error": 1,
"raw_brier": 0.5,
"calibrated_brier": 0.5,
"holdout": "five_groups",
"gain_standard_errors": 123,
"raw_accuracy": 0.5,
"calibrated_accuracy": 0.5,
"raw_level_distance": 1,
"calibrated_level_distance": 1,
"ranking": {
"base_rate": 0.5,
"raw": {
"auroc": 0.5,
"precision_at_5": 0.5,
"precision_at_10": 0.5
},
"calibrated": {
"auroc": 0.5,
"precision_at_5": 0.5,
"precision_at_10": 0.5
}
}
},
"reliability": [
{
"lower": 0.5,
"upper": 0.5,
"count": 1,
"mean_predicted": 0.5,
"observed": 0.5
}
]
},
"difference": {
"log_loss": 123,
"standard_error": 1
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}{
"error": {
"code": "invalid_request",
"message": "<string>",
"details": {
"reason": "<string>",
"suggested_context": {
"recipe": {
"fields": [
"<string>"
],
"last_n": {},
"window": {},
"max_tokens": 2,
"related": {},
"previous": {
"fields": [
"<string>"
],
"anchor": "<string>"
}
},
"cost_per_answer": 1,
"whole_state_cost_per_answer": 1,
"sampled_documents": 2,
"excluded": [
{
"path": "<string>",
"reason": "id"
}
]
}
}
}
}Authorizations
An organization API key. Keys carry a role (read_write or
read_only) and may be restricted to a namespace prefix such as
acme/*, or to one namespace such as acme/prod/tenant_1. A prefix
matches on a / boundary: acme/prod/tenant_1* covers
acme/prod/tenant_1 and everything under acme/prod/tenant_1/,
never acme/prod/tenant_12.
Path Parameters
A namespace name, or a template prefix ending in /*, with any
/ sent as %2F: acme%2Fprod%2Ftenant_123 or acme%2Fprod%2F*.
A namespace name, or a template prefix: a namespace path ending in
/*, such as acme/prod/*, which every namespace under acme/prod/
inherits judgments from. Up to 256 bytes. Never exactly
. or .., which a URL path can't carry.
Required string length:
1 - 256Pattern:
^(?!\.\.?$)[A-Za-z0-9._:/-]+(/\*)?$A judgment, attribute or threshold name. Names are path segments in field references, so they never contain ..
Required string length:
1 - 128Pattern:
^[A-Za-z0-9_-]+$A fit's id, as answers and GET .../calibration/fits show it.
Pattern:
^fit_[0-9a-f]{16}$Query Parameters
The other fit's id.
Pattern:
^fit_[0-9a-f]{16}$Response
Both fits on the same outcomes.
Two fits of one version and epoch, scored on the same outcomes.
Required range:
x >= 1The outcomes both were scored on.
Required range:
x >= 0queue when the epoch's labelling-queue labels alone meet the minimums: both are scored on them, weighted back to real traffic.
Available options:
all, queue Show child attributes
Show child attributes
Show child attributes
Show child attributes
The mean per-outcome log loss of a minus b (positive favours b), with its standard error. Null with fewer than two outcomes.
Show child attributes
Show child attributes