List the judgment's calibration fits

curl --request GET \
  --url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "mode": "auto",
  "fits": [
    {
      "run": "<string>",
      "fitted_at": "2023-11-07T05:31:56Z",
      "cause": "nightly",
      "version": 2,
      "engine": "<string>",
      "engine_version": "<string>",
      "fit": "<string>",
      "applied": true,
      "stage": "early",
      "outcomes": 1,
      "pinned": true,
      "current": true,
      "label": "<string>",
      "filter": {
        "sources": [
          "posted"
        ],
        "since": "2023-11-07T05:31:56Z",
        "until": "2023-11-07T05:31:56Z"
      },
      "pinned_from": "<string>",
      "not_fitted": {
        "reason": "too_few_outcomes",
        "message": "<string>",
        "gain_standard_errors": 123
      },
      "outcomes_by_source": {
        "posted": 1,
        "rule": 1,
        "queue": 1,
        "implicit": 1
      },
      "fitted_on": "all",
      "source": "outcomes",
      "held_out": {
        "raw_log_loss": 1,
        "calibrated_log_loss": 1,
        "raw_ece": 1,
        "calibrated_ece": 1,
        "standard_error": 1,
        "raw_brier": 0.5,
        "calibrated_brier": 0.5,
        "holdout": "five_groups",
        "gain_standard_errors": 123,
        "raw_accuracy": 0.5,
        "calibrated_accuracy": 0.5,
        "raw_level_distance": 1,
        "calibrated_level_distance": 1,
        "ranking": {
          "base_rate": 0.5,
          "raw": {
            "auroc": 0.5,
            "precision_at_5": 0.5,
            "precision_at_10": 0.5
          },
          "calibrated": {
            "auroc": 0.5,
            "precision_at_5": 0.5,
            "precision_at_10": 0.5
          }
        }
      },
      "features_only": {
        "raw_log_loss": 1,
        "calibrated_log_loss": 1,
        "raw_ece": 1,
        "calibrated_ece": 1,
        "standard_error": 1,
        "raw_brier": 0.5,
        "calibrated_brier": 0.5,
        "holdout": "five_groups",
        "gain_standard_errors": 123,
        "raw_accuracy": 0.5,
        "calibrated_accuracy": 0.5,
        "raw_level_distance": 1,
        "calibrated_level_distance": 1
      },
      "role": "current",
      "promotion": "first",
      "held": "too_few_outcomes_since",
      "versus_current": {
        "outcomes": 1,
        "gain": 123,
        "standard_error": 1,
        "shift": 1,
        "moved_picks": 0.5
      }
    }
  ],
  "next_cursor": "<string>"
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

Outcomes and calibration

List the judgment's calibration fits

Every fit of every calibration run, newest run first, one row per version and engine epoch. A run is one execution of calibration: the nightly run, Fit now, a pin, or a named run. Every run is kept for as long as the namespace exists, so any fit an answer was ever read with can be listed, inspected and pinned. A fit’s fit id is the same in every run that holds the same map or model, so a refit that changes nothing keeps its id. current marks the fits answers read now.

held_out and features_only are as the fit recorded them when it was made, on the outcomes it had then: two runs’ numbers rest on different outcomes, so compare two fits with GET .../calibration/fits/{fit}/compare. About 100 rows a page, never splitting a run; next_cursor continues. On a namespace that inherits the judgment, the template’s fits, which are what its answers read. Not metered.

GET

namespaces

{ns}

judgments

{name}

calibration

fits

List the judgment's calibration fits

curl --request GET \
  --url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/calibration/fits")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "mode": "auto",
  "fits": [
    {
      "run": "<string>",
      "fitted_at": "2023-11-07T05:31:56Z",
      "cause": "nightly",
      "version": 2,
      "engine": "<string>",
      "engine_version": "<string>",
      "fit": "<string>",
      "applied": true,
      "stage": "early",
      "outcomes": 1,
      "pinned": true,
      "current": true,
      "label": "<string>",
      "filter": {
        "sources": [
          "posted"
        ],
        "since": "2023-11-07T05:31:56Z",
        "until": "2023-11-07T05:31:56Z"
      },
      "pinned_from": "<string>",
      "not_fitted": {
        "reason": "too_few_outcomes",
        "message": "<string>",
        "gain_standard_errors": 123
      },
      "outcomes_by_source": {
        "posted": 1,
        "rule": 1,
        "queue": 1,
        "implicit": 1
      },
      "fitted_on": "all",
      "source": "outcomes",
      "held_out": {
        "raw_log_loss": 1,
        "calibrated_log_loss": 1,
        "raw_ece": 1,
        "calibrated_ece": 1,
        "standard_error": 1,
        "raw_brier": 0.5,
        "calibrated_brier": 0.5,
        "holdout": "five_groups",
        "gain_standard_errors": 123,
        "raw_accuracy": 0.5,
        "calibrated_accuracy": 0.5,
        "raw_level_distance": 1,
        "calibrated_level_distance": 1,
        "ranking": {
          "base_rate": 0.5,
          "raw": {
            "auroc": 0.5,
            "precision_at_5": 0.5,
            "precision_at_10": 0.5
          },
          "calibrated": {
            "auroc": 0.5,
            "precision_at_5": 0.5,
            "precision_at_10": 0.5
          }
        }
      },
      "features_only": {
        "raw_log_loss": 1,
        "calibrated_log_loss": 1,
        "raw_ece": 1,
        "calibrated_ece": 1,
        "standard_error": 1,
        "raw_brier": 0.5,
        "calibrated_brier": 0.5,
        "holdout": "five_groups",
        "gain_standard_errors": 123,
        "raw_accuracy": 0.5,
        "calibrated_accuracy": 0.5,
        "raw_level_distance": 1,
        "calibrated_level_distance": 1
      },
      "role": "current",
      "promotion": "first",
      "held": "too_few_outcomes_since",
      "versus_current": {
        "outcomes": 1,
        "gain": 123,
        "standard_error": 1,
        "shift": 1,
        "moved_picks": 0.5
      }
    }
  ],
  "next_cursor": "<string>"
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

Authorizations

Authorization

string

header

required

An organization API key. Keys carry a role (read_write or read_only) and may be restricted to a namespace prefix such as acme/*, or to one namespace such as acme/prod/tenant_1. A prefix matches on a / boundary: acme/prod/tenant_1* covers acme/prod/tenant_1 and everything under acme/prod/tenant_1/, never acme/prod/tenant_12.

Path Parameters

string

required

A namespace name, or a template prefix ending in /*, with any / sent as %2F: acme%2Fprod%2Ftenant_123 or acme%2Fprod%2F*.

A namespace name, or a template prefix: a namespace path ending in /*, such as acme/prod/*, which every namespace under acme/prod/ inherits judgments from. Up to 256 bytes. Never exactly . or .., which a URL path can't carry.

Required string length: 1 - 256

Pattern: ^(?!\.\.?$)[A-Za-z0-9._:/-]+(/\*)?$

name

string

required

A judgment, attribute or threshold name. Names are path segments in field references, so they never contain ..

Required string length: 1 - 128

Pattern: ^[A-Za-z0-9_-]+$

Query Parameters

version

integer<int32>

Required range: x >= 1

engine_version

string

One engine epoch.

since

string<date-time>

Only runs made at or after this time. RFC 3339, UTC.

cause

enum<string>

Only runs of this kind, such as named. Why a run ran. nightly: the scheduled run, or one brought forward (after outcomes first arrive, an engine drift or switch, or a lifted pin). fit_now: POST .../calibration/fit. pin: a pin, which writes a run with the pinned fit. named: a named run. shadow: a composite's combiner fitted by its confirmed shadow job.

Available options:

nightly,

fit_now,

pin,

named,

shadow

cursor

string

The next_cursor of the page before.

Response

A page of fits.

mode

enum<string>

required

auto: answers read the newest run's fit for their version and epoch. pinned: at least one version and epoch reads a pinned fit.

Available options:

auto,

pinned

fits

object[]

required

Show child attributes

next_cursor

string | null

required

Pass as cursor for the next page; null on the last.

Namespaces

Documents

Judgments

Outcomes and calibration

Groups

Subscriptions

Jobs

Engines

Organization

Webhooks

List the judgment's calibration fits

Authorizations

Path Parameters

Query Parameters

Response