Score the alternatives on your outcomes

curl --request POST \
  --url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "confirm": true
}
'

import requests

url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run"

payload = { "confirm": True }
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({confirm: true})
};

fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'confirm' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run"

	payload := strings.NewReader("{\n  \"confirm\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"confirm\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"confirm\": true\n}"

response = http.request(request)
puts response.read_body

{
  "version": 2,
  "engine_version": "<string>",
  "status": "not_run",
  "reason": "awaiting_confirm",
  "message": "<string>",
  "automatic": true,
  "skip_engines": [
    {
      "name": "jev",
      "version": "<string>"
    }
  ],
  "labels": {
    "count": 1,
    "sampled": true,
    "outcomes": {
      "sources": [
        "posted"
      ],
      "since": "2023-11-07T05:31:56Z",
      "until": "2023-11-07T05:31:56Z"
    }
  },
  "corrected_for": 1,
  "quote": {
    "status": "ready",
    "labels": 1,
    "alternatives": [
      {
        "id": "alt_3f9a2c7e1b04d6a8",
        "contexts": "stored",
        "labels": 1,
        "tokens": 1,
        "judgment_units": 1,
        "cost_usd": 1,
        "new_provider": true
      }
    ],
    "judgment_units": 1,
    "tokens": 1,
    "cost_usd": 1,
    "detectable": {
      "log_loss": 1,
      "auroc": 1,
      "k": 2
    },
    "new_providers": [
      "<string>"
    ],
    "upstream": [
      {
        "alternative": "alt_3f9a2c7e1b04d6a8",
        "judgment": "<string>",
        "documents": 1,
        "judgment_units": 1,
        "cost_usd": 1
      }
    ],
    "generation": 1,
    "computed_at": "2023-11-07T05:31:56Z",
    "reason": "<string>"
  },
  "current": {
    "held_out": {
      "read_with": "raw",
      "log_loss": 123,
      "ece": 123,
      "brier": 123,
      "auroc": 123,
      "precision_at_5": 123,
      "precision_at_10": 123,
      "accuracy": 123,
      "features_only": 123
    },
    "cost_per_answer": 123
  },
  "alternatives": [
    {
      "id": "alt_3f9a2c7e1b04d6a8",
      "source": "automatic",
      "drafted": true,
      "named_by": "<string>",
      "named_at": "2023-11-07T05:31:56Z",
      "differs": [
        "<string>"
      ],
      "definition": {
        "name": "<string>",
        "question": "<string>",
        "type": "bool",
        "criteria": "<string>",
        "context": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "engine": {
          "name": "jev",
          "version": "<string>"
        },
        "horizon": "0s",
        "applies_to": {},
        "thresholds": {},
        "parts": [
          {
            "name": "<string>",
            "question": "<string>"
          }
        ],
        "features": [
          "<string>"
        ]
      },
      "in_k": true,
      "contexts": "stored",
      "as_of": "now",
      "labels": 1,
      "labels_skipped": {
        "deleted": 1,
        "image_erased": 1,
        "not_derivable": 1,
        "failed": 1
      },
      "cost_per_answer": 123,
      "unit_reduction": 123,
      "held_out": {
        "read_with": "raw",
        "log_loss": 123,
        "ece": 123,
        "brier": 123,
        "auroc": 123,
        "precision_at_5": 123,
        "precision_at_10": 123,
        "accuracy": 123,
        "features_only": 123
      },
      "change": {
        "log_loss": {
          "mean": 123,
          "interval": {
            "lower": 123,
            "upper": 123
          },
          "standard_error": 123,
          "p": 123
        },
        "auroc": {
          "mean": 123,
          "interval": {
            "lower": 123,
            "upper": 123
          },
          "standard_error": 123,
          "p": 123
        },
        "accuracy": 123
      },
      "verdict": "better",
      "moves_ranking": true,
      "scored_at": "2023-11-07T05:31:56Z",
      "billed": {
        "judgment_units": 1,
        "cost_usd": 1
      }
    }
  ],
  "suggestion": {
    "id": "alt_3f9a2c7e1b04d6a8",
    "because": "better"
  },
  "headline": "<string>",
  "computed_at": "2023-11-07T05:31:56Z",
  "next_run_after": "2023-11-07T05:31:56Z",
  "plan_required": {
    "message": "<string>",
    "capability": "alternatives",
    "required_plan": "team",
    "plan": "developer"
  }
}

{
  "version": 2,
  "engine_version": "<string>",
  "status": "not_run",
  "reason": "awaiting_confirm",
  "message": "<string>",
  "automatic": true,
  "skip_engines": [
    {
      "name": "jev",
      "version": "<string>"
    }
  ],
  "labels": {
    "count": 1,
    "sampled": true,
    "outcomes": {
      "sources": [
        "posted"
      ],
      "since": "2023-11-07T05:31:56Z",
      "until": "2023-11-07T05:31:56Z"
    }
  },
  "corrected_for": 1,
  "quote": {
    "status": "ready",
    "labels": 1,
    "alternatives": [
      {
        "id": "alt_3f9a2c7e1b04d6a8",
        "contexts": "stored",
        "labels": 1,
        "tokens": 1,
        "judgment_units": 1,
        "cost_usd": 1,
        "new_provider": true
      }
    ],
    "judgment_units": 1,
    "tokens": 1,
    "cost_usd": 1,
    "detectable": {
      "log_loss": 1,
      "auroc": 1,
      "k": 2
    },
    "new_providers": [
      "<string>"
    ],
    "upstream": [
      {
        "alternative": "alt_3f9a2c7e1b04d6a8",
        "judgment": "<string>",
        "documents": 1,
        "judgment_units": 1,
        "cost_usd": 1
      }
    ],
    "generation": 1,
    "computed_at": "2023-11-07T05:31:56Z",
    "reason": "<string>"
  },
  "current": {
    "held_out": {
      "read_with": "raw",
      "log_loss": 123,
      "ece": 123,
      "brier": 123,
      "auroc": 123,
      "precision_at_5": 123,
      "precision_at_10": 123,
      "accuracy": 123,
      "features_only": 123
    },
    "cost_per_answer": 123
  },
  "alternatives": [
    {
      "id": "alt_3f9a2c7e1b04d6a8",
      "source": "automatic",
      "drafted": true,
      "named_by": "<string>",
      "named_at": "2023-11-07T05:31:56Z",
      "differs": [
        "<string>"
      ],
      "definition": {
        "name": "<string>",
        "question": "<string>",
        "type": "bool",
        "criteria": "<string>",
        "context": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "engine": {
          "name": "jev",
          "version": "<string>"
        },
        "horizon": "0s",
        "applies_to": {},
        "thresholds": {},
        "parts": [
          {
            "name": "<string>",
            "question": "<string>"
          }
        ],
        "features": [
          "<string>"
        ]
      },
      "in_k": true,
      "contexts": "stored",
      "as_of": "now",
      "labels": 1,
      "labels_skipped": {
        "deleted": 1,
        "image_erased": 1,
        "not_derivable": 1,
        "failed": 1
      },
      "cost_per_answer": 123,
      "unit_reduction": 123,
      "held_out": {
        "read_with": "raw",
        "log_loss": 123,
        "ece": 123,
        "brier": 123,
        "auroc": 123,
        "precision_at_5": 123,
        "precision_at_10": 123,
        "accuracy": 123,
        "features_only": 123
      },
      "change": {
        "log_loss": {
          "mean": 123,
          "interval": {
            "lower": 123,
            "upper": 123
          },
          "standard_error": 123,
          "p": 123
        },
        "auroc": {
          "mean": 123,
          "interval": {
            "lower": 123,
            "upper": 123
          },
          "standard_error": 123,
          "p": 123
        },
        "accuracy": 123
      },
      "verdict": "better",
      "moves_ranking": true,
      "scored_at": "2023-11-07T05:31:56Z",
      "billed": {
        "judgment_units": 1,
        "cost_usd": 1
      }
    }
  ],
  "suggestion": {
    "id": "alt_3f9a2c7e1b04d6a8",
    "because": "better"
  },
  "headline": "<string>",
  "computed_at": "2023-11-07T05:31:56Z",
  "next_run_after": "2023-11-07T05:31:56Z",
  "plan_required": {
    "message": "<string>",
    "capability": "alternatives",
    "required_plan": "team",
    "plan": "developer"
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

Outcomes and calibration

Score the alternatives on your outcomes

Without confirm, returns the quote of what this month’s run would score and bill, and runs nothing. With confirm: true, starts this month’s run, or extends a finished one with the alternatives named since, scored on the same labelled answers (202). Every answered call is billed as judging at its engine’s weight, held by your budget like any judging; a failed call bills nothing. A confirm is against the current quote: when alternatives or outcomes changed since, it is conflict and the response says to read the quote again.

outcomes scores the alternatives on the outcomes you choose, by source and when they were observed, as a named calibration run chooses them. The month’s first run fixes them; an extension uses the same. Extending a run re-checks every verdict against all the alternatives scored this month, so a verdict can change.

One run a month per version and engine epoch, plus its extensions. On the Team plan and above.

POST

namespaces

{ns}

judgments

{name}

alternatives

run

Score the alternatives on your outcomes

curl --request POST \
  --url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "confirm": true
}
'

import requests

url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run"

payload = { "confirm": True }
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({confirm: true})
};

fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'confirm' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run"

	payload := strings.NewReader("{\n  \"confirm\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"confirm\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/alternatives/run")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"confirm\": true\n}"

response = http.request(request)
puts response.read_body

{
  "version": 2,
  "engine_version": "<string>",
  "status": "not_run",
  "reason": "awaiting_confirm",
  "message": "<string>",
  "automatic": true,
  "skip_engines": [
    {
      "name": "jev",
      "version": "<string>"
    }
  ],
  "labels": {
    "count": 1,
    "sampled": true,
    "outcomes": {
      "sources": [
        "posted"
      ],
      "since": "2023-11-07T05:31:56Z",
      "until": "2023-11-07T05:31:56Z"
    }
  },
  "corrected_for": 1,
  "quote": {
    "status": "ready",
    "labels": 1,
    "alternatives": [
      {
        "id": "alt_3f9a2c7e1b04d6a8",
        "contexts": "stored",
        "labels": 1,
        "tokens": 1,
        "judgment_units": 1,
        "cost_usd": 1,
        "new_provider": true
      }
    ],
    "judgment_units": 1,
    "tokens": 1,
    "cost_usd": 1,
    "detectable": {
      "log_loss": 1,
      "auroc": 1,
      "k": 2
    },
    "new_providers": [
      "<string>"
    ],
    "upstream": [
      {
        "alternative": "alt_3f9a2c7e1b04d6a8",
        "judgment": "<string>",
        "documents": 1,
        "judgment_units": 1,
        "cost_usd": 1
      }
    ],
    "generation": 1,
    "computed_at": "2023-11-07T05:31:56Z",
    "reason": "<string>"
  },
  "current": {
    "held_out": {
      "read_with": "raw",
      "log_loss": 123,
      "ece": 123,
      "brier": 123,
      "auroc": 123,
      "precision_at_5": 123,
      "precision_at_10": 123,
      "accuracy": 123,
      "features_only": 123
    },
    "cost_per_answer": 123
  },
  "alternatives": [
    {
      "id": "alt_3f9a2c7e1b04d6a8",
      "source": "automatic",
      "drafted": true,
      "named_by": "<string>",
      "named_at": "2023-11-07T05:31:56Z",
      "differs": [
        "<string>"
      ],
      "definition": {
        "name": "<string>",
        "question": "<string>",
        "type": "bool",
        "criteria": "<string>",
        "context": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "engine": {
          "name": "jev",
          "version": "<string>"
        },
        "horizon": "0s",
        "applies_to": {},
        "thresholds": {},
        "parts": [
          {
            "name": "<string>",
            "question": "<string>"
          }
        ],
        "features": [
          "<string>"
        ]
      },
      "in_k": true,
      "contexts": "stored",
      "as_of": "now",
      "labels": 1,
      "labels_skipped": {
        "deleted": 1,
        "image_erased": 1,
        "not_derivable": 1,
        "failed": 1
      },
      "cost_per_answer": 123,
      "unit_reduction": 123,
      "held_out": {
        "read_with": "raw",
        "log_loss": 123,
        "ece": 123,
        "brier": 123,
        "auroc": 123,
        "precision_at_5": 123,
        "precision_at_10": 123,
        "accuracy": 123,
        "features_only": 123
      },
      "change": {
        "log_loss": {
          "mean": 123,
          "interval": {
            "lower": 123,
            "upper": 123
          },
          "standard_error": 123,
          "p": 123
        },
        "auroc": {
          "mean": 123,
          "interval": {
            "lower": 123,
            "upper": 123
          },
          "standard_error": 123,
          "p": 123
        },
        "accuracy": 123
      },
      "verdict": "better",
      "moves_ranking": true,
      "scored_at": "2023-11-07T05:31:56Z",
      "billed": {
        "judgment_units": 1,
        "cost_usd": 1
      }
    }
  ],
  "suggestion": {
    "id": "alt_3f9a2c7e1b04d6a8",
    "because": "better"
  },
  "headline": "<string>",
  "computed_at": "2023-11-07T05:31:56Z",
  "next_run_after": "2023-11-07T05:31:56Z",
  "plan_required": {
    "message": "<string>",
    "capability": "alternatives",
    "required_plan": "team",
    "plan": "developer"
  }
}

{
  "version": 2,
  "engine_version": "<string>",
  "status": "not_run",
  "reason": "awaiting_confirm",
  "message": "<string>",
  "automatic": true,
  "skip_engines": [
    {
      "name": "jev",
      "version": "<string>"
    }
  ],
  "labels": {
    "count": 1,
    "sampled": true,
    "outcomes": {
      "sources": [
        "posted"
      ],
      "since": "2023-11-07T05:31:56Z",
      "until": "2023-11-07T05:31:56Z"
    }
  },
  "corrected_for": 1,
  "quote": {
    "status": "ready",
    "labels": 1,
    "alternatives": [
      {
        "id": "alt_3f9a2c7e1b04d6a8",
        "contexts": "stored",
        "labels": 1,
        "tokens": 1,
        "judgment_units": 1,
        "cost_usd": 1,
        "new_provider": true
      }
    ],
    "judgment_units": 1,
    "tokens": 1,
    "cost_usd": 1,
    "detectable": {
      "log_loss": 1,
      "auroc": 1,
      "k": 2
    },
    "new_providers": [
      "<string>"
    ],
    "upstream": [
      {
        "alternative": "alt_3f9a2c7e1b04d6a8",
        "judgment": "<string>",
        "documents": 1,
        "judgment_units": 1,
        "cost_usd": 1
      }
    ],
    "generation": 1,
    "computed_at": "2023-11-07T05:31:56Z",
    "reason": "<string>"
  },
  "current": {
    "held_out": {
      "read_with": "raw",
      "log_loss": 123,
      "ece": 123,
      "brier": 123,
      "auroc": 123,
      "precision_at_5": 123,
      "precision_at_10": 123,
      "accuracy": 123,
      "features_only": 123
    },
    "cost_per_answer": 123
  },
  "alternatives": [
    {
      "id": "alt_3f9a2c7e1b04d6a8",
      "source": "automatic",
      "drafted": true,
      "named_by": "<string>",
      "named_at": "2023-11-07T05:31:56Z",
      "differs": [
        "<string>"
      ],
      "definition": {
        "name": "<string>",
        "question": "<string>",
        "type": "bool",
        "criteria": "<string>",
        "context": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "engine": {
          "name": "jev",
          "version": "<string>"
        },
        "horizon": "0s",
        "applies_to": {},
        "thresholds": {},
        "parts": [
          {
            "name": "<string>",
            "question": "<string>"
          }
        ],
        "features": [
          "<string>"
        ]
      },
      "in_k": true,
      "contexts": "stored",
      "as_of": "now",
      "labels": 1,
      "labels_skipped": {
        "deleted": 1,
        "image_erased": 1,
        "not_derivable": 1,
        "failed": 1
      },
      "cost_per_answer": 123,
      "unit_reduction": 123,
      "held_out": {
        "read_with": "raw",
        "log_loss": 123,
        "ece": 123,
        "brier": 123,
        "auroc": 123,
        "precision_at_5": 123,
        "precision_at_10": 123,
        "accuracy": 123,
        "features_only": 123
      },
      "change": {
        "log_loss": {
          "mean": 123,
          "interval": {
            "lower": 123,
            "upper": 123
          },
          "standard_error": 123,
          "p": 123
        },
        "auroc": {
          "mean": 123,
          "interval": {
            "lower": 123,
            "upper": 123
          },
          "standard_error": 123,
          "p": 123
        },
        "accuracy": 123
      },
      "verdict": "better",
      "moves_ranking": true,
      "scored_at": "2023-11-07T05:31:56Z",
      "billed": {
        "judgment_units": 1,
        "cost_usd": 1
      }
    }
  ],
  "suggestion": {
    "id": "alt_3f9a2c7e1b04d6a8",
    "because": "better"
  },
  "headline": "<string>",
  "computed_at": "2023-11-07T05:31:56Z",
  "next_run_after": "2023-11-07T05:31:56Z",
  "plan_required": {
    "message": "<string>",
    "capability": "alternatives",
    "required_plan": "team",
    "plan": "developer"
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

Authorizations

Authorization

string

header

required

An organization API key. Keys carry a role (read_write or read_only) and may be restricted to a namespace prefix such as acme/*, or to one namespace such as acme/prod/tenant_1. A prefix matches on a / boundary: acme/prod/tenant_1* covers acme/prod/tenant_1 and everything under acme/prod/tenant_1/, never acme/prod/tenant_12.

Headers

Idempotency-Key

string

One key per logical request, reused only on its retries. A key belongs to one request: within your organization, the same method, path, query and body. For 24 hours after a successful response, a request with the key and the same body gets that response back verbatim, with Idempotent-Replayed: true, and runs nothing. Only a successful response is kept, so the retry of a request that failed runs again. While the first request runs or its response is kept, the key with a different request is idempotency_key_reused (422). A request sent while one with its key is still running is rate_limited with Retry-After: 1, without running: retry it to get the first one's response. In the rare case the key can't be checked, the request runs as if it had none.

Required string length: 1 - 255

Path Parameters

string

required

A namespace name, or a template prefix ending in /*, with any / sent as %2F: acme%2Fprod%2Ftenant_123 or acme%2Fprod%2F*.

A namespace name, or a template prefix: a namespace path ending in /*, such as acme/prod/*, which every namespace under acme/prod/ inherits judgments from. Up to 256 bytes. Never exactly . or .., which a URL path can't carry.

Required string length: 1 - 256

Pattern: ^(?!\.\.?$)[A-Za-z0-9._:/-]+(/\*)?$

name

string

required

A judgment, attribute or threshold name. Names are path segments in field references, so they never contain ..

Required string length: 1 - 128

Pattern: ^[A-Za-z0-9_-]+$

Body

application/json

confirm

boolean

Start the run at the current quote. Without it, the quote is returned and nothing runs.

quote_generation

integer

The quote.generation you read. With it, a confirm against any other quote is conflict.

Required range: x >= 0

outcomes

object

The outcomes a named run is fitted on. An outcome left out still counts as an answer's outcome, so it never makes that answer an implicit negative.

Show child attributes

Response

The quote; nothing runs.

A version's alternatives: which engine, recipe or wording its outcomes prefer, what the labels can and cannot tell, and what scoring costs.

version

integer<int32>

required

Required range: x >= 1

engine_version

string | null

required

The engine epoch of the calibration fit whose labelled answers the alternatives are scored on. Null before a fit.

status

enum<string>

required

not_run: nothing scored this month; reason says why.
quoting: the quote is being priced; read again shortly.
running: this month's run is scoring.
ready: suggestion names what the outcomes prefer.
none: the run found nothing better beyond noise, or nothing cheaper; message says what was detectable.

Available options:

not_run,

quoting,

running,

ready,

none

reason

enum<string> | null

required

Not run: awaiting_confirm (there is something to score and automatic is off: confirm the quote), awaiting_run (the next calibration run prices it), too_few_outcomes (no calibration fit yet), composite, inherited (the template has the table), budget_paused (the run waits for budget). None: nothing_better (nothing beat the version beyond noise; message says the smallest gain the labels could show), accuracy_would_drop. A narrowing run's own reasons are recipe tuning's. Null when ready, running or quoting.

Available options:

awaiting_confirm,

awaiting_run,

too_few_outcomes,

composite,

inherited,

budget_paused,

nothing_better,

accuracy_would_drop,

superseded

message

string

required

The status in a sentence.

automatic

boolean

required

Whether the run starts by itself after a fit, at most monthly (alternatives.automatic on the judgment). Off by default.

skip_engines

object[]

required

Engines the judgment's setting skips as automatic alternatives.

Show child attributes

labels

object

required

The labelled answers the alternatives are scored on.

Show child attributes

corrected_for

integer

required

Engine and named alternatives scored this month. Every better and moves_ranking holds after correcting for all of them.

Required range: x >= 0

quote

object | null

required

What scoring the unscored engine and named alternatives would bill, priced before anything runs, and the smallest gain the labels could show. Each answered call is billed as judging: its size class times its engine's weight, at the graduated prices of the tiers your organization is in now.

Show child attributes

current

object | null

required

The active version on the same labels, scored the same way. Null before a run.

Show child attributes

alternatives

object[]

required

Engine and named alternatives first (in_k: true), then recipe tuning's narrowings.

Show child attributes

suggestion

object | null

required

The alternative the outcomes prefer, by id, and why. Nothing is applied.

Show child attributes

headline

string | null

required

The suggestion in a sentence, with the correction it holds after.

computed_at

string<date-time> | null

required

RFC 3339, UTC.

next_run_after

string<date-time> | null

required

When the automatic run may next start. Null while automatic is off.

plan_required

object | null

required

Below Team, why scoring engine and named alternatives, and the narrowings' recipes, are withheld.

Show child attributes

Namespaces

Documents

Judgments

Outcomes and calibration

Groups

Subscriptions

Jobs

Engines

Organization

Webhooks

Score the alternatives on your outcomes

Authorizations

Headers

Path Parameters

Body

Response