Draw documents to label

curl --request POST \
  --url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '{
  "count": 50
}'

import requests

url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue"

payload = { "count": 50 }
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({count: 50})
};

fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'count' => 50
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue"

	payload := strings.NewReader("{\n  \"count\": 50\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"count\": 50\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"count\": 50\n}"

response = http.request(request)
puts response.read_body

{
  "judgment": "<string>",
  "version": 2,
  "engine": "<string>",
  "engine_version": "<string>",
  "seed": 1,
  "lease_expires_at": "2023-11-07T05:31:56Z",
  "items": [
    {
      "queue_item_id": "<string>",
      "document_id": "<string>",
      "band": {
        "index": 4,
        "lower": 0.5,
        "upper": 0.5
      },
      "probability": 0.5,
      "value": "<string>",
      "evaluation_id": "<string>",
      "document": {
        "id": "<string>",
        "revision": 1,
        "attributes": {},
        "state": {}
      }
    }
  ]
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

Outcomes and calibration

Draw documents to label

Draws up to count documents to label, a random sample stratified across the probability range, so the queue hands out documents from every band, not just where most answers sit. Within a band, documents come in an order fixed by seed, so the same state and seed give the same items. Documents already labelled from the queue for their current answer, and those leased to someone else, are skipped.

Each draw leases its documents for 7 days, so two people are not given the same one. Post each label as an ordinary outcome with the item’s queue_item_id: it counts as source: queue. Once an epoch’s queue labels alone meet the calibration minimums, the epoch is fitted on them (fitted_on: queue), weighted back to your real traffic.

Drawing is part of the learning loop, on the Team plan and above: below it, plan_required. The preview (GET) is on every plan. A judgment with a non-zero horizon has no queue (invalid_request).

POST

namespaces

{ns}

judgments

{name}

labelling-queue

Draw documents to label

curl --request POST \
  --url https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '{
  "count": 50
}'

import requests

url = "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue"

payload = { "count": 50 }
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({count: 50})
};

fetch('https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'count' => 50
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue"

	payload := strings.NewReader("{\n  \"count\": 50\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"count\": 50\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.brussle.com/v1/namespaces/{ns}/judgments/{name}/labelling-queue")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"count\": 50\n}"

response = http.request(request)
puts response.read_body

{
  "judgment": "<string>",
  "version": 2,
  "engine": "<string>",
  "engine_version": "<string>",
  "seed": 1,
  "lease_expires_at": "2023-11-07T05:31:56Z",
  "items": [
    {
      "queue_item_id": "<string>",
      "document_id": "<string>",
      "band": {
        "index": 4,
        "lower": 0.5,
        "upper": 0.5
      },
      "probability": 0.5,
      "value": "<string>",
      "evaluation_id": "<string>",
      "document": {
        "id": "<string>",
        "revision": 1,
        "attributes": {},
        "state": {}
      }
    }
  ]
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

{
  "error": {
    "code": "invalid_request",
    "message": "<string>",
    "details": {
      "reason": "<string>",
      "suggested_context": {
        "recipe": {
          "fields": [
            "<string>"
          ],
          "last_n": {},
          "window": {},
          "max_tokens": 2,
          "related": {},
          "previous": {
            "fields": [
              "<string>"
            ],
            "anchor": "<string>"
          }
        },
        "cost_per_answer": 1,
        "whole_state_cost_per_answer": 1,
        "sampled_documents": 2,
        "excluded": [
          {
            "path": "<string>",
            "reason": "id"
          }
        ]
      }
    }
  }
}

Authorizations

Authorization

string

header

required

An organization API key. Keys carry a role (read_write or read_only) and may be restricted to a namespace prefix such as acme/*, or to one namespace such as acme/prod/tenant_1. A prefix matches on a / boundary: acme/prod/tenant_1* covers acme/prod/tenant_1 and everything under acme/prod/tenant_1/, never acme/prod/tenant_12.

Headers

Idempotency-Key

string

One key per logical request, reused only on its retries. A key belongs to one request: within your organization, the same method, path, query and body. For 24 hours after a successful response, a request with the key and the same body gets that response back verbatim, with Idempotent-Replayed: true, and runs nothing. Only a successful response is kept, so the retry of a request that failed runs again. While the first request runs or its response is kept, the key with a different request is idempotency_key_reused (422). A request sent while one with its key is still running is rate_limited with Retry-After: 1, without running: retry it to get the first one's response. In the rare case the key can't be checked, the request runs as if it had none.

Required string length: 1 - 255

Path Parameters

string

required

The namespace name, with any / sent as %2F. Up to 256 bytes. / separates levels of the hierarchy, as in acme/prod/tenant_123. Never exactly . or .., which a URL path can't carry.

Required string length: 1 - 256

Pattern: ^(?!\.\.?$)[A-Za-z0-9._:/-]+$

name

string

required

A judgment, attribute or threshold name. Names are path segments in field references, so they never contain ..

Required string length: 1 - 128

Pattern: ^[A-Za-z0-9_-]+$

Body

application/json

count

integer

default:50

How many documents to hand out at most.

Required range: 1 <= x <= 200

seed

integer<int64>

default:0

Fixes each band's random order. The same state and seed give the same items.

Required range: x >= 0

Response

The items drawn and leased; none when nothing is available.

Documents drawn from the labelling queue and leased.

judgment

string

required

A judgment, attribute or threshold name. Names are path segments in field references, so they never contain ..

Required string length: 1 - 128

Pattern: ^[A-Za-z0-9_-]+$

version

integer<int32>

required

Required range: x >= 1

engine

string | null

required

engine_version

string | null

required

The epoch drawn from. Null while the version has no answers.

seed

integer<int64>

required

Required range: x >= 0

lease_expires_at

string<date-time> | null

required

When the items go back into the queue if not labelled. Null when nothing was drawn.

items

object[]

required

Lowest band first; within a band, in the seed's order.

Maximum array length: 200

Show child attributes

Namespaces

Documents

Judgments

Outcomes and calibration

Groups

Subscriptions

Jobs

Engines

Organization

Webhooks

Draw documents to label

Authorizations

Headers

Path Parameters

Body

Response