Start an evaluation
curl --request POST \
--url https://api.trajectory.ai/api/v1/eval \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <api-key>' \
--data '
{
"bench_id": "<string>",
"agent_name": "<string>",
"benchmark_name": "<string>",
"bypass_ownership": false,
"agent_id": "<string>",
"options": {
"load_weights_only": true,
"density_aware_training": true,
"train_on_truncation": true,
"train_on_error": true,
"train_on_timeout": true,
"val_enabled": true,
"val_interval_steps": 50,
"val_num_tasks": 2,
"val_trajectories_per_task": 2,
"val_max_concurrent_trajectories": 2,
"val_final_wait_timeout_seconds": 1,
"learning_rate": 0.005000005,
"lr_warmup_steps": 50000,
"lr_hold_steps": 50000,
"lr_min_ratio": 0.5,
"temperature": 1,
"top_p": 0.5,
"top_k": 0,
"val_temperature": 1,
"val_top_p": 0.5,
"val_top_k": 0,
"val_max_output_tokens_per_turn": 131072,
"val_max_output_tokens_per_trajectory": 1048576,
"val_max_turns_per_trajectory": 250,
"val_trajectory_timeout_seconds": 18000,
"num_steps": 50000,
"batch_size": 128,
"group_size": 16,
"min_trajectories_per_group": 16,
"max_concurrent_trajectories": 512,
"max_staleness": 50,
"train_on_all_fail": true,
"train_on_all_pass": true,
"train_on_zero_advantage": true,
"trajectory_timeout_seconds": 18000,
"batch_timeout_seconds": 19200,
"max_output_tokens_per_turn": 131072,
"max_output_tokens_per_trajectory": 1048576,
"max_turns_per_trajectory": 250
},
"parent_checkpoint_id": "<string>",
"display_name": "<string>",
"deployment_id": "<string>",
"task_split": "test"
}
'import requests
url = "https://api.trajectory.ai/api/v1/eval"
payload = {
"bench_id": "<string>",
"agent_name": "<string>",
"benchmark_name": "<string>",
"bypass_ownership": False,
"agent_id": "<string>",
"options": {
"load_weights_only": True,
"density_aware_training": True,
"train_on_truncation": True,
"train_on_error": True,
"train_on_timeout": True,
"val_enabled": True,
"val_interval_steps": 50,
"val_num_tasks": 2,
"val_trajectories_per_task": 2,
"val_max_concurrent_trajectories": 2,
"val_final_wait_timeout_seconds": 1,
"learning_rate": 0.005000005,
"lr_warmup_steps": 50000,
"lr_hold_steps": 50000,
"lr_min_ratio": 0.5,
"temperature": 1,
"top_p": 0.5,
"top_k": 0,
"val_temperature": 1,
"val_top_p": 0.5,
"val_top_k": 0,
"val_max_output_tokens_per_turn": 131072,
"val_max_output_tokens_per_trajectory": 1048576,
"val_max_turns_per_trajectory": 250,
"val_trajectory_timeout_seconds": 18000,
"num_steps": 50000,
"batch_size": 128,
"group_size": 16,
"min_trajectories_per_group": 16,
"max_concurrent_trajectories": 512,
"max_staleness": 50,
"train_on_all_fail": True,
"train_on_all_pass": True,
"train_on_zero_advantage": True,
"trajectory_timeout_seconds": 18000,
"batch_timeout_seconds": 19200,
"max_output_tokens_per_turn": 131072,
"max_output_tokens_per_trajectory": 1048576,
"max_turns_per_trajectory": 250
},
"parent_checkpoint_id": "<string>",
"display_name": "<string>",
"deployment_id": "<string>",
"task_split": "test"
}
headers = {
"X-API-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
bench_id: '<string>',
agent_name: '<string>',
benchmark_name: '<string>',
bypass_ownership: false,
agent_id: '<string>',
options: {
load_weights_only: true,
density_aware_training: true,
train_on_truncation: true,
train_on_error: true,
train_on_timeout: true,
val_enabled: true,
val_interval_steps: 50,
val_num_tasks: 2,
val_trajectories_per_task: 2,
val_max_concurrent_trajectories: 2,
val_final_wait_timeout_seconds: 1,
learning_rate: 0.005000005,
lr_warmup_steps: 50000,
lr_hold_steps: 50000,
lr_min_ratio: 0.5,
temperature: 1,
top_p: 0.5,
top_k: 0,
val_temperature: 1,
val_top_p: 0.5,
val_top_k: 0,
val_max_output_tokens_per_turn: 131072,
val_max_output_tokens_per_trajectory: 1048576,
val_max_turns_per_trajectory: 250,
val_trajectory_timeout_seconds: 18000,
num_steps: 50000,
batch_size: 128,
group_size: 16,
min_trajectories_per_group: 16,
max_concurrent_trajectories: 512,
max_staleness: 50,
train_on_all_fail: true,
train_on_all_pass: true,
train_on_zero_advantage: true,
trajectory_timeout_seconds: 18000,
batch_timeout_seconds: 19200,
max_output_tokens_per_turn: 131072,
max_output_tokens_per_trajectory: 1048576,
max_turns_per_trajectory: 250
},
parent_checkpoint_id: '<string>',
display_name: '<string>',
deployment_id: '<string>',
task_split: 'test'
})
};
fetch('https://api.trajectory.ai/api/v1/eval', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.trajectory.ai/api/v1/eval",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'bench_id' => '<string>',
'agent_name' => '<string>',
'benchmark_name' => '<string>',
'bypass_ownership' => false,
'agent_id' => '<string>',
'options' => [
'load_weights_only' => true,
'density_aware_training' => true,
'train_on_truncation' => true,
'train_on_error' => true,
'train_on_timeout' => true,
'val_enabled' => true,
'val_interval_steps' => 50,
'val_num_tasks' => 2,
'val_trajectories_per_task' => 2,
'val_max_concurrent_trajectories' => 2,
'val_final_wait_timeout_seconds' => 1,
'learning_rate' => 0.005000005,
'lr_warmup_steps' => 50000,
'lr_hold_steps' => 50000,
'lr_min_ratio' => 0.5,
'temperature' => 1,
'top_p' => 0.5,
'top_k' => 0,
'val_temperature' => 1,
'val_top_p' => 0.5,
'val_top_k' => 0,
'val_max_output_tokens_per_turn' => 131072,
'val_max_output_tokens_per_trajectory' => 1048576,
'val_max_turns_per_trajectory' => 250,
'val_trajectory_timeout_seconds' => 18000,
'num_steps' => 50000,
'batch_size' => 128,
'group_size' => 16,
'min_trajectories_per_group' => 16,
'max_concurrent_trajectories' => 512,
'max_staleness' => 50,
'train_on_all_fail' => true,
'train_on_all_pass' => true,
'train_on_zero_advantage' => true,
'trajectory_timeout_seconds' => 18000,
'batch_timeout_seconds' => 19200,
'max_output_tokens_per_turn' => 131072,
'max_output_tokens_per_trajectory' => 1048576,
'max_turns_per_trajectory' => 250
],
'parent_checkpoint_id' => '<string>',
'display_name' => '<string>',
'deployment_id' => '<string>',
'task_split' => 'test'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.trajectory.ai/api/v1/eval"
payload := strings.NewReader("{\n \"bench_id\": \"<string>\",\n \"agent_name\": \"<string>\",\n \"benchmark_name\": \"<string>\",\n \"bypass_ownership\": false,\n \"agent_id\": \"<string>\",\n \"options\": {\n \"load_weights_only\": true,\n \"density_aware_training\": true,\n \"train_on_truncation\": true,\n \"train_on_error\": true,\n \"train_on_timeout\": true,\n \"val_enabled\": true,\n \"val_interval_steps\": 50,\n \"val_num_tasks\": 2,\n \"val_trajectories_per_task\": 2,\n \"val_max_concurrent_trajectories\": 2,\n \"val_final_wait_timeout_seconds\": 1,\n \"learning_rate\": 0.005000005,\n \"lr_warmup_steps\": 50000,\n \"lr_hold_steps\": 50000,\n \"lr_min_ratio\": 0.5,\n \"temperature\": 1,\n \"top_p\": 0.5,\n \"top_k\": 0,\n \"val_temperature\": 1,\n \"val_top_p\": 0.5,\n \"val_top_k\": 0,\n \"val_max_output_tokens_per_turn\": 131072,\n \"val_max_output_tokens_per_trajectory\": 1048576,\n \"val_max_turns_per_trajectory\": 250,\n \"val_trajectory_timeout_seconds\": 18000,\n \"num_steps\": 50000,\n \"batch_size\": 128,\n \"group_size\": 16,\n \"min_trajectories_per_group\": 16,\n \"max_concurrent_trajectories\": 512,\n \"max_staleness\": 50,\n \"train_on_all_fail\": true,\n \"train_on_all_pass\": true,\n \"train_on_zero_advantage\": true,\n \"trajectory_timeout_seconds\": 18000,\n \"batch_timeout_seconds\": 19200,\n \"max_output_tokens_per_turn\": 131072,\n \"max_output_tokens_per_trajectory\": 1048576,\n \"max_turns_per_trajectory\": 250\n },\n \"parent_checkpoint_id\": \"<string>\",\n \"display_name\": \"<string>\",\n \"deployment_id\": \"<string>\",\n \"task_split\": \"test\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.trajectory.ai/api/v1/eval")
.header("X-API-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"bench_id\": \"<string>\",\n \"agent_name\": \"<string>\",\n \"benchmark_name\": \"<string>\",\n \"bypass_ownership\": false,\n \"agent_id\": \"<string>\",\n \"options\": {\n \"load_weights_only\": true,\n \"density_aware_training\": true,\n \"train_on_truncation\": true,\n \"train_on_error\": true,\n \"train_on_timeout\": true,\n \"val_enabled\": true,\n \"val_interval_steps\": 50,\n \"val_num_tasks\": 2,\n \"val_trajectories_per_task\": 2,\n \"val_max_concurrent_trajectories\": 2,\n \"val_final_wait_timeout_seconds\": 1,\n \"learning_rate\": 0.005000005,\n \"lr_warmup_steps\": 50000,\n \"lr_hold_steps\": 50000,\n \"lr_min_ratio\": 0.5,\n \"temperature\": 1,\n \"top_p\": 0.5,\n \"top_k\": 0,\n \"val_temperature\": 1,\n \"val_top_p\": 0.5,\n \"val_top_k\": 0,\n \"val_max_output_tokens_per_turn\": 131072,\n \"val_max_output_tokens_per_trajectory\": 1048576,\n \"val_max_turns_per_trajectory\": 250,\n \"val_trajectory_timeout_seconds\": 18000,\n \"num_steps\": 50000,\n \"batch_size\": 128,\n \"group_size\": 16,\n \"min_trajectories_per_group\": 16,\n \"max_concurrent_trajectories\": 512,\n \"max_staleness\": 50,\n \"train_on_all_fail\": true,\n \"train_on_all_pass\": true,\n \"train_on_zero_advantage\": true,\n \"trajectory_timeout_seconds\": 18000,\n \"batch_timeout_seconds\": 19200,\n \"max_output_tokens_per_turn\": 131072,\n \"max_output_tokens_per_trajectory\": 1048576,\n \"max_turns_per_trajectory\": 250\n },\n \"parent_checkpoint_id\": \"<string>\",\n \"display_name\": \"<string>\",\n \"deployment_id\": \"<string>\",\n \"task_split\": \"test\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.trajectory.ai/api/v1/eval")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"bench_id\": \"<string>\",\n \"agent_name\": \"<string>\",\n \"benchmark_name\": \"<string>\",\n \"bypass_ownership\": false,\n \"agent_id\": \"<string>\",\n \"options\": {\n \"load_weights_only\": true,\n \"density_aware_training\": true,\n \"train_on_truncation\": true,\n \"train_on_error\": true,\n \"train_on_timeout\": true,\n \"val_enabled\": true,\n \"val_interval_steps\": 50,\n \"val_num_tasks\": 2,\n \"val_trajectories_per_task\": 2,\n \"val_max_concurrent_trajectories\": 2,\n \"val_final_wait_timeout_seconds\": 1,\n \"learning_rate\": 0.005000005,\n \"lr_warmup_steps\": 50000,\n \"lr_hold_steps\": 50000,\n \"lr_min_ratio\": 0.5,\n \"temperature\": 1,\n \"top_p\": 0.5,\n \"top_k\": 0,\n \"val_temperature\": 1,\n \"val_top_p\": 0.5,\n \"val_top_k\": 0,\n \"val_max_output_tokens_per_turn\": 131072,\n \"val_max_output_tokens_per_trajectory\": 1048576,\n \"val_max_turns_per_trajectory\": 250,\n \"val_trajectory_timeout_seconds\": 18000,\n \"num_steps\": 50000,\n \"batch_size\": 128,\n \"group_size\": 16,\n \"min_trajectories_per_group\": 16,\n \"max_concurrent_trajectories\": 512,\n \"max_staleness\": 50,\n \"train_on_all_fail\": true,\n \"train_on_all_pass\": true,\n \"train_on_zero_advantage\": true,\n \"trajectory_timeout_seconds\": 18000,\n \"batch_timeout_seconds\": 19200,\n \"max_output_tokens_per_turn\": 131072,\n \"max_output_tokens_per_trajectory\": 1048576,\n \"max_turns_per_trajectory\": 250\n },\n \"parent_checkpoint_id\": \"<string>\",\n \"display_name\": \"<string>\",\n \"deployment_id\": \"<string>\",\n \"task_split\": \"test\"\n}"
response = http.request(request)
puts response.read_body{
"bench_id": "<string>",
"eval_run_id": "<string>",
"display_name": "<string>",
"task_selection": {
"split": "test",
"available_task_count": 1,
"selected_task_count": 1,
"max_samples": 123
}
}{
"error": {
"type": "invalid_request_error",
"code": "invalid_request",
"message": "<string>",
"doc_url": "<string>",
"context": {},
"param": "<string>"
}
}{
"error": {
"type": "billing_error",
"code": "credits_required",
"message": "<string>",
"doc_url": "<string>",
"context": {},
"param": "<string>"
}
}{
"error": {
"type": "rate_limit_error",
"code": "concurrency_limit_exceeded",
"message": "<string>",
"doc_url": "<string>",
"context": {
"resource": "evaluation",
"limit": 123,
"usage": 123
},
"param": "<string>"
}
}{
"error": {
"type": "invalid_request_error",
"code": "<string>",
"message": "<string>",
"doc_url": "<string>",
"context": {},
"param": "<string>"
}
}Evaluations
Start an evaluation
Create an evaluation with bench_id, base_model_slug, shared options, and an optional parent_checkpoint_id. Save the returned evaluation run ID to inspect progress and results.
POST
/
api
/
v1
/
eval
Start an evaluation
curl --request POST \
--url https://api.trajectory.ai/api/v1/eval \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <api-key>' \
--data '
{
"bench_id": "<string>",
"agent_name": "<string>",
"benchmark_name": "<string>",
"bypass_ownership": false,
"agent_id": "<string>",
"options": {
"load_weights_only": true,
"density_aware_training": true,
"train_on_truncation": true,
"train_on_error": true,
"train_on_timeout": true,
"val_enabled": true,
"val_interval_steps": 50,
"val_num_tasks": 2,
"val_trajectories_per_task": 2,
"val_max_concurrent_trajectories": 2,
"val_final_wait_timeout_seconds": 1,
"learning_rate": 0.005000005,
"lr_warmup_steps": 50000,
"lr_hold_steps": 50000,
"lr_min_ratio": 0.5,
"temperature": 1,
"top_p": 0.5,
"top_k": 0,
"val_temperature": 1,
"val_top_p": 0.5,
"val_top_k": 0,
"val_max_output_tokens_per_turn": 131072,
"val_max_output_tokens_per_trajectory": 1048576,
"val_max_turns_per_trajectory": 250,
"val_trajectory_timeout_seconds": 18000,
"num_steps": 50000,
"batch_size": 128,
"group_size": 16,
"min_trajectories_per_group": 16,
"max_concurrent_trajectories": 512,
"max_staleness": 50,
"train_on_all_fail": true,
"train_on_all_pass": true,
"train_on_zero_advantage": true,
"trajectory_timeout_seconds": 18000,
"batch_timeout_seconds": 19200,
"max_output_tokens_per_turn": 131072,
"max_output_tokens_per_trajectory": 1048576,
"max_turns_per_trajectory": 250
},
"parent_checkpoint_id": "<string>",
"display_name": "<string>",
"deployment_id": "<string>",
"task_split": "test"
}
'import requests
url = "https://api.trajectory.ai/api/v1/eval"
payload = {
"bench_id": "<string>",
"agent_name": "<string>",
"benchmark_name": "<string>",
"bypass_ownership": False,
"agent_id": "<string>",
"options": {
"load_weights_only": True,
"density_aware_training": True,
"train_on_truncation": True,
"train_on_error": True,
"train_on_timeout": True,
"val_enabled": True,
"val_interval_steps": 50,
"val_num_tasks": 2,
"val_trajectories_per_task": 2,
"val_max_concurrent_trajectories": 2,
"val_final_wait_timeout_seconds": 1,
"learning_rate": 0.005000005,
"lr_warmup_steps": 50000,
"lr_hold_steps": 50000,
"lr_min_ratio": 0.5,
"temperature": 1,
"top_p": 0.5,
"top_k": 0,
"val_temperature": 1,
"val_top_p": 0.5,
"val_top_k": 0,
"val_max_output_tokens_per_turn": 131072,
"val_max_output_tokens_per_trajectory": 1048576,
"val_max_turns_per_trajectory": 250,
"val_trajectory_timeout_seconds": 18000,
"num_steps": 50000,
"batch_size": 128,
"group_size": 16,
"min_trajectories_per_group": 16,
"max_concurrent_trajectories": 512,
"max_staleness": 50,
"train_on_all_fail": True,
"train_on_all_pass": True,
"train_on_zero_advantage": True,
"trajectory_timeout_seconds": 18000,
"batch_timeout_seconds": 19200,
"max_output_tokens_per_turn": 131072,
"max_output_tokens_per_trajectory": 1048576,
"max_turns_per_trajectory": 250
},
"parent_checkpoint_id": "<string>",
"display_name": "<string>",
"deployment_id": "<string>",
"task_split": "test"
}
headers = {
"X-API-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
bench_id: '<string>',
agent_name: '<string>',
benchmark_name: '<string>',
bypass_ownership: false,
agent_id: '<string>',
options: {
load_weights_only: true,
density_aware_training: true,
train_on_truncation: true,
train_on_error: true,
train_on_timeout: true,
val_enabled: true,
val_interval_steps: 50,
val_num_tasks: 2,
val_trajectories_per_task: 2,
val_max_concurrent_trajectories: 2,
val_final_wait_timeout_seconds: 1,
learning_rate: 0.005000005,
lr_warmup_steps: 50000,
lr_hold_steps: 50000,
lr_min_ratio: 0.5,
temperature: 1,
top_p: 0.5,
top_k: 0,
val_temperature: 1,
val_top_p: 0.5,
val_top_k: 0,
val_max_output_tokens_per_turn: 131072,
val_max_output_tokens_per_trajectory: 1048576,
val_max_turns_per_trajectory: 250,
val_trajectory_timeout_seconds: 18000,
num_steps: 50000,
batch_size: 128,
group_size: 16,
min_trajectories_per_group: 16,
max_concurrent_trajectories: 512,
max_staleness: 50,
train_on_all_fail: true,
train_on_all_pass: true,
train_on_zero_advantage: true,
trajectory_timeout_seconds: 18000,
batch_timeout_seconds: 19200,
max_output_tokens_per_turn: 131072,
max_output_tokens_per_trajectory: 1048576,
max_turns_per_trajectory: 250
},
parent_checkpoint_id: '<string>',
display_name: '<string>',
deployment_id: '<string>',
task_split: 'test'
})
};
fetch('https://api.trajectory.ai/api/v1/eval', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.trajectory.ai/api/v1/eval",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'bench_id' => '<string>',
'agent_name' => '<string>',
'benchmark_name' => '<string>',
'bypass_ownership' => false,
'agent_id' => '<string>',
'options' => [
'load_weights_only' => true,
'density_aware_training' => true,
'train_on_truncation' => true,
'train_on_error' => true,
'train_on_timeout' => true,
'val_enabled' => true,
'val_interval_steps' => 50,
'val_num_tasks' => 2,
'val_trajectories_per_task' => 2,
'val_max_concurrent_trajectories' => 2,
'val_final_wait_timeout_seconds' => 1,
'learning_rate' => 0.005000005,
'lr_warmup_steps' => 50000,
'lr_hold_steps' => 50000,
'lr_min_ratio' => 0.5,
'temperature' => 1,
'top_p' => 0.5,
'top_k' => 0,
'val_temperature' => 1,
'val_top_p' => 0.5,
'val_top_k' => 0,
'val_max_output_tokens_per_turn' => 131072,
'val_max_output_tokens_per_trajectory' => 1048576,
'val_max_turns_per_trajectory' => 250,
'val_trajectory_timeout_seconds' => 18000,
'num_steps' => 50000,
'batch_size' => 128,
'group_size' => 16,
'min_trajectories_per_group' => 16,
'max_concurrent_trajectories' => 512,
'max_staleness' => 50,
'train_on_all_fail' => true,
'train_on_all_pass' => true,
'train_on_zero_advantage' => true,
'trajectory_timeout_seconds' => 18000,
'batch_timeout_seconds' => 19200,
'max_output_tokens_per_turn' => 131072,
'max_output_tokens_per_trajectory' => 1048576,
'max_turns_per_trajectory' => 250
],
'parent_checkpoint_id' => '<string>',
'display_name' => '<string>',
'deployment_id' => '<string>',
'task_split' => 'test'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.trajectory.ai/api/v1/eval"
payload := strings.NewReader("{\n \"bench_id\": \"<string>\",\n \"agent_name\": \"<string>\",\n \"benchmark_name\": \"<string>\",\n \"bypass_ownership\": false,\n \"agent_id\": \"<string>\",\n \"options\": {\n \"load_weights_only\": true,\n \"density_aware_training\": true,\n \"train_on_truncation\": true,\n \"train_on_error\": true,\n \"train_on_timeout\": true,\n \"val_enabled\": true,\n \"val_interval_steps\": 50,\n \"val_num_tasks\": 2,\n \"val_trajectories_per_task\": 2,\n \"val_max_concurrent_trajectories\": 2,\n \"val_final_wait_timeout_seconds\": 1,\n \"learning_rate\": 0.005000005,\n \"lr_warmup_steps\": 50000,\n \"lr_hold_steps\": 50000,\n \"lr_min_ratio\": 0.5,\n \"temperature\": 1,\n \"top_p\": 0.5,\n \"top_k\": 0,\n \"val_temperature\": 1,\n \"val_top_p\": 0.5,\n \"val_top_k\": 0,\n \"val_max_output_tokens_per_turn\": 131072,\n \"val_max_output_tokens_per_trajectory\": 1048576,\n \"val_max_turns_per_trajectory\": 250,\n \"val_trajectory_timeout_seconds\": 18000,\n \"num_steps\": 50000,\n \"batch_size\": 128,\n \"group_size\": 16,\n \"min_trajectories_per_group\": 16,\n \"max_concurrent_trajectories\": 512,\n \"max_staleness\": 50,\n \"train_on_all_fail\": true,\n \"train_on_all_pass\": true,\n \"train_on_zero_advantage\": true,\n \"trajectory_timeout_seconds\": 18000,\n \"batch_timeout_seconds\": 19200,\n \"max_output_tokens_per_turn\": 131072,\n \"max_output_tokens_per_trajectory\": 1048576,\n \"max_turns_per_trajectory\": 250\n },\n \"parent_checkpoint_id\": \"<string>\",\n \"display_name\": \"<string>\",\n \"deployment_id\": \"<string>\",\n \"task_split\": \"test\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.trajectory.ai/api/v1/eval")
.header("X-API-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"bench_id\": \"<string>\",\n \"agent_name\": \"<string>\",\n \"benchmark_name\": \"<string>\",\n \"bypass_ownership\": false,\n \"agent_id\": \"<string>\",\n \"options\": {\n \"load_weights_only\": true,\n \"density_aware_training\": true,\n \"train_on_truncation\": true,\n \"train_on_error\": true,\n \"train_on_timeout\": true,\n \"val_enabled\": true,\n \"val_interval_steps\": 50,\n \"val_num_tasks\": 2,\n \"val_trajectories_per_task\": 2,\n \"val_max_concurrent_trajectories\": 2,\n \"val_final_wait_timeout_seconds\": 1,\n \"learning_rate\": 0.005000005,\n \"lr_warmup_steps\": 50000,\n \"lr_hold_steps\": 50000,\n \"lr_min_ratio\": 0.5,\n \"temperature\": 1,\n \"top_p\": 0.5,\n \"top_k\": 0,\n \"val_temperature\": 1,\n \"val_top_p\": 0.5,\n \"val_top_k\": 0,\n \"val_max_output_tokens_per_turn\": 131072,\n \"val_max_output_tokens_per_trajectory\": 1048576,\n \"val_max_turns_per_trajectory\": 250,\n \"val_trajectory_timeout_seconds\": 18000,\n \"num_steps\": 50000,\n \"batch_size\": 128,\n \"group_size\": 16,\n \"min_trajectories_per_group\": 16,\n \"max_concurrent_trajectories\": 512,\n \"max_staleness\": 50,\n \"train_on_all_fail\": true,\n \"train_on_all_pass\": true,\n \"train_on_zero_advantage\": true,\n \"trajectory_timeout_seconds\": 18000,\n \"batch_timeout_seconds\": 19200,\n \"max_output_tokens_per_turn\": 131072,\n \"max_output_tokens_per_trajectory\": 1048576,\n \"max_turns_per_trajectory\": 250\n },\n \"parent_checkpoint_id\": \"<string>\",\n \"display_name\": \"<string>\",\n \"deployment_id\": \"<string>\",\n \"task_split\": \"test\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.trajectory.ai/api/v1/eval")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"bench_id\": \"<string>\",\n \"agent_name\": \"<string>\",\n \"benchmark_name\": \"<string>\",\n \"bypass_ownership\": false,\n \"agent_id\": \"<string>\",\n \"options\": {\n \"load_weights_only\": true,\n \"density_aware_training\": true,\n \"train_on_truncation\": true,\n \"train_on_error\": true,\n \"train_on_timeout\": true,\n \"val_enabled\": true,\n \"val_interval_steps\": 50,\n \"val_num_tasks\": 2,\n \"val_trajectories_per_task\": 2,\n \"val_max_concurrent_trajectories\": 2,\n \"val_final_wait_timeout_seconds\": 1,\n \"learning_rate\": 0.005000005,\n \"lr_warmup_steps\": 50000,\n \"lr_hold_steps\": 50000,\n \"lr_min_ratio\": 0.5,\n \"temperature\": 1,\n \"top_p\": 0.5,\n \"top_k\": 0,\n \"val_temperature\": 1,\n \"val_top_p\": 0.5,\n \"val_top_k\": 0,\n \"val_max_output_tokens_per_turn\": 131072,\n \"val_max_output_tokens_per_trajectory\": 1048576,\n \"val_max_turns_per_trajectory\": 250,\n \"val_trajectory_timeout_seconds\": 18000,\n \"num_steps\": 50000,\n \"batch_size\": 128,\n \"group_size\": 16,\n \"min_trajectories_per_group\": 16,\n \"max_concurrent_trajectories\": 512,\n \"max_staleness\": 50,\n \"train_on_all_fail\": true,\n \"train_on_all_pass\": true,\n \"train_on_zero_advantage\": true,\n \"trajectory_timeout_seconds\": 18000,\n \"batch_timeout_seconds\": 19200,\n \"max_output_tokens_per_turn\": 131072,\n \"max_output_tokens_per_trajectory\": 1048576,\n \"max_turns_per_trajectory\": 250\n },\n \"parent_checkpoint_id\": \"<string>\",\n \"display_name\": \"<string>\",\n \"deployment_id\": \"<string>\",\n \"task_split\": \"test\"\n}"
response = http.request(request)
puts response.read_body{
"bench_id": "<string>",
"eval_run_id": "<string>",
"display_name": "<string>",
"task_selection": {
"split": "test",
"available_task_count": 1,
"selected_task_count": 1,
"max_samples": 123
}
}{
"error": {
"type": "invalid_request_error",
"code": "invalid_request",
"message": "<string>",
"doc_url": "<string>",
"context": {},
"param": "<string>"
}
}{
"error": {
"type": "billing_error",
"code": "credits_required",
"message": "<string>",
"doc_url": "<string>",
"context": {},
"param": "<string>"
}
}{
"error": {
"type": "rate_limit_error",
"code": "concurrency_limit_exceeded",
"message": "<string>",
"doc_url": "<string>",
"context": {
"resource": "evaluation",
"limit": 123,
"usage": 123
},
"param": "<string>"
}
}{
"error": {
"type": "invalid_request_error",
"code": "<string>",
"message": "<string>",
"doc_url": "<string>",
"context": {},
"param": "<string>"
}
}Authorizations
Organization API key
Headers
Optional. Without a key, repeated submissions are treated as separate actions. A blank or over-long key is rejected. Reuse this key when retrying a submission; the SDK generates one per call and reuses it across automatic retries.
Required string length:
1 - 256Body
application/json
Benchmark ID; use instead of agent_name and benchmark_name.
Use with benchmark_name to select its latest benchmark version.
Use with agent_name to select the latest benchmark version.
Allow a supplied agent to differ from the benchmark owner within the same organization; run attribution uses the benchmark owner.
Optional agent for ownership checks; not a benchmark selector.
Required string length:
1 - 64Registered base model; mutually exclusive with deployment_id.
Available options:
thinkingmachines/Inkling-Small, Qwen/Qwen3.5-4B, Qwen/Qwen3.5-397B-A17B, Qwen/Qwen3.6-35B-A3B, Qwen/Qwen3.8-27B, nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16, nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16, nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16, nvidia/NVIDIA-Nemotron-3.5-Super-120B-A12B-BF16, openai/gpt-6.1-sol, openai/gpt-6-sol, openai/gpt-6-astra, openai/gpt-5.6-sol, openai/gpt-5.6-luna, anthropic/claude-opus-5.5, anthropic/claude-sonnet-5.5, openai/gpt-5-mini, openai/gpt-5.4-mini, openai/gpt-5.5, anthropic/claude-sonnet-4.6, google/gemini-3.1-pro-preview, z-ai/glm-5.3, z-ai/glm-5.3-flash, moonshotai/kimi-k3 Run settings; omission, null, and an empty object preserve defaults.
Show child attributes
Show child attributes
Checkpoint to resume training from or evaluate; must match the selected base model. Checkpoint evaluations inherit supported token and turn limits from the source training run unless the evaluation overrides them.
Required string length:
1 - 64Required string length:
1 - 256Evaluate the exact hosted checkpoint and pool of a ready deployment.
Required string length:
1 - 64Evaluate test tasks by default; all includes both test and train tasks.
Available options:
test, all Response
Successful Response
Benchmark evaluated by this run.
Evaluation run created for this request.
User-facing name for the evaluation run.
Task coverage fixed at launch; null for legacy or diagnostic runs.
Show child attributes
Show child attributes