Compare retained simulation outcomes and customer measurements
curl --request GET \
--url https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"schemaVersion": 1,
"projectId": "<string>",
"observedAtMs": 0,
"comparability": {
"sameBuild": true,
"sameCases": true,
"caseSelectionDigest": "<string>",
"incompleteBatchIds": [
"<string>"
]
},
"batches": [
{
"batchId": "<string>",
"environmentId": "<string>",
"planId": "<string>",
"planDigest": "<string>",
"buildHash": "<string>",
"state": "preparing",
"counts": {
"logicalCases": 0,
"attemptCases": 0,
"pending": 0,
"active": 0,
"passed": 0,
"failed": 0,
"inconclusive": 0,
"error": 0,
"cancelled": 0
},
"metrics": {
"retainedRunResults": 0,
"attemptsWithoutRetainedRunResult": 0,
"totalInteractions": 0,
"processedInteractions": -4503599627345496,
"withMetrics": 0,
"missingMetrics": 0,
"invalidMetrics": 0,
"truncated": true,
"wallTime": {
"scope": "customer-interaction",
"samples": 0,
"unknownInteractions": 0,
"p50Ms": 1,
"p95Ms": 1,
"minMs": 1,
"maxMs": 1
},
"recordedModelCalls": 0,
"usage": {
"inputTokens": {
"sum": 0,
"knownCalls": 0,
"unknownCalls": 0,
"overflow": true
},
"outputTokens": {
"sum": 0,
"knownCalls": 0,
"unknownCalls": 0,
"overflow": true
},
"cacheReadInputTokens": {
"sum": 0,
"knownCalls": 0,
"unknownCalls": 0,
"overflow": true
},
"cacheCreationInputTokens": {
"sum": 0,
"knownCalls": 0,
"unknownCalls": 0,
"overflow": true
}
},
"costs": [
{
"currency": "<string>",
"basis": "reported",
"amount": 1,
"knownCalls": 0,
"overflow": true,
"pricingDescription": "<string>"
}
],
"unknownCostCalls": 0,
"benchmark": {
"status": "consistent",
"value": {
"campaignId": "<string>",
"variantId": "<string>",
"label": "<string>",
"configurationDigest": "<string>"
}
},
"evaluations": [
{
"name": "<string>",
"version": "<string>",
"provenance": "customer-evaluator",
"samples": 4503599627370495,
"mean": 123,
"min": 123,
"max": 123,
"minimum": 123,
"maximum": 123
}
],
"models": [
{
"provider": "<string>",
"requestedModel": "<string>",
"responseModel": "<string>",
"calls": 4503599627370495
}
],
"groupTruncated": true
},
"sourceEnvironmentId": "<string>"
}
]
}{
"schemaVersion": 1,
"code": "<string>",
"source": "framework",
"message": "<string>",
"retryable": true,
"issues": [],
"correlationId": "<string>",
"details": {},
"evidence": {
"runId": "<string>",
"sequence": 4503599627370495
}
}Compare retained simulation outcomes and customer measurements
Read one through eight unique batches in this authorized project. Compare exact build and case selections before interpreting outcomes, latency, recorded model usage and reported or explicitly estimated costs. Includes retained failed/cancelled/retry attempt measurements; missing values stay unknown. Reads at most 50,000 interactions and 128 groups with explicit coverage/truncation. Does not run agents, select a winner or change native verdicts.
Compare retained simulation outcomes and customer measurements
curl --request GET \
--url https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.firedrill.run/v1/projects/{projectId}/simulation-comparisons")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"schemaVersion": 1,
"projectId": "<string>",
"observedAtMs": 0,
"comparability": {
"sameBuild": true,
"sameCases": true,
"caseSelectionDigest": "<string>",
"incompleteBatchIds": [
"<string>"
]
},
"batches": [
{
"batchId": "<string>",
"environmentId": "<string>",
"planId": "<string>",
"planDigest": "<string>",
"buildHash": "<string>",
"state": "preparing",
"counts": {
"logicalCases": 0,
"attemptCases": 0,
"pending": 0,
"active": 0,
"passed": 0,
"failed": 0,
"inconclusive": 0,
"error": 0,
"cancelled": 0
},
"metrics": {
"retainedRunResults": 0,
"attemptsWithoutRetainedRunResult": 0,
"totalInteractions": 0,
"processedInteractions": -4503599627345496,
"withMetrics": 0,
"missingMetrics": 0,
"invalidMetrics": 0,
"truncated": true,
"wallTime": {
"scope": "customer-interaction",
"samples": 0,
"unknownInteractions": 0,
"p50Ms": 1,
"p95Ms": 1,
"minMs": 1,
"maxMs": 1
},
"recordedModelCalls": 0,
"usage": {
"inputTokens": {
"sum": 0,
"knownCalls": 0,
"unknownCalls": 0,
"overflow": true
},
"outputTokens": {
"sum": 0,
"knownCalls": 0,
"unknownCalls": 0,
"overflow": true
},
"cacheReadInputTokens": {
"sum": 0,
"knownCalls": 0,
"unknownCalls": 0,
"overflow": true
},
"cacheCreationInputTokens": {
"sum": 0,
"knownCalls": 0,
"unknownCalls": 0,
"overflow": true
}
},
"costs": [
{
"currency": "<string>",
"basis": "reported",
"amount": 1,
"knownCalls": 0,
"overflow": true,
"pricingDescription": "<string>"
}
],
"unknownCostCalls": 0,
"benchmark": {
"status": "consistent",
"value": {
"campaignId": "<string>",
"variantId": "<string>",
"label": "<string>",
"configurationDigest": "<string>"
}
},
"evaluations": [
{
"name": "<string>",
"version": "<string>",
"provenance": "customer-evaluator",
"samples": 4503599627370495,
"mean": 123,
"min": 123,
"max": 123,
"minimum": 123,
"maximum": 123
}
],
"models": [
{
"provider": "<string>",
"requestedModel": "<string>",
"responseModel": "<string>",
"calls": 4503599627370495
}
],
"groupTruncated": true
},
"sourceEnvironmentId": "<string>"
}
]
}{
"schemaVersion": 1,
"code": "<string>",
"source": "framework",
"message": "<string>",
"retryable": true,
"issues": [],
"correlationId": "<string>",
"details": {},
"evidence": {
"runId": "<string>",
"sequence": 4503599627370495
}
}Authorizations
Opaque control credential
Path Parameters
Pattern:
^prj_[0-9a-z]{12,32}$Query Parameters
One through eight unique simulation batch IDs separated by commas, without whitespace.
Maximum string length:
328Response
Factual simulation comparison
Available options:
1 Pattern:
^prj_[0-9a-z]{12,32}$Required range:
-9007199254740991 <= x <= 9007199254740991Show child attributes
Show child attributes
Required array length:
1 - 8 elementsShow child attributes
Show child attributes