Compare persisted serving evidence - InferCrane

POST

/

api

/

v1

/

lab

/

evaluations

Compare persisted serving evidence

curl --request POST \
  --url http://127.0.0.1:18000/api/v1/lab/evaluations \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "model_identity": "<string>",
  "max_error_rate": 0.5,
  "max_gpu_count": 512,
  "max_hourly_cost": 1,
  "max_tpot_p95_ms": 1,
  "max_ttft_p95_ms": 1,
  "min_goodput": 1,
  "min_output_tokens_second": 1,
  "objective": "latency",
  "region": "<string>",
  "workload_digest": "<string>"
}
'

import requests

url = "http://127.0.0.1:18000/api/v1/lab/evaluations"

payload = {
    "model_identity": "<string>",
    "max_error_rate": 0.5,
    "max_gpu_count": 512,
    "max_hourly_cost": 1,
    "max_tpot_p95_ms": 1,
    "max_ttft_p95_ms": 1,
    "min_goodput": 1,
    "min_output_tokens_second": 1,
    "objective": "latency",
    "region": "<string>",
    "workload_digest": "<string>"
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    model_identity: '<string>',
    max_error_rate: 0.5,
    max_gpu_count: 512,
    max_hourly_cost: 1,
    max_tpot_p95_ms: 1,
    max_ttft_p95_ms: 1,
    min_goodput: 1,
    min_output_tokens_second: 1,
    objective: 'latency',
    region: '<string>',
    workload_digest: '<string>'
  })
};

fetch('http://127.0.0.1:18000/api/v1/lab/evaluations', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "http://127.0.0.1:18000/api/v1/lab/evaluations"

	payload := strings.NewReader("{\n  \"model_identity\": \"<string>\",\n  \"max_error_rate\": 0.5,\n  \"max_gpu_count\": 512,\n  \"max_hourly_cost\": 1,\n  \"max_tpot_p95_ms\": 1,\n  \"max_ttft_p95_ms\": 1,\n  \"min_goodput\": 1,\n  \"min_output_tokens_second\": 1,\n  \"objective\": \"latency\",\n  \"region\": \"<string>\",\n  \"workload_digest\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

{}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "category": "<string>",
    "remediation": "<string>",
    "request_id": "<string>",
    "retryable": true
  }
}

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Body

application/json

model_identity

string

required

Minimum string length: 1

max_error_rate

number

Required range: 0 <= x <= 1

max_gpu_count

integer

Required range: 1 <= x <= 1024

max_hourly_cost

number

Required range: x >= 0

max_tpot_p95_ms

number

Required range: x >= 0

max_ttft_p95_ms

number

Required range: x >= 0

min_goodput

number

Required range: x >= 0

min_output_tokens_second

number

Required range: x >= 0

objective

enum<string>

default:latency

Available options:

interactive,

latency,

throughput,

cost-efficiency

region

string

Maximum string length: 128

workload_digest

string

Pattern: ^[a-f0-9]{64}$

workload_profile

enum<string>

Available options:

balanced,

buffered,

interactive,

long-context,

long-generation,

overload,

public-decode-heavy,

public-interactive,

public-long-prefill,

throughput

Response

Created

The response is of type object.

Compare persisted serving evidence

curl --request POST \
  --url http://127.0.0.1:18000/api/v1/lab/evaluations \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "model_identity": "<string>",
  "max_error_rate": 0.5,
  "max_gpu_count": 512,
  "max_hourly_cost": 1,
  "max_tpot_p95_ms": 1,
  "max_ttft_p95_ms": 1,
  "min_goodput": 1,
  "min_output_tokens_second": 1,
  "objective": "latency",
  "region": "<string>",
  "workload_digest": "<string>"
}
'

import requests

url = "http://127.0.0.1:18000/api/v1/lab/evaluations"

payload = {
    "model_identity": "<string>",
    "max_error_rate": 0.5,
    "max_gpu_count": 512,
    "max_hourly_cost": 1,
    "max_tpot_p95_ms": 1,
    "max_ttft_p95_ms": 1,
    "min_goodput": 1,
    "min_output_tokens_second": 1,
    "objective": "latency",
    "region": "<string>",
    "workload_digest": "<string>"
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    model_identity: '<string>',
    max_error_rate: 0.5,
    max_gpu_count: 512,
    max_hourly_cost: 1,
    max_tpot_p95_ms: 1,
    max_ttft_p95_ms: 1,
    min_goodput: 1,
    min_output_tokens_second: 1,
    objective: 'latency',
    region: '<string>',
    workload_digest: '<string>'
  })
};

fetch('http://127.0.0.1:18000/api/v1/lab/evaluations', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "http://127.0.0.1:18000/api/v1/lab/evaluations"

	payload := strings.NewReader("{\n  \"model_identity\": \"<string>\",\n  \"max_error_rate\": 0.5,\n  \"max_gpu_count\": 512,\n  \"max_hourly_cost\": 1,\n  \"max_tpot_p95_ms\": 1,\n  \"max_ttft_p95_ms\": 1,\n  \"min_goodput\": 1,\n  \"min_output_tokens_second\": 1,\n  \"objective\": \"latency\",\n  \"region\": \"<string>\",\n  \"workload_digest\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

{}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "category": "<string>",
    "remediation": "<string>",
    "request_id": "<string>",
    "retryable": true
  }
}