Run and persist an AIPerf benchmark - InferCrane

POST

/

api

/

v1

/

deployments

/

{name}

/

benchmarks

Run and persist an AIPerf benchmark

curl --request POST \
  --url http://127.0.0.1:18000/api/v1/deployments/{name}/benchmarks \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "concurrency": 2,
  "requests": 50000,
  "input_tokens": 128,
  "latency_slo_ms": 1,
  "output_tokens": 32,
  "profile_version": "benchmark-profile-v1",
  "random_seed": 17,
  "request_rate": 123,
  "revision": "<string>",
  "run_cooldown_seconds": 0,
  "runs": 1,
  "streaming": true,
  "tpot_slo_ms": 1,
  "ttft_slo_ms": 1,
  "warmup_requests": 0
}
'

import requests

url = "http://127.0.0.1:18000/api/v1/deployments/{name}/benchmarks"

payload = {
    "concurrency": 2,
    "requests": 50000,
    "input_tokens": 128,
    "latency_slo_ms": 1,
    "output_tokens": 32,
    "profile_version": "benchmark-profile-v1",
    "random_seed": 17,
    "request_rate": 123,
    "revision": "<string>",
    "run_cooldown_seconds": 0,
    "runs": 1,
    "streaming": True,
    "tpot_slo_ms": 1,
    "ttft_slo_ms": 1,
    "warmup_requests": 0
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    concurrency: 2,
    requests: 50000,
    input_tokens: 128,
    latency_slo_ms: 1,
    output_tokens: 32,
    profile_version: 'benchmark-profile-v1',
    random_seed: 17,
    request_rate: 123,
    revision: '<string>',
    run_cooldown_seconds: 0,
    runs: 1,
    streaming: true,
    tpot_slo_ms: 1,
    ttft_slo_ms: 1,
    warmup_requests: 0
  })
};

fetch('http://127.0.0.1:18000/api/v1/deployments/{name}/benchmarks', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "http://127.0.0.1:18000/api/v1/deployments/{name}/benchmarks"

	payload := strings.NewReader("{\n  \"concurrency\": 2,\n  \"requests\": 50000,\n  \"input_tokens\": 128,\n  \"latency_slo_ms\": 1,\n  \"output_tokens\": 32,\n  \"profile_version\": \"benchmark-profile-v1\",\n  \"random_seed\": 17,\n  \"request_rate\": 123,\n  \"revision\": \"<string>\",\n  \"run_cooldown_seconds\": 0,\n  \"runs\": 1,\n  \"streaming\": true,\n  \"tpot_slo_ms\": 1,\n  \"ttft_slo_ms\": 1,\n  \"warmup_requests\": 0\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

{}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "category": "<string>",
    "remediation": "<string>",
    "request_id": "<string>",
    "retryable": true
  }
}

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Path Parameters

name

string

required

Minimum string length: 1

Body

application/json

concurrency

integer

required

Required range: x >= 1

requests

integer

required

Required range: 1 <= x <= 100000

arrival_pattern

enum<string>

Available options:

constant,

poisson,

gamma

input_tokens

integer

default:128

Required range: 1 <= x <= 1000000

latency_slo_ms

number

Required range: x >= 0

output_tokens

integer

default:32

Required range: 1 <= x <= 1000000

profile

enum<string>

Available options:

balanced,

buffered,

interactive,

long-context,

long-generation,

overload,

public-decode-heavy,

public-interactive,

public-long-prefill,

throughput

profile_version

enum<string>

Available options:

benchmark-profile-v1

random_seed

integer

default:17

request_rate

number

revision

string

run_cooldown_seconds

integer

default:0

Required range: 0 <= x <= 3600

runs

integer

default:1

Required range: 1 <= x <= 10

streaming

boolean

default:true

tpot_slo_ms

number

Required range: x >= 0

ttft_slo_ms

number

Required range: x >= 0

warmup_requests

integer

default:0

Required range: 0 <= x <= 100000

Response

Created

The response is of type object.

Run and persist an AIPerf benchmark

curl --request POST \
  --url http://127.0.0.1:18000/api/v1/deployments/{name}/benchmarks \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "concurrency": 2,
  "requests": 50000,
  "input_tokens": 128,
  "latency_slo_ms": 1,
  "output_tokens": 32,
  "profile_version": "benchmark-profile-v1",
  "random_seed": 17,
  "request_rate": 123,
  "revision": "<string>",
  "run_cooldown_seconds": 0,
  "runs": 1,
  "streaming": true,
  "tpot_slo_ms": 1,
  "ttft_slo_ms": 1,
  "warmup_requests": 0
}
'

import requests

url = "http://127.0.0.1:18000/api/v1/deployments/{name}/benchmarks"

payload = {
    "concurrency": 2,
    "requests": 50000,
    "input_tokens": 128,
    "latency_slo_ms": 1,
    "output_tokens": 32,
    "profile_version": "benchmark-profile-v1",
    "random_seed": 17,
    "request_rate": 123,
    "revision": "<string>",
    "run_cooldown_seconds": 0,
    "runs": 1,
    "streaming": True,
    "tpot_slo_ms": 1,
    "ttft_slo_ms": 1,
    "warmup_requests": 0
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    concurrency: 2,
    requests: 50000,
    input_tokens: 128,
    latency_slo_ms: 1,
    output_tokens: 32,
    profile_version: 'benchmark-profile-v1',
    random_seed: 17,
    request_rate: 123,
    revision: '<string>',
    run_cooldown_seconds: 0,
    runs: 1,
    streaming: true,
    tpot_slo_ms: 1,
    ttft_slo_ms: 1,
    warmup_requests: 0
  })
};

fetch('http://127.0.0.1:18000/api/v1/deployments/{name}/benchmarks', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "http://127.0.0.1:18000/api/v1/deployments/{name}/benchmarks"

	payload := strings.NewReader("{\n  \"concurrency\": 2,\n  \"requests\": 50000,\n  \"input_tokens\": 128,\n  \"latency_slo_ms\": 1,\n  \"output_tokens\": 32,\n  \"profile_version\": \"benchmark-profile-v1\",\n  \"random_seed\": 17,\n  \"request_rate\": 123,\n  \"revision\": \"<string>\",\n  \"run_cooldown_seconds\": 0,\n  \"runs\": 1,\n  \"streaming\": true,\n  \"tpot_slo_ms\": 1,\n  \"ttft_slo_ms\": 1,\n  \"warmup_requests\": 0\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

{}

{
  "error": {
    "code": "<string>",
    "message": "<string>",
    "category": "<string>",
    "remediation": "<string>",
    "request_id": "<string>",
    "retryable": true
  }
}