Read the newest content-free GPU profile capture of the active revision behind an endpoint
GET
/
api
/
v1
/
endpoints
/
{name}
/
gpu-profile
Read the newest content-free GPU profile capture of the active revision behind an endpoint
curl --request GET \
--url http://127.0.0.1:18000/api/v1/endpoints/{name}/gpu-profile \
--header 'Authorization: Bearer <token>'import requests
url = "http://127.0.0.1:18000/api/v1/endpoints/{name}/gpu-profile"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('http://127.0.0.1:18000/api/v1/endpoints/{name}/gpu-profile', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "http://127.0.0.1:18000/api/v1/endpoints/{name}/gpu-profile"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}{
"capture": {
"accelerator": "<string>",
"content_recorded": false,
"ended_at": "2023-11-07T05:31:56Z",
"evidence_class": "measured",
"gpus": 512,
"id": "<string>",
"started_at": "2023-11-07T05:31:56Z",
"tool": "<string>",
"tool_version": "<string>"
},
"deployment": "<string>",
"endpoint": "<string>",
"frames": [
{
"kind": "python",
"name": "<string>",
"file": "<string>",
"line": 2
}
],
"recorded_at": "2023-11-07T05:31:56Z",
"revision": 2,
"schema_version": "infercrane.gpu-profile/v1",
"stacks": [
{
"frames": [
1
],
"gpu_ns": 2
}
],
"timeline": {
"bucket_seconds": 1800,
"buckets": [
{
"gpu_seconds": 1,
"start": "2023-11-07T05:31:56Z"
}
]
},
"unavailable_layers": [
"python"
],
"analysis": {
"algorithm_version": "gpu-trace-analysis-v1",
"categories": [
{
"category": "attention",
"count": 1,
"kernel_ms": 1,
"share": 0.5
}
],
"findings": [
{
"comparison": ">=",
"observed": 123,
"rule": "<string>",
"sentence": "<string>",
"threshold": 123,
"unit": "<string>"
}
],
"hardware": {
"interval_ms": 1,
"series": [
{
"field": "<string>",
"unit": "<string>",
"values": [
1
]
}
],
"source": "dcgm-exporter"
},
"idle": {
"bubbles": {
"count": 1,
"max_us": 1,
"p50_us": 1,
"p95_us": 1,
"top": [
{
"cause": "host_wait",
"device": 1,
"duration_us": 1,
"start_ms": 1
}
]
},
"host_wait_share": 0.5,
"idle_ms": 1,
"launch_gap_share": 0.5,
"other_share": 0.5,
"reason": "<string>"
},
"launch": {
"graph_launches": 1,
"host_launch_share": 0.5,
"launch_delay_p50_us": 1,
"launch_delay_p95_us": 1,
"launches": 1,
"per_second": 1,
"reason": "<string>",
"short_kernel_share": 0.5
},
"overlap": {
"communication_ms": 1,
"overlapped_share": 0.5,
"reason": "<string>"
},
"phases": {
"decode": {
"gpu_ms": 1,
"share": 0.5,
"steps": 1
},
"mixed": {
"gpu_ms": 1,
"share": 0.5,
"steps": 1
},
"prefill": {
"gpu_ms": 1,
"share": 0.5,
"steps": 1
},
"source": "annotations",
"unknown": {
"gpu_ms": 1,
"share": 0.5,
"steps": 1
}
},
"roofline": {
"accelerator": "<string>",
"covered_share": 0.5,
"memory_bound_share": 0.5,
"peak": {
"compute_tflops": 1,
"memory_tbps": 1,
"precision": "<string>",
"source": "<string>"
},
"points": [
{
"achieved_tflops": 1,
"bound": "memory",
"bytes": 1,
"category": "attention",
"flops": 1,
"intensity": 1,
"kernel_ms": 1,
"label": "<string>"
}
],
"position": "memory_bound",
"reason": "<string>",
"ridge_flop_per_byte": 1
},
"steps": {
"count": 1,
"items": [
{
"duration_ms": 1,
"gpu_busy_ms": 1,
"kernels": 1,
"phase": "prefill",
"start_ms": 1
}
],
"marker": "<string>",
"p50_ms": 1,
"p95_ms": 1,
"reason": "<string>",
"source": "annotations"
},
"top_kernels": [
{
"category": "attention",
"count": 1,
"max_us": 1,
"mean_us": 1,
"name": "<string>",
"phase": "<string>",
"share": 0.5,
"total_ms": 1
}
],
"trace": {
"cpu_ops_recorded": true,
"devices": 2,
"engine": "vllm",
"engine_version": "<string>",
"format": "kineto",
"kernels": 2,
"launches_recorded": true,
"shapes_recorded": true,
"streams": 2,
"truncated": true,
"window_ms": 1
},
"utilization": {
"bucket_ms": 1,
"busy_series": [
0.5
],
"busy_share": 0.5,
"compute_share": 0.5,
"idle_share": 0.5,
"non_compute_share": 0.5
},
"verdict": {
"bottleneck": "host-launch",
"evidence": "<string>",
"sentence": "<string>"
}
},
"timeline_excerpt": {
"categories": [
"attention"
],
"duration_ms": 1000,
"events": [
[
1
]
],
"lanes": [
{
"device": 123,
"kind": "gpu",
"label": "<string>",
"stream": 123
}
],
"names": [
"<string>"
],
"start_ms": 1,
"steps": [
{
"duration_us": 1,
"phase": "prefill",
"start_us": 1
}
],
"truncated": true
}
}{
"error": {
"code": "<string>",
"message": "<string>",
"category": "<string>",
"remediation": "<string>",
"request_id": "<string>",
"retryable": true
}
}Authorizations
Response
Success
⌘I
Read the newest content-free GPU profile capture of the active revision behind an endpoint
curl --request GET \
--url http://127.0.0.1:18000/api/v1/endpoints/{name}/gpu-profile \
--header 'Authorization: Bearer <token>'import requests
url = "http://127.0.0.1:18000/api/v1/endpoints/{name}/gpu-profile"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('http://127.0.0.1:18000/api/v1/endpoints/{name}/gpu-profile', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "http://127.0.0.1:18000/api/v1/endpoints/{name}/gpu-profile"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}{
"capture": {
"accelerator": "<string>",
"content_recorded": false,
"ended_at": "2023-11-07T05:31:56Z",
"evidence_class": "measured",
"gpus": 512,
"id": "<string>",
"started_at": "2023-11-07T05:31:56Z",
"tool": "<string>",
"tool_version": "<string>"
},
"deployment": "<string>",
"endpoint": "<string>",
"frames": [
{
"kind": "python",
"name": "<string>",
"file": "<string>",
"line": 2
}
],
"recorded_at": "2023-11-07T05:31:56Z",
"revision": 2,
"schema_version": "infercrane.gpu-profile/v1",
"stacks": [
{
"frames": [
1
],
"gpu_ns": 2
}
],
"timeline": {
"bucket_seconds": 1800,
"buckets": [
{
"gpu_seconds": 1,
"start": "2023-11-07T05:31:56Z"
}
]
},
"unavailable_layers": [
"python"
],
"analysis": {
"algorithm_version": "gpu-trace-analysis-v1",
"categories": [
{
"category": "attention",
"count": 1,
"kernel_ms": 1,
"share": 0.5
}
],
"findings": [
{
"comparison": ">=",
"observed": 123,
"rule": "<string>",
"sentence": "<string>",
"threshold": 123,
"unit": "<string>"
}
],
"hardware": {
"interval_ms": 1,
"series": [
{
"field": "<string>",
"unit": "<string>",
"values": [
1
]
}
],
"source": "dcgm-exporter"
},
"idle": {
"bubbles": {
"count": 1,
"max_us": 1,
"p50_us": 1,
"p95_us": 1,
"top": [
{
"cause": "host_wait",
"device": 1,
"duration_us": 1,
"start_ms": 1
}
]
},
"host_wait_share": 0.5,
"idle_ms": 1,
"launch_gap_share": 0.5,
"other_share": 0.5,
"reason": "<string>"
},
"launch": {
"graph_launches": 1,
"host_launch_share": 0.5,
"launch_delay_p50_us": 1,
"launch_delay_p95_us": 1,
"launches": 1,
"per_second": 1,
"reason": "<string>",
"short_kernel_share": 0.5
},
"overlap": {
"communication_ms": 1,
"overlapped_share": 0.5,
"reason": "<string>"
},
"phases": {
"decode": {
"gpu_ms": 1,
"share": 0.5,
"steps": 1
},
"mixed": {
"gpu_ms": 1,
"share": 0.5,
"steps": 1
},
"prefill": {
"gpu_ms": 1,
"share": 0.5,
"steps": 1
},
"source": "annotations",
"unknown": {
"gpu_ms": 1,
"share": 0.5,
"steps": 1
}
},
"roofline": {
"accelerator": "<string>",
"covered_share": 0.5,
"memory_bound_share": 0.5,
"peak": {
"compute_tflops": 1,
"memory_tbps": 1,
"precision": "<string>",
"source": "<string>"
},
"points": [
{
"achieved_tflops": 1,
"bound": "memory",
"bytes": 1,
"category": "attention",
"flops": 1,
"intensity": 1,
"kernel_ms": 1,
"label": "<string>"
}
],
"position": "memory_bound",
"reason": "<string>",
"ridge_flop_per_byte": 1
},
"steps": {
"count": 1,
"items": [
{
"duration_ms": 1,
"gpu_busy_ms": 1,
"kernels": 1,
"phase": "prefill",
"start_ms": 1
}
],
"marker": "<string>",
"p50_ms": 1,
"p95_ms": 1,
"reason": "<string>",
"source": "annotations"
},
"top_kernels": [
{
"category": "attention",
"count": 1,
"max_us": 1,
"mean_us": 1,
"name": "<string>",
"phase": "<string>",
"share": 0.5,
"total_ms": 1
}
],
"trace": {
"cpu_ops_recorded": true,
"devices": 2,
"engine": "vllm",
"engine_version": "<string>",
"format": "kineto",
"kernels": 2,
"launches_recorded": true,
"shapes_recorded": true,
"streams": 2,
"truncated": true,
"window_ms": 1
},
"utilization": {
"bucket_ms": 1,
"busy_series": [
0.5
],
"busy_share": 0.5,
"compute_share": 0.5,
"idle_share": 0.5,
"non_compute_share": 0.5
},
"verdict": {
"bottleneck": "host-launch",
"evidence": "<string>",
"sentence": "<string>"
}
},
"timeline_excerpt": {
"categories": [
"attention"
],
"duration_ms": 1000,
"events": [
[
1
]
],
"lanes": [
{
"device": 123,
"kind": "gpu",
"label": "<string>",
"stream": 123
}
],
"names": [
"<string>"
],
"start_ms": 1,
"steps": [
{
"duration_us": 1,
"phase": "prefill",
"start_us": 1
}
],
"truncated": true
}
}{
"error": {
"code": "<string>",
"message": "<string>",
"category": "<string>",
"remediation": "<string>",
"request_id": "<string>",
"retryable": true
}
}