SDK Open Quick Integration

This page provides TypeScript snippets that can be moved into Web or Electron apps.

Open the Local Demo Site

When the runtime is installed, open http://127.0.0.1:17081 to access the local demo site. Enter an API key you have obtained and create a session to try the SDK capabilities authorized for that API key.

Callable methods and remaining quota are determined by the capabilities and quota_buckets returned for the session. Capabilities that are not authorized or have no remaining quota are not accepted.

Build Command Requests

export function buildCommandRequest(method: string, params: Record<string, unknown> = {}) {
  return {
    request_id: `${method.replace(/[^a-z0-9]+/gi, '-')}-${Date.now()}`,
    method,
    params,
    client: {
      source: 'your-app',
      protocol_version: '2.0.0',
      trace_id: `trc-${Date.now()}`
    }
  }
}

Connect, Authorize, and Route Responses

type Pending = {
  resolve: (value: any) => void
  reject: (reason: Error) => void
}

const command = new WebSocket('wss://sdk-runtime.localhost:18090')
const pending = new Map<string, Pending>()
let sessionToken = ''

function send<T>(method: string, params: Record<string, unknown> = {}): Promise<T> {
  const request = buildCommandRequest(method, params)
  command.send(JSON.stringify(request))
  return new Promise((resolve, reject) => {
    pending.set(request.request_id, { resolve, reject })
  })
}

command.addEventListener('message', (event) => {
  const payload = JSON.parse(event.data)
  if (payload.event) {
    console.log('runtime event', payload.event, payload.payload)
    return
  }
  const slot = pending.get(payload.request_id)
  if (!slot) return
  pending.delete(payload.request_id)
  payload.code === 0 ? slot.resolve(payload.data) : slot.reject(new Error(payload.message))
})

command.addEventListener('open', async () => {
  await send('system.ping')
  const auth = await send<{ session_token: string }>('auth.create_session', {
    token: 'sk-sq-v1-...'
  })
  sessionToken = auth.session_token
})

Open a Device and Start Video

const list = await send<{ devices: Array<{ device_id: string }> }>('device.list')
const deviceId = list.devices[0].device_id

await send('device.open', {
  device_id: deviceId,
  width: 1280,
  height: 720,
  fps: 15,
  pixel_format: 'mjpeg'
})

const stream = await send<{
  stream_id: string
  session_token: string
  width: number
  height: number
}>('video.start', {
  device_id: deviceId,
  width: 1280,
  height: 720,
  fps: 15,
  pixel_format: 'mjpeg'
})

Render MJPEG Video Frames

const video = new WebSocket(
  `wss://sdk-runtime.localhost:18091?session_token=${encodeURIComponent(stream.session_token)}&stream_id=${encodeURIComponent(stream.stream_id)}`
)
video.binaryType = 'arraybuffer'

let frameMeta: { width: number; height: number; pixel_format: string } | null = null
const ctx = canvas.getContext('2d')

async function drawJpegFrame(buffer: ArrayBuffer) {
  if (!ctx || !frameMeta || frameMeta.pixel_format !== 'mjpeg') return
  const blob = new Blob([buffer], { type: 'image/jpeg' })
  if ('createImageBitmap' in window) {
    const bitmap = await createImageBitmap(blob)
    canvas.width = frameMeta.width || bitmap.width
    canvas.height = frameMeta.height || bitmap.height
    ctx.drawImage(bitmap, 0, 0, canvas.width, canvas.height)
    bitmap.close()
    return
  }

  const url = URL.createObjectURL(blob)
  const image = new Image()
  image.onload = () => {
    canvas.width = frameMeta?.width || image.naturalWidth
    canvas.height = frameMeta?.height || image.naturalHeight
    ctx.drawImage(image, 0, 0, canvas.width, canvas.height)
    URL.revokeObjectURL(url)
  }
  image.onerror = () => URL.revokeObjectURL(url)
  image.src = url
}

video.addEventListener('message', async (event) => {
  if (typeof event.data === 'string') {
    const parsed = JSON.parse(event.data)
    if (parsed.event === 'stream.frame_meta') frameMeta = parsed.payload
    return
  }

  await drawJpegFrame(event.data)
})

Upload Images and Poll Tasks

async function upload(file: File) {
  const form = new FormData()
  form.set('file', file)
  const res = await fetch('https://sdk-runtime.localhost:18082/api/uploads/images', {
    method: 'POST',
    headers: { Authorization: `Bearer ${sessionToken}` },
    body: form
  })
  return res.json() as Promise<{ upload_id: string }>
}

async function pollTask(taskId: string) {
  for (;;) {
    const data = await send<{ task: { status: string; progress: number } }>('image.enhance_get', { task_id: taskId })
    if (['completed', 'failed', 'cancelled'].includes(data.task.status)) return data.task
    await new Promise((resolve) => setTimeout(resolve, 800))
  }
}