Batches - Portkey Docs

With Portkey, you can perform Azure OpenAI Batch Inference operations. This is the most efficient way to

Test your data with different foundation models
Perform A/B testing with different foundation models
Perform batch inference with different foundation models

Portkey supports two modes on Azure OpenAI:

Provider Batch API (cheapest, completion_window: 24h, 48h, etc.)
Portkey Batch API (fast, provider-agnostic, completion_window: immediate)

Using Azure OpenAI Batch API through Portkey

Create Batch Job

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

start_batch_response = portkey.batches.create(
  input_file_id="file_id", # file id of the input file
  endpoint="endpoint", # ex: /v1/chat/completions
  completion_window="completion_window", # ex: 24h
  metadata={} # metadata for the batch
)

print(start_batch_response)

import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const startBatch = async () => {
  const startBatchResponse = await portkey.batches.create({
    input_file_id: "file_id", // file id of the input file
    endpoint: "endpoint", // ex: /v1/chat/completions
    completion_window: "completion_window", // ex: 24h
    metadata: {} // metadata for the batch
  });

  console.log(startBatchResponse);
}

await startBatch();

curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider' \
--header 'Content-Type: application/json' \
--data '{
    "input_file_id": "<file_id>",
    "endpoint": "<endpoint>",
    "completion_window": "<completion_window>",
    "metadata": {}
}'

import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const startBatch = async () => {
  const startBatchResponse = await openai.batches.create({
    input_file_id: "file_id", // file id of the input file
    endpoint: "endpoint", // ex: /v1/chat/completions
    completion_window: "completion_window", // ex: 24h
    metadata: {} // metadata for the batch
  });

  console.log(startBatchResponse);
}

await startBatch();

from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

start_batch_response = openai.batches.create(
  input_file_id="file_id", # file id of the input file
  endpoint="endpoint", # ex: /v1/chat/completions
  completion_window="completion_window", # ex: 24h
  metadata={} # metadata for the batch
)

print(start_batch_response)

Create Batch Job with Blob Storage

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

start_batch_response = portkey.batches.create(
  endpoint="endpoint",  # ex: /v1/chat/completions
  completion_window="completion_window",  # ex: 24h
  metadata={},  # metadata for the batch
  input_blob="<blob_url>",
  output_folder={
    "url": "<output_blob_folder>" # both error file and output file will be saved in this folder
  }
)

print(start_batch_response)

import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const startBatch = async () => {
  const startBatchResponse = await portkey.batches.create({
    endpoint: "endpoint", // ex: /v1/chat/completions
    completion_window: "completion_window", // ex: 24h
    metadata: {}, // metadata for the batch
    input_blob: "<blob_url>",
    output_folder: {
      url: "<output_blob_folder>" // both error file and output file will be saved in this folder
    }
  });

  console.log(startBatchResponse);
}

await startBatch();

curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider' \
--header 'Content-Type: application/json' \
--data '{
    "endpoint": "<endpoint>",
    "completion_window": "<completion_window>",
    "metadata": {},
    "input_blob": "<blob_url>",
    "output_folder": {
      "url": "<output_blob_folder>" # both error file and output file will be saved in this folder
    }
}'

import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const startBatch = async () => {
  const startBatchResponse = await openai.batches.create({
    endpoint: "endpoint", // ex: /v1/chat/completions
    completion_window: "completion_window", // ex: 24h
    metadata: {}, // metadata for the batch
    extra_body: {
      input_blob: "<blob_url>",
      output_folder: {
        url: "<output_blob_folder>" // both error file and output file will be saved in this folder
      }
    }
  });

  console.log(startBatchResponse);
}

await startBatch();

from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

start_batch_response = openai.batches.create(
  endpoint="endpoint",  # ex: /v1/chat/completions
  completion_window="completion_window",  # ex: 24h
  metadata={},  # metadata for the batch
  extra_body={
    "input_blob": "<blob_url>",
    "output_folder": {
      "url": "<output_blob_folder>"  # both error file and output file will be saved in this folder
    }
  }
)

print(start_batch_response)

List Batch Jobs

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

batches = portkey.batches.list()

print(batches)

import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const listBatches = async () => {
  const batches = await portkey.batches.list();

  console.log(batches);
}

await listBatches();

curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'

import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const listBatches = async () => {
  const batches = await openai.batches.list();

  console.log(batches);
}

await listBatches();

from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

batches = openai.batches.list()

print(batches)

Get Batch Job Details

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

batch = portkey.batches.retrieve(batch_id="batch_id")

print(batch)

import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const getBatch = async () => {
  const batch = await portkey.batches.retrieve(batch_id="batch_id");

  console.log(batch);
}

await getBatch();

curl --location 'https://api.portkey.ai/v1/batches/<batch_id>' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'

import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const getBatch = async () => {
  const batch = await openai.batches.retrieve(batch_id="batch_id");

  console.log(batch);
}

await getBatch();

from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

batch = openai.batches.retrieve(batch_id="batch_id")

print(batch)

Get Batch Output

curl --location 'https://api.portkey.ai/v1/batches/<batch_id>/output' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'

List Batch Jobs

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

batches = portkey.batches.list()

print(batches)

import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const listBatchingJobs = async () => {
  const batching_jobs = await portkey.batches.list();

  console.log(batching_jobs);
}

await listBatchingJobs();

curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'

import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const listBatchingJobs = async () => {
  const batching_jobs = await openai.batches.list();

  console.log(batching_jobs);
}

await listBatchingJobs();

from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

batching_jobs = openai.batches.list()

print(batching_jobs)

Cancel Batch Job

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

cancel_batch_response = portkey.batches.cancel(batch_id="batch_id")

print(cancel_batch_response)

import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const cancelBatch = async () => {
  const cancel_batch_response = await portkey.batches.cancel(batch_id="batch_id");

  console.log(cancel_batch_response);
}

await cancelBatch();

curl --request POST --location 'https://api.portkey.ai/v1/batches/<batch_id>/cancel' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'

import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const cancelBatch = async () => {
  const cancel_batch_response = await openai.batches.cancel(batch_id="batch_id");

  console.log(cancel_batch_response);
}

await cancelBatch();

from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

cancel_batch_response = openai.batches.cancel(batch_id="batch_id")

print(cancel_batch_response)

​Using Azure OpenAI Batch API through Portkey

​Create Batch Job

​Create Batch Job with Blob Storage

​List Batch Jobs

​Get Batch Job Details

​Get Batch Output

​List Batch Jobs

​Cancel Batch Job

Using Azure OpenAI Batch API through Portkey

Create Batch Job

Create Batch Job with Blob Storage

List Batch Jobs

Get Batch Job Details

Get Batch Output

List Batch Jobs

Cancel Batch Job