Batch Encoding

Overview

Batch encoding is the process of submitting multiple transcoding jobs at once — for example, encoding an entire video library, processing user uploads in bulk, or generating multiple renditions of a content catalog. Transcodely does not have a dedicated “batch” endpoint. Instead, you create individual jobs in parallel and use idempotency keys to make the process safe and repeatable.

This approach gives you full control over per-job configuration, error handling, and retry logic.

Creating Jobs in Parallel

Submit multiple jobs concurrently by making parallel API calls. Each job is independent and processes on its own worker.

Sequential

The simplest approach: loop over your files in a shell script and create one job per file. Each request carries a per-file idempotency_key, so re-running the loop after a failure never double-submits a job that already exists:

for video in source1.mp4 source2.mp4 source3.mp4; do
  curl -X POST https://api.transcodely.com/transcodely.v1.JobService/Create 
    -H "Content-Type: application/json" 
    -H "Authorization: Bearer {{API_KEY}}" 
    -H "X-Organization-ID: {{ORG_ID}}" 
    -d "{
      "input_origin_id": "ori_input12345",
      "input_path": "uploads/${video}",
      "output_origin_id": "ori_output6789",
      "idempotency_key": "batch_2026-02-28_${video}",
      "outputs": [
        {
          "type": "mp4",
          "video": [{ "codec": "h264", "resolution": "1080p", "quality": "standard" }]
        }
      ]
    }"
done

Parallel

For real throughput, fan out the creates so they run at the same time — Promise.allSettled in TypeScript, asyncio.gather over to_thread in Python, one goroutine per file in Go. Each create still uses a per-file idempotency_key, and the code collects every settled result, then splits them into successes and failures so one bad input never sinks the whole batch:

import {
  Transcodely,
  OutputFormat,
  VideoCodec,
  Resolution,
  QualityTier,
} from "@transcodely/sdk";

const client = new Transcodely({ apiKey: process.env.TRANSCODELY_API_KEY! });

const videos = [
  "uploads/episode-01.mp4",
  "uploads/episode-02.mp4",
  "uploads/episode-03.mp4",
  "uploads/episode-04.mp4",
  "uploads/episode-05.mp4",
];

// Create all jobs in parallel
const results = await Promise.allSettled(
  videos.map((inputPath) =>
    client.jobs.create({
      inputOriginId: "ori_input12345",
      inputPath,
      outputOriginId: "ori_output6789",
      idempotencyKey: `batch_2026-02-28_${inputPath}`,
      outputs: [
        {
          type: OutputFormat.MP4,
          video: [{ codec: VideoCodec.H264, resolution: Resolution.RESOLUTION_1080P, quality: QualityTier.STANDARD }],
        },
        {
          type: OutputFormat.HLS,
          video: [
            { codec: VideoCodec.H264, resolution: Resolution.RESOLUTION_1080P, quality: QualityTier.STANDARD },
            { codec: VideoCodec.H264, resolution: Resolution.RESOLUTION_720P, quality: QualityTier.STANDARD },
            { codec: VideoCodec.H264, resolution: Resolution.RESOLUTION_480P, quality: QualityTier.ECONOMY },
          ],
        },
      ],
    })
  )
);

// Separate successes and failures
const created = results
  .filter((r) => r.status === "fulfilled")
  .map((r) => r.value);

const failed = results
  .filter((r) => r.status === "rejected")
  .map((r, i) => ({ video: videos[i], error: r.reason }));

console.warn(`Created ${created.length} jobs, ${failed.length} failures`);

import asyncio

videos = [
    "uploads/episode-01.mp4",
    "uploads/episode-02.mp4",
    "uploads/episode-03.mp4",
]

async def create_job(input_path: str):
    # client.jobs.create is synchronous — run it off the event loop so the
    # creates still fan out concurrently.
    return await asyncio.to_thread(
        client.jobs.create,
        input_origin_id="ori_input12345",
        input_path=input_path,
        output_origin_id="ori_output6789",
        idempotency_key=f"batch_2026-02-28_{input_path}",
        outputs=[{
            "type": "mp4",
            "video": [{"codec": "h264", "resolution": "1080p", "quality": "standard"}],
        }],
    )

async def main():
    tasks = [create_job(video) for video in videos]
    results = await asyncio.gather(*tasks, return_exceptions=True)

    for video, result in zip(videos, results):
        if isinstance(result, Exception):
            print(f"Failed: {video} - {result}")
        else:
            print(f"Created: {result.id} for {video}")

asyncio.run(main())

package main

import (
	"context"
	"log"
	"os"
	"sync"

	"github.com/transcodely/transcodely-go"
	"google.golang.org/protobuf/proto"
)

func main() {
	client, err := transcodely.New(os.Getenv("TRANSCODELY_API_KEY"))
	if err != nil {
		log.Fatal(err)
	}

	videos := []string{
		"uploads/episode-01.mp4",
		"uploads/episode-02.mp4",
		"uploads/episode-03.mp4",
		"uploads/episode-04.mp4",
		"uploads/episode-05.mp4",
	}

	type result struct {
		video string
		job   *transcodely.Job
		err   error
	}

	results := make([]result, len(videos))
	var wg sync.WaitGroup

	// Launch one goroutine per video; each creates its job in parallel.
	for i, inputPath := range videos {
		wg.Add(1)
		go func(i int, inputPath string) {
			defer wg.Done()
			job, err := client.Jobs.Create(context.Background(), &transcodely.JobCreateParams{
				InputOriginId:  proto.String("ori_input12345"),
				InputPath:      proto.String(inputPath),
				OutputOriginId: proto.String("ori_output6789"),
				IdempotencyKey: proto.String("batch_2026-02-28_" + inputPath),
				Outputs: []*transcodely.OutputSpec{
					{
						Type: transcodely.OutputFormatMP4,
						Video: []*transcodely.VideoVariant{
							{Codec: transcodely.VideoCodecH264, Resolution: transcodely.Resolution1080P, Quality: transcodely.QualityTierStandard},
						},
					},
					{
						Type: transcodely.OutputFormatHLS,
						Video: []*transcodely.VideoVariant{
							{Codec: transcodely.VideoCodecH264, Resolution: transcodely.Resolution1080P, Quality: transcodely.QualityTierStandard},
							{Codec: transcodely.VideoCodecH264, Resolution: transcodely.Resolution720P, Quality: transcodely.QualityTierStandard},
							{Codec: transcodely.VideoCodecH264, Resolution: transcodely.Resolution480P, Quality: transcodely.QualityTierEconomy},
						},
					},
				},
			})
			results[i] = result{video: inputPath, job: job, err: err}
		}(i, inputPath)
	}

	wg.Wait()

	var created, failed int
	for _, r := range results {
		if r.err != nil {
			failed++
			log.Printf("Failed: %s - %v", r.video, r.err)
			continue
		}
		created++
		log.Printf("Created: %s for %s", r.job.GetId(), r.video)
	}
	log.Printf("Created %d jobs, %d failures", created, failed)
}

Idempotency Keys

Idempotency keys are critical for batch processing. They ensure that if a request is retried (due to network errors, timeouts, or application restarts), the same job is returned instead of creating a duplicate.

How Idempotency Works

Include an idempotency_key in your create request
If a job with that key already exists, the existing job is returned (no duplicate is created)
The key is scoped to your app — different apps can use the same key without conflict
Keys are permanent — they never expire

Key Design Patterns

Choose idempotency keys that uniquely identify the intent:

Pattern	Example	Use Case
Source file path	`encode_uploads/video.mp4`	One encoding per source file
Batch + file	`batch_2026-02-28_episode-01.mp4`	Daily batch runs
User + upload	`user_usr_abc123_upload_12345`	Per-user upload processing
Content ID	`content_cid_789_v2`	Versioned content library

curl -X POST https://api.transcodely.com/transcodely.v1.JobService/Create 
  -H "Content-Type: application/json" 
  -H "Authorization: Bearer {{API_KEY}}" 
  -H "X-Organization-ID: {{ORG_ID}}" 
  -d '{
    "input_origin_id": "ori_input12345",
    "input_path": "uploads/episode-01.mp4",
    "output_origin_id": "ori_output6789",
    "idempotency_key": "batch_2026-02-28_episode-01.mp4",
    "outputs": [
      {
        "type": "mp4",
        "video": [{ "codec": "h264", "resolution": "1080p", "quality": "standard" }]
      }
    ]
  }'

const job = await client.jobs.create({
  inputOriginId: "ori_input12345",
  inputPath: "uploads/episode-01.mp4",
  outputOriginId: "ori_output6789",
  idempotencyKey: "batch_2026-02-28_episode-01.mp4",
  outputs: [
    {
      type: OutputFormat.MP4,
      video: [
        {
          codec: VideoCodec.H264,
          resolution: Resolution.RESOLUTION_1080P,
          quality: QualityTier.STANDARD,
        },
      ],
    },
  ],
});

job = client.jobs.create(
    input_origin_id="ori_input12345",
    input_path="uploads/episode-01.mp4",
    output_origin_id="ori_output6789",
    idempotency_key="batch_2026-02-28_episode-01.mp4",
    outputs=[{
        "type": "mp4",
        "video": [{"codec": "h264", "resolution": "1080p", "quality": "standard"}],
    }],
)

job, err := client.Jobs.Create(ctx, &transcodely.JobCreateParams{
    InputOriginId:  proto.String("ori_input12345"),
    InputPath:      proto.String("uploads/episode-01.mp4"),
    OutputOriginId: proto.String("ori_output6789"),
    IdempotencyKey: proto.String("batch_2026-02-28_episode-01.mp4"),
    Outputs: []*transcodely.OutputSpec{{
        Type: transcodely.OutputFormatMP4,
        Video: []*transcodely.VideoVariant{{
            Codec:      transcodely.VideoCodecH264,
            Resolution: transcodely.Resolution1080P,
            Quality:    transcodely.QualityTierStandard,
        }},
    }},
})

If you run this request again with the same idempotency_key, you get back the existing job without creating a new one. This makes your entire batch script safe to re-run.

Rate Limiting

When submitting large batches, avoid firing thousands of requests at once. Unbounded concurrency can exhaust your own network resources and trigger timeouts and connection errors, so cap how many job creates you have in flight at any given time.

For large batches (hundreds or thousands of videos), add concurrency control:

// Process in batches of 20 concurrent requests
const CONCURRENCY = 20;

async function processBatch(videos: string[]) {
  const results = [];

  for (let i = 0; i < videos.length; i += CONCURRENCY) {
    const batch = videos.slice(i, i + CONCURRENCY);
    const batchResults = await Promise.allSettled(
      batch.map((video) => createJob(video))
    );
    results.push(...batchResults);

    // Brief pause between batches
    if (i + CONCURRENCY < videos.length) {
      await new Promise((resolve) => setTimeout(resolve, 100));
    }
  }

  return results;
}

import asyncio

# Cap in-flight requests with a semaphore instead of fixed-size chunks.
CONCURRENCY = 20

async def process_batch(videos: list[str]):
    sem = asyncio.Semaphore(CONCURRENCY)

    async def run(video: str):
        async with sem:
            # create_job wraps client.jobs.create with a per-video idempotency key
            return await create_job(video)

    return await asyncio.gather(
        *(run(video) for video in videos),
        return_exceptions=True,
    )

// Cap in-flight requests with a buffered channel as a semaphore.
const concurrency = 20

func processBatch(ctx context.Context, videos []string) []*transcodely.Job {
	sem := make(chan struct{}, concurrency)
	jobs := make([]*transcodely.Job, len(videos))
	var wg sync.WaitGroup

	for i, video := range videos {
		wg.Add(1)
		go func(i int, video string) {
			defer wg.Done()
			sem <- struct{}{}        // acquire a slot
			defer func() { <-sem }() // release it
			// createJob wraps client.Jobs.Create with a per-video idempotency key
			jobs[i], _ = createJob(ctx, video)
		}(i, video)
	}

	wg.Wait()
	return jobs
}

Monitoring Batch Progress

Polling All Jobs

With the job IDs in hand, poll them together on a fixed interval. Each tick fetches every job, tallies how many are completed, failed, and still in progress, and stops the loop once nothing remains in flight:

import { JobStatus } from "@transcodely/sdk";

async function monitorBatch(jobIds: string[]) {
  const interval = setInterval(async () => {
    const jobs = await Promise.all(jobIds.map((id) => client.jobs.get(id)));

    const completed = jobs.filter((j) => j.status === JobStatus.COMPLETED).length;
    const failed = jobs.filter((j) => j.status === JobStatus.FAILED).length;
    const processing = jobs.filter((j) =>
      [JobStatus.PROCESSING, JobStatus.PENDING, JobStatus.PROBING].includes(j.status)
    ).length;

    console.warn(`Progress: ${completed} done, ${failed} failed, ${processing} in progress`);

    if (processing === 0) {
      clearInterval(interval);
      console.warn("Batch complete!");
    }
  }, 10000); // Check every 10 seconds
}

import time

from transcodely.v1 import job_pb2

IN_PROGRESS = {
    job_pb2.JOB_STATUS_PROCESSING,
    job_pb2.JOB_STATUS_PENDING,
    job_pb2.JOB_STATUS_PROBING,
}

def monitor_batch(job_ids: list[str]) -> None:
    while True:
        jobs = [client.jobs.get(job_id) for job_id in job_ids]

        completed = sum(1 for j in jobs if j.status == job_pb2.JOB_STATUS_COMPLETED)
        failed = sum(1 for j in jobs if j.status == job_pb2.JOB_STATUS_FAILED)
        processing = sum(1 for j in jobs if j.status in IN_PROGRESS)

        print(f"Progress: {completed} done, {failed} failed, {processing} in progress")

        if processing == 0:
            print("Batch complete!")
            return
        time.sleep(10)  # Check every 10 seconds

func monitorBatch(ctx context.Context, jobIDs []string) error {
	ticker := time.NewTicker(10 * time.Second) // Check every 10 seconds
	defer ticker.Stop()

	for {
		var completed, failed, processing int
		for _, id := range jobIDs {
			job, err := client.Jobs.Get(ctx, id)
			if err != nil {
				return err
			}
			switch job.GetStatus() {
			case transcodely.JobStatusCompleted:
				completed++
			case transcodely.JobStatusFailed:
				failed++
			case transcodely.JobStatusProcessing,
				transcodely.JobStatusPending,
				transcodely.JobStatusProbing:
				processing++
			}
		}

		log.Printf("Progress: %d done, %d failed, %d in progress", completed, failed, processing)

		if processing == 0 {
			log.Print("Batch complete!")
			return nil
		}
		<-ticker.C
	}
}

Using Webhooks

For production systems, prefer webhooks over polling — Transcodely pushes an event the moment each job finishes, so you do no busy-waiting. Tag every job with a batch_id (and any other identifiers) in metadata so the handler can group deliveries back to the batch they belong to:

{
  "input_origin_id": "ori_input12345",
  "input_path": "uploads/episode-01.mp4",
  "output_origin_id": "ori_output6789",
  "metadata": {
    "batch_id": "batch_2026-02-28",
    "content_id": "episode-01",
    "user_id": "usr_abc123"
  },
  "outputs": [
    {
      "type": "mp4",
      "video": [{ "codec": "h264", "resolution": "1080p", "quality": "standard" }]
    }
  ]
}

In your webhook handler, track batch completion. The handler receives the parsed event; job here is the resource snapshot from event.data (the data field of the webhook envelope), not the raw request body:

async function handleJobCompleted(job: any) {
  const batchId = job.metadata.batch_id;

  // Record completion
  await db.batchJobs.update({
    where: { jobId: job.id },
    data: { status: "completed", completedAt: new Date() },
  });

  // Check if batch is complete
  const remaining = await db.batchJobs.count({
    where: { batchId, status: "pending" },
  });

  if (remaining === 0) {
    await notifyBatchComplete(batchId);
  }
}

def handle_job_completed(job):
    batch_id = job.metadata["batch_id"]

    # Record completion
    db.batch_jobs.update(job_id=job.id, status="completed", completed_at=now())

    # Check if batch is complete
    remaining = db.batch_jobs.count(batch_id=batch_id, status="pending")
    if remaining == 0:
        notify_batch_complete(batch_id)

func handleJobCompleted(job *transcodely.Job) error {
	batchID := job.GetMetadata()["batch_id"]

	// Record completion
	if err := db.BatchJobs.Update(job.GetId(), "completed", time.Now()); err != nil {
		return err
	}

	// Check if batch is complete
	remaining, err := db.BatchJobs.Count(batchID, "pending")
	if err != nil {
		return err
	}
	if remaining == 0 {
		return notifyBatchComplete(batchID)
	}
	return nil
}

Handling Partial Failures

In any batch, some creates may fail while others succeed — a bad input path, a transient timeout. The pattern below separates the rejected results from the fulfilled ones and retries only the failures:

async function handleBatchResults(results: PromiseSettledResult<any>[]) {
  const failures = results
    .map((r, i) => ({ result: r, index: i }))
    .filter((r) => r.result.status === "rejected");

  if (failures.length === 0) {
    console.warn("All jobs created successfully");
    return;
  }

  console.warn(`${failures.length} jobs failed to create`);

  // Retry failed jobs
  for (const failure of failures) {
    console.warn(`Retrying job ${failure.index}:`, failure.result.reason);
    try {
      // Safe to retry because we use idempotency keys
      await createJob(videos[failure.index]);
    } catch (err) {
      console.error(`Retry failed for ${failure.index}:`, err);
    }
  }
}

# results comes from asyncio.gather(..., return_exceptions=True):
# successful entries are jobs, failed entries are exceptions.
async def handle_batch_results(videos: list[str], results: list) -> None:
    failures = [(i, r) for i, r in enumerate(results) if isinstance(r, Exception)]

    if not failures:
        print("All jobs created successfully")
        return

    print(f"{len(failures)} jobs failed to create")

    for index, reason in failures:
        print(f"Retrying job {index}: {reason}")
        try:
            # Safe to retry because we use idempotency keys
            await create_job(videos[index])
        except Exception as err:
            print(f"Retry failed for {index}: {err}")

// result mirrors the struct returned by the parallel create above
// (video string, job *transcodely.Job, err error).
func handleBatchResults(ctx context.Context, results []result) {
	var failures []result
	for _, r := range results {
		if r.err != nil {
			failures = append(failures, r)
		}
	}

	if len(failures) == 0 {
		log.Print("All jobs created successfully")
		return
	}

	log.Printf("%d jobs failed to create", len(failures))

	for _, f := range failures {
		log.Printf("Retrying %s: %v", f.video, f.err)
		// Safe to retry because we use idempotency keys
		if _, err := createJob(ctx, f.video); err != nil {
			log.Printf("Retry failed for %s: %v", f.video, err)
		}
	}
}

Because idempotency keys are included, retrying a job that actually succeeded (e.g., the original request timed out but the job was created) will simply return the existing job.

Best Practices

Practice	Rationale
Always use idempotency keys	Makes batch scripts safe to re-run after failures
Limit concurrency	Respect rate limits and avoid overwhelming your system
Use metadata for tracking	Tag jobs with `batch_id`, `content_id` for easy filtering
Prefer webhooks over polling	More efficient for monitoring large batches
Handle partial failures	Not all jobs in a batch will necessarily succeed
Use quality tier `economy` for non-urgent work	Lowers cost via the 0.75x quality multiplier — priority only affects worker scheduling, not cost
Log all job IDs	Essential for debugging and support
Use consistent key naming	Makes it easy to identify and deduplicate across runs

#Batch Encoding

#Overview

#Creating Jobs in Parallel

#Sequential

#Parallel

#Idempotency Keys

#How Idempotency Works

#Key Design Patterns

#Rate Limiting

#Monitoring Batch Progress

#Polling All Jobs

#Using Webhooks

#Handling Partial Failures

#Best Practices