Batch Encoding — Transcodely Docs

Overview

Batch encoding is the process of submitting multiple transcoding jobs at once — for example, encoding an entire video library, processing user uploads in bulk, or generating multiple renditions of a content catalog. Transcodely does not have a dedicated “batch” endpoint. Instead, you create individual jobs in parallel and use idempotency keys to make the process safe and repeatable.

This approach gives you full control over per-job configuration, error handling, and retry logic.

Creating Jobs in Parallel

Submit multiple jobs concurrently by making parallel API calls. Each job is independent and processes on its own worker.

Sequential

Process files one at a time with cURL:

for video in source1.mp4 source2.mp4 source3.mp4; do
  curl -X POST https://api.transcodely.com/transcodely.v1.JobService/Create 
    -H "Content-Type: application/json" 
    -H "Authorization: Bearer {{API_KEY}}" 
    -H "X-Organization-ID: {{ORG_ID}}" 
    -d "{
      "input_origin_id": "ori_input12345",
      "input_path": "uploads/${video}",
      "output_origin_id": "ori_output6789",
      "idempotency_key": "batch_2026-02-28_${video}",
      "outputs": [
        {
          "type": "mp4",
          "video": [{ "codec": "h264", "resolution": "1080p", "quality": "standard" }]
        }
      ]
    }"
done

Parallel

Submit all jobs concurrently for faster throughput:

import { createOrgApiClient } from "$lib/api/client";
import { JobService } from "$lib/gen/transcodely/v1/job_connect";

const jobClient = createOrgApiClient(JobService);

const videos = [
  "uploads/episode-01.mp4",
  "uploads/episode-02.mp4",
  "uploads/episode-03.mp4",
  "uploads/episode-04.mp4",
  "uploads/episode-05.mp4",
];

// Create all jobs in parallel
const results = await Promise.allSettled(
  videos.map((inputPath) =>
    jobClient.create({
      inputOriginId: "ori_input12345",
      inputPath,
      outputOriginId: "ori_output6789",
      idempotencyKey: `batch_2026-02-28_${inputPath}`,
      outputs: [
        {
          type: "mp4",
          video: [{ codec: "h264", resolution: "1080p", quality: "standard" }],
        },
        {
          type: "hls",
          video: [
            { codec: "h264", resolution: "1080p", quality: "standard" },
            { codec: "h264", resolution: "720p", quality: "standard" },
            { codec: "h264", resolution: "480p", quality: "economy" },
          ],
        },
      ],
    })
  )
);

// Separate successes and failures
const created = results
  .filter((r) => r.status === "fulfilled")
  .map((r) => r.value.job);

const failed = results
  .filter((r) => r.status === "rejected")
  .map((r, i) => ({ video: videos[i], error: r.reason }));

console.warn(`Created ${created.length} jobs, ${failed.length} failures`);

import asyncio

videos = [
    "uploads/episode-01.mp4",
    "uploads/episode-02.mp4",
    "uploads/episode-03.mp4",
]

async def create_job(input_path: str):
    # client.jobs.create is synchronous — run it off the event loop so the
    # creates still fan out concurrently.
    return await asyncio.to_thread(
        client.jobs.create,
        input_origin_id="ori_input12345",
        input_path=input_path,
        output_origin_id="ori_output6789",
        idempotency_key=f"batch_2026-02-28_{input_path}",
        outputs=[{
            "type": "mp4",
            "video": [{"codec": "h264", "resolution": "1080p", "quality": "standard"}],
        }],
    )

async def main():
    tasks = [create_job(video) for video in videos]
    results = await asyncio.gather(*tasks, return_exceptions=True)

    for video, result in zip(videos, results):
        if isinstance(result, Exception):
            print(f"Failed: {video} - {result}")
        else:
            print(f"Created: {result.id} for {video}")

asyncio.run(main())

package main

import (
	"context"
	"log"
	"os"
	"sync"

	"github.com/transcodely/transcodely-go"
	"google.golang.org/protobuf/proto"
)

func main() {
	client, err := transcodely.New(os.Getenv("TRANSCODELY_API_KEY"))
	if err != nil {
		log.Fatal(err)
	}

	videos := []string{
		"uploads/episode-01.mp4",
		"uploads/episode-02.mp4",
		"uploads/episode-03.mp4",
		"uploads/episode-04.mp4",
		"uploads/episode-05.mp4",
	}

	type result struct {
		video string
		job   *transcodely.Job
		err   error
	}

	results := make([]result, len(videos))
	var wg sync.WaitGroup

	// Launch one goroutine per video; each creates its job in parallel.
	for i, inputPath := range videos {
		wg.Add(1)
		go func(i int, inputPath string) {
			defer wg.Done()
			job, err := client.Jobs.Create(context.Background(), &transcodely.JobCreateParams{
				InputOriginId:  proto.String("ori_input12345"),
				InputPath:      proto.String(inputPath),
				OutputOriginId: proto.String("ori_output6789"),
				IdempotencyKey: proto.String("batch_2026-02-28_" + inputPath),
				Outputs: []*transcodely.OutputSpec{
					{
						Type: transcodely.OutputFormatMP4,
						Video: []*transcodely.VideoVariant{
							{Codec: transcodely.VideoCodecH264, Resolution: transcodely.Resolution1080P, Quality: transcodely.QualityTierStandard},
						},
					},
					{
						Type: transcodely.OutputFormatHLS,
						Video: []*transcodely.VideoVariant{
							{Codec: transcodely.VideoCodecH264, Resolution: transcodely.Resolution1080P, Quality: transcodely.QualityTierStandard},
							{Codec: transcodely.VideoCodecH264, Resolution: transcodely.Resolution720P, Quality: transcodely.QualityTierStandard},
							{Codec: transcodely.VideoCodecH264, Resolution: transcodely.Resolution480P, Quality: transcodely.QualityTierEconomy},
						},
					},
				},
			})
			results[i] = result{video: inputPath, job: job, err: err}
		}(i, inputPath)
	}

	wg.Wait()

	var created, failed int
	for _, r := range results {
		if r.err != nil {
			failed++
			log.Printf("Failed: %s - %v", r.video, r.err)
			continue
		}
		created++
		log.Printf("Created: %s for %s", r.job.GetId(), r.video)
	}
	log.Printf("Created %d jobs, %d failures", created, failed)
}

Idempotency Keys

Idempotency keys are critical for batch processing. They ensure that if a request is retried (due to network errors, timeouts, or application restarts), the same job is returned instead of creating a duplicate.

How Idempotency Works

Include an idempotency_key in your create request
If a job with that key already exists, the existing job is returned (no duplicate is created)
The key is scoped to your app — different apps can use the same key without conflict
Keys are permanent — they never expire

Key Design Patterns

Choose idempotency keys that uniquely identify the intent:

Pattern	Example	Use Case
Source file path	`encode_uploads/video.mp4`	One encoding per source file
Batch + file	`batch_2026-02-28_episode-01.mp4`	Daily batch runs
User + upload	`user_usr_abc123_upload_12345`	Per-user upload processing
Content ID	`content_cid_789_v2`	Versioned content library

curl -X POST https://api.transcodely.com/transcodely.v1.JobService/Create 
  -H "Content-Type: application/json" 
  -H "Authorization: Bearer {{API_KEY}}" 
  -H "X-Organization-ID: {{ORG_ID}}" 
  -d '{
    "input_origin_id": "ori_input12345",
    "input_path": "uploads/episode-01.mp4",
    "output_origin_id": "ori_output6789",
    "idempotency_key": "batch_2026-02-28_episode-01.mp4",
    "outputs": [
      {
        "type": "mp4",
        "video": [{ "codec": "h264", "resolution": "1080p", "quality": "standard" }]
      }
    ]
  }'

const job = await client.jobs.create({
  inputOriginId: "ori_input12345",
  inputPath: "uploads/episode-01.mp4",
  outputOriginId: "ori_output6789",
  idempotencyKey: "batch_2026-02-28_episode-01.mp4",
  outputs: [
    {
      type: OutputFormat.MP4,
      video: [
        {
          codec: VideoCodec.H264,
          resolution: Resolution.RESOLUTION_1080P,
          quality: QualityTier.STANDARD,
        },
      ],
    },
  ],
});

job = client.jobs.create(
    input_origin_id="ori_input12345",
    input_path="uploads/episode-01.mp4",
    output_origin_id="ori_output6789",
    idempotency_key="batch_2026-02-28_episode-01.mp4",
    outputs=[{
        "type": "mp4",
        "video": [{"codec": "h264", "resolution": "1080p", "quality": "standard"}],
    }],
)

job, err := client.Jobs.Create(ctx, &transcodely.JobCreateParams{
    InputOriginId:  proto.String("ori_input12345"),
    InputPath:      proto.String("uploads/episode-01.mp4"),
    OutputOriginId: proto.String("ori_output6789"),
    IdempotencyKey: proto.String("batch_2026-02-28_episode-01.mp4"),
    Outputs: []*transcodely.OutputSpec{{
        Type: transcodely.OutputFormatMP4,
        Video: []*transcodely.VideoVariant{{
            Codec:      transcodely.VideoCodecH264,
            Resolution: transcodely.Resolution1080P,
            Quality:    transcodely.QualityTierStandard,
        }},
    }},
})

If you run this request again with the same idempotency_key, you get back the existing job without creating a new one. This makes your entire batch script safe to re-run.

Rate Limiting

When submitting large batches, be mindful of API rate limits. Transcodely applies per-app rate limits to prevent abuse:

Tier	Rate Limit	Burst
Standard	100 requests/second	200
Premium	500 requests/second	1000

For large batches (hundreds or thousands of videos), add concurrency control:

// Process in batches of 20 concurrent requests
const CONCURRENCY = 20;

async function processBatch(videos: string[]) {
  const results = [];

  for (let i = 0; i < videos.length; i += CONCURRENCY) {
    const batch = videos.slice(i, i + CONCURRENCY);
    const batchResults = await Promise.allSettled(
      batch.map((video) => createJob(video))
    );
    results.push(...batchResults);

    // Brief pause between batches
    if (i + CONCURRENCY < videos.length) {
      await new Promise((resolve) => setTimeout(resolve, 100));
    }
  }

  return results;
}

import asyncio

# Cap in-flight requests with a semaphore instead of fixed-size chunks.
CONCURRENCY = 20

async def process_batch(videos: list[str]):
    sem = asyncio.Semaphore(CONCURRENCY)

    async def run(video: str):
        async with sem:
            # create_job wraps client.jobs.create with a per-video idempotency key
            return await create_job(video)

    return await asyncio.gather(
        *(run(video) for video in videos),
        return_exceptions=True,
    )

// Cap in-flight requests with a buffered channel as a semaphore.
const concurrency = 20

func processBatch(ctx context.Context, videos []string) []*transcodely.Job {
	sem := make(chan struct{}, concurrency)
	jobs := make([]*transcodely.Job, len(videos))
	var wg sync.WaitGroup

	for i, video := range videos {
		wg.Add(1)
		go func(i int, video string) {
			defer wg.Done()
			sem <- struct{}{}        // acquire a slot
			defer func() { <-sem }() // release it
			// createJob wraps client.Jobs.Create with a per-video idempotency key
			jobs[i], _ = createJob(ctx, video)
		}(i, video)
	}

	wg.Wait()
	return jobs
}

Monitoring Batch Progress

Polling All Jobs

After submitting a batch, poll all job statuses to track progress:

async function monitorBatch(jobIds: string[]) {
  const interval = setInterval(async () => {
    const jobs = await Promise.all(
      jobIds.map((id) => jobClient.get({ id }).then((r) => r.job))
    );

    const completed = jobs.filter((j) => j.status === "completed").length;
    const failed = jobs.filter((j) => j.status === "failed").length;
    const processing = jobs.filter(
      (j) => j.status === "processing" || j.status === "pending" || j.status === "probing"
    ).length;

    console.warn(`Progress: ${completed} done, ${failed} failed, ${processing} in progress`);

    if (processing === 0) {
      clearInterval(interval);
      console.warn("Batch complete!");
    }
  }, 10000); // Check every 10 seconds
}

import time

from transcodely.v1 import job_pb2

IN_PROGRESS = {
    job_pb2.JOB_STATUS_PROCESSING,
    job_pb2.JOB_STATUS_PENDING,
    job_pb2.JOB_STATUS_PROBING,
}

def monitor_batch(job_ids: list[str]) -> None:
    while True:
        jobs = [client.jobs.get(job_id) for job_id in job_ids]

        completed = sum(1 for j in jobs if j.status == job_pb2.JOB_STATUS_COMPLETED)
        failed = sum(1 for j in jobs if j.status == job_pb2.JOB_STATUS_FAILED)
        processing = sum(1 for j in jobs if j.status in IN_PROGRESS)

        print(f"Progress: {completed} done, {failed} failed, {processing} in progress")

        if processing == 0:
            print("Batch complete!")
            return
        time.sleep(10)  # Check every 10 seconds

func monitorBatch(ctx context.Context, jobIDs []string) error {
	ticker := time.NewTicker(10 * time.Second) // Check every 10 seconds
	defer ticker.Stop()

	for {
		var completed, failed, processing int
		for _, id := range jobIDs {
			job, err := client.Jobs.Get(ctx, id)
			if err != nil {
				return err
			}
			switch job.GetStatus() {
			case transcodely.JobStatusCompleted:
				completed++
			case transcodely.JobStatusFailed:
				failed++
			case transcodely.JobStatusProcessing,
				transcodely.JobStatusPending,
				transcodely.JobStatusProbing:
				processing++
			}
		}

		log.Printf("Progress: %d done, %d failed, %d in progress", completed, failed, processing)

		if processing == 0 {
			log.Print("Batch complete!")
			return nil
		}
		<-ticker.C
	}
}

Using Webhooks

For production systems, use webhooks instead of polling. Tag each job with metadata to identify the batch:

{
  "input_origin_id": "ori_input12345",
  "input_path": "uploads/episode-01.mp4",
  "output_origin_id": "ori_output6789",
  "metadata": {
    "batch_id": "batch_2026-02-28",
    "content_id": "episode-01",
    "user_id": "usr_abc123"
  },
  "outputs": [
    {
      "type": "mp4",
      "video": [{ "codec": "h264", "resolution": "1080p", "quality": "standard" }]
    }
  ]
}

In your webhook handler, track batch completion:

async function handleJobCompleted(job: any) {
  const batchId = job.metadata.batch_id;

  // Record completion
  await db.batchJobs.update({
    where: { jobId: job.id },
    data: { status: "completed", completedAt: new Date() },
  });

  // Check if batch is complete
  const remaining = await db.batchJobs.count({
    where: { batchId, status: "pending" },
  });

  if (remaining === 0) {
    await notifyBatchComplete(batchId);
  }
}

def handle_job_completed(job):
    batch_id = job.metadata["batch_id"]

    # Record completion
    db.batch_jobs.update(job_id=job.id, status="completed", completed_at=now())

    # Check if batch is complete
    remaining = db.batch_jobs.count(batch_id=batch_id, status="pending")
    if remaining == 0:
        notify_batch_complete(batch_id)

func handleJobCompleted(job *transcodely.Job) error {
	batchID := job.GetMetadata()["batch_id"]

	// Record completion
	if err := db.BatchJobs.Update(job.GetId(), "completed", time.Now()); err != nil {
		return err
	}

	// Check if batch is complete
	remaining, err := db.BatchJobs.Count(batchID, "pending")
	if err != nil {
		return err
	}
	if remaining == 0 {
		return notifyBatchComplete(batchID)
	}
	return nil
}

Handling Partial Failures

In a batch, some jobs may fail while others succeed. Handle failures gracefully:

async function handleBatchResults(results: PromiseSettledResult<any>[]) {
  const failures = results
    .map((r, i) => ({ result: r, index: i }))
    .filter((r) => r.result.status === "rejected");

  if (failures.length === 0) {
    console.warn("All jobs created successfully");
    return;
  }

  console.warn(`${failures.length} jobs failed to create`);

  // Retry failed jobs
  for (const failure of failures) {
    console.warn(`Retrying job ${failure.index}:`, failure.result.reason);
    try {
      // Safe to retry because we use idempotency keys
      await createJob(videos[failure.index]);
    } catch (err) {
      console.error(`Retry failed for ${failure.index}:`, err);
    }
  }
}

# results comes from asyncio.gather(..., return_exceptions=True):
# successful entries are jobs, failed entries are exceptions.
async def handle_batch_results(videos: list[str], results: list) -> None:
    failures = [(i, r) for i, r in enumerate(results) if isinstance(r, Exception)]

    if not failures:
        print("All jobs created successfully")
        return

    print(f"{len(failures)} jobs failed to create")

    for index, reason in failures:
        print(f"Retrying job {index}: {reason}")
        try:
            # Safe to retry because we use idempotency keys
            await create_job(videos[index])
        except Exception as err:
            print(f"Retry failed for {index}: {err}")

// result mirrors the struct returned by the parallel create above
// (video string, job *transcodely.Job, err error).
func handleBatchResults(ctx context.Context, results []result) {
	var failures []result
	for _, r := range results {
		if r.err != nil {
			failures = append(failures, r)
		}
	}

	if len(failures) == 0 {
		log.Print("All jobs created successfully")
		return
	}

	log.Printf("%d jobs failed to create", len(failures))

	for _, f := range failures {
		log.Printf("Retrying %s: %v", f.video, f.err)
		// Safe to retry because we use idempotency keys
		if _, err := createJob(ctx, f.video); err != nil {
			log.Printf("Retry failed for %s: %v", f.video, err)
		}
	}
}

Because idempotency keys are included, retrying a job that actually succeeded (e.g., the original request timed out but the job was created) will simply return the existing job.

Best Practices

Practice	Rationale
Always use idempotency keys	Makes batch scripts safe to re-run after failures
Limit concurrency	Respect rate limits and avoid overwhelming your system
Use metadata for tracking	Tag jobs with `batch_id`, `content_id` for easy filtering
Prefer webhooks over polling	More efficient for monitoring large batches
Handle partial failures	Not all jobs in a batch will necessarily succeed
Use economy priority for bulk work	Lower cost for non-urgent batch processing
Log all job IDs	Essential for debugging and support
Use consistent key naming	Makes it easy to identify and deduplicate across runs

#Overview

#Creating Jobs in Parallel

#Sequential

#Parallel

#Idempotency Keys

#How Idempotency Works

#Key Design Patterns

#Rate Limiting

#Monitoring Batch Progress

#Polling All Jobs

#Using Webhooks

#Handling Partial Failures

#Best Practices