# Moving Pill Captions

Synchronized captions with a pill that moves between spoken words.

Element page: [https://www.remotion.dev/elements/captions/moving-pill-captions/](https://www.remotion.dev/elements/captions/moving-pill-captions/)

## Installation

Install this Element into a composition of your Remotion project:

```sh
npx remotion elements use https://www.remotion.dev/elements/captions/moving-pill-captions/ --composition <composition-id> --yes
```

Replace `<composition-id>` with the ID of the composition to add the Element to. The command installs the dependencies, downloads the assets, writes the source file and adds the Element to the composition. See [`npx remotion elements use`](https://www.remotion.dev/docs/cli/elements/use.md).

## Manual installation

To install the Element manually instead, save the source below as `moving-pill-captions.element.tsx` in your Remotion project and import its exported component.

## Source

```tsx
import type {Caption, TikTokPage, TikTokToken} from '@remotion/captions';
import {createTikTokStyleCaptions} from '@remotion/captions';
import {loadFont} from '@remotion/google-fonts/Montserrat';
import {fitText} from '@remotion/layout-utils';
import React, {
	useEffect,
	useLayoutEffect,
	useMemo,
	useRef,
	useState,
} from 'react';
import {
	cancelRender,
	Interactive,
	interpolate,
	spring,
	useCurrentFrame,
	useVideoConfig,
	type InteractiveTransformProps,
	type InteractivitySchema,
	type SequenceProps,
} from 'remotion';

type MovingPillCaptionsProps = InteractiveTransformProps &
	Pick<SequenceProps, 'width'> & {
		readonly captions: Caption[];
		readonly combineTokensWithinMilliseconds?: number;
	};

const desiredFontSize = 80;
const fontWeight = '700';
const textColor = '#ffffff';
const backgroundColor = '#2563eb';
const pillHorizontalPadding = 12;
const pillVerticalPadding = 12;
const pillBorderRadius = 10;
const pillMoveDurationInFrames = 5;
const defaultCombineTokensWithinMilliseconds = 800;
const defaultWidth = 682;

const movingPillCaptionsSchema = {
	...Interactive.captionsSchema,
	width: {
		type: 'number',
		min: 1,
		step: 1,
		default: undefined,
		description: 'Caption area width',
		hiddenFromList: false,
	},
	combineTokensWithinMilliseconds: {
		type: 'number',
		min: 0,
		step: 50,
		default: defaultCombineTokensWithinMilliseconds,
		description: 'Time between caption pages',
		hiddenFromList: false,
	},
} as const satisfies InteractivitySchema;

const {fontFamily, waitUntilDone} = loadFont('normal', {
	weights: [fontWeight],
	subsets: ['latin'],
});

const isTimeWithinHalfOpenInterval = (
	timeMs: number,
	fromMs: number,
	toMs: number,
) => timeMs >= fromMs && timeMs < toMs;

const getActivePageIndex = (
	pages: readonly Pick<TikTokPage, 'startMs' | 'durationMs'>[],
	timeMs: number,
) =>
	pages.findIndex((page) =>
		isTimeWithinHalfOpenInterval(
			timeMs,
			page.startMs,
			page.startMs + page.durationMs,
		),
	);

const getLatestStartedTokenIndex = (
	tokens: readonly Pick<TikTokToken, 'fromMs'>[],
	timeMs: number,
) =>
	tokens.reduce(
		(latestIndex, token, tokenIndex) =>
			token.fromMs <= timeMs ? tokenIndex : latestIndex,
		-1,
	);

type TokenLayout = {
	readonly height: number;
	readonly left: number;
	readonly top: number;
	readonly width: number;
};

const CaptionPage: React.FC<{
	readonly captionAreaWidth: number | null;
	readonly currentTimeMs: number;
	readonly fps: number;
	readonly page: TikTokPage;
	readonly pageIndex: number;
}> = ({captionAreaWidth, currentTimeMs, fps, page, pageIndex}) => {
	const {width: compositionWidth} = useVideoConfig();
	const availableWidth = captionAreaWidth ?? compositionWidth;
	const fontSize = useMemo(() => {
		const maximumTokenWidth = Math.max(
			1,
			availableWidth - pillHorizontalPadding * 2,
		);
		const tokenFontSizes = page.tokens
			.map((token) => token.text.trim())
			.filter(Boolean)
			.map(
				(text) =>
					fitText({
						fontFamily,
						fontWeight,
						text,
						validateFontIsLoaded: true,
						withinWidth: maximumTokenWidth,
					}).fontSize,
			);

		return Math.min(
			desiredFontSize,
			fitText({
				fontFamily,
				fontWeight,
				text: page.text,
				validateFontIsLoaded: true,
				withinWidth: availableWidth,
			}).fontSize,
			...tokenFontSizes,
		);
	}, [availableWidth, page.text, page.tokens]);
	const textContainerRef = useRef<HTMLDivElement>(null);
	const tokenRefs = useRef<Array<HTMLSpanElement | null>>([]);
	const [tokenLayouts, setTokenLayouts] = useState<TokenLayout[]>([]);

	useLayoutEffect(() => {
		const container = textContainerRef.current;

		if (!container) {
			return;
		}

		const updateTokenLayouts = () => {
			setTokenLayouts(
				page.tokens.map((_, tokenIndex) => {
					const token = tokenRefs.current[tokenIndex];

					if (!token) {
						return {height: 0, left: 0, top: 0, width: 0};
					}

					return {
						height: token.offsetHeight,
						left: token.offsetLeft,
						top: token.offsetTop,
						width: token.offsetWidth,
					};
				}),
			);
		};

		updateTokenLayouts();
		const resizeObserver = new ResizeObserver(updateTokenLayouts);
		resizeObserver.observe(container);

		return () => resizeObserver.disconnect();
	}, [fontSize, page.tokens]);
	const latestStartedTokenIndex = getLatestStartedTokenIndex(
		page.tokens,
		currentTimeMs,
	);
	const pageLocalFrame = ((currentTimeMs - page.startMs) / 1000) * fps;
	const backgroundWordIndex = page.tokens.reduce(
		(wordIndex, token, tokenIndex) => {
			if (tokenIndex === 0) {
				return wordIndex;
			}

			const tokenStartFrame = ((token.fromMs - page.startMs) / 1000) * fps;
			return (
				wordIndex +
				spring({
					config: {damping: 100},
					delay: Math.max(0, tokenStartFrame - pillMoveDurationInFrames / 2),
					durationInFrames: pillMoveDurationInFrames,
					fps,
					frame: pageLocalFrame,
				})
			);
		},
		0,
	);
	const hasTokenLayouts = tokenLayouts.length === page.tokens.length;
	const clampedBackgroundWordIndex = Math.min(
		Math.max(backgroundWordIndex, 0),
		Math.max(0, tokenLayouts.length - 1),
	);
	const interpolationInput = tokenLayouts.map((_, tokenIndex) =>
		Number(tokenIndex),
	);
	const interpolateTokenLayout = (values: number[]) => {
		if (values.length === 0) {
			return 0;
		}

		if (values.length === 1) {
			return values[0];
		}

		return interpolate(clampedBackgroundWordIndex, interpolationInput, values);
	};
	const pillHeight = fontSize + pillVerticalPadding * 2;
	const pillLeft = interpolateTokenLayout(
		tokenLayouts.map((measurement) => measurement.left),
	);
	const pillTop = interpolateTokenLayout(
		tokenLayouts.map(
			(measurement) => measurement.top + (measurement.height - pillHeight) / 2,
		),
	);
	const pillWidth = interpolateTokenLayout(
		tokenLayouts.map((measurement) => measurement.width),
	);
	const textStrokeWidth = fontSize / 7;

	return (
		<div
			style={{
				alignItems: 'center',
				display: 'flex',
				height: '100%',
				justifyContent: 'center',
				width: '100%',
			}}
		>
			<div
				ref={textContainerRef}
				style={{
					color: textColor,
					fontFamily,
					fontSize,
					fontWeight,
					lineHeight: 1.5,
					paintOrder: 'stroke fill',
					position: 'relative',
					textAlign: 'center',
					WebkitTextStroke: `${textStrokeWidth}px #000000`,
					whiteSpace: 'normal',
					width: '100%',
				}}
			>
				{hasTokenLayouts && latestStartedTokenIndex >= 0 ? (
					<div
						style={{
							backgroundColor,
							borderRadius: pillBorderRadius,
							height: pillHeight,
							left: pillLeft - pillHorizontalPadding,
							position: 'absolute',
							top: pillTop,
							width: pillWidth + pillHorizontalPadding * 2,
						}}
					/>
				) : null}
				{page.tokens.map((token, tokenIndex) => {
					const visibleText = token.text.trim();
					const visibleTextIndex = token.text.indexOf(visibleText);
					const leadingWhitespace = visibleText
						? token.text.slice(0, visibleTextIndex)
						: token.text;
					const trailingWhitespace = visibleText
						? token.text.slice(visibleTextIndex + visibleText.length)
						: '';

					return (
						<React.Fragment
							key={`${pageIndex}-${page.startMs}-${token.fromMs}-${tokenIndex}`}
						>
							{leadingWhitespace}
							<span
								ref={(element) => {
									tokenRefs.current[tokenIndex] = element;
								}}
								style={{
									color: textColor,
									display: 'inline-block',
									position: 'relative',
									whiteSpace: 'pre',
									zIndex: 1,
								}}
							>
								{visibleText}
							</span>
							{trailingWhitespace}
						</React.Fragment>
					);
				})}
			</div>
		</div>
	);
};

const MovingPillCaptionsContent: React.FC<MovingPillCaptionsProps> = ({
	captions,
	combineTokensWithinMilliseconds = defaultCombineTokensWithinMilliseconds,
	style,
	width = defaultWidth,
}) => {
	const [fontLoaded, setFontLoaded] = useState(false);

	useEffect(() => {
		waitUntilDone()
			.then(() => {
				setFontLoaded(true);
			})
			.catch((error) => {
				cancelRender(error instanceof Error ? error : new Error(String(error)));
			});
	}, []);

	const frame = useCurrentFrame();
	const {fps} = useVideoConfig();
	const pages = useMemo(
		() =>
			createTikTokStyleCaptions({
				captions,
				combineTokensWithinMilliseconds,
			}).pages,
		[captions, combineTokensWithinMilliseconds],
	);
	const currentTimeMs = (frame / fps) * 1000;
	const activePageIndex = getActivePageIndex(pages, currentTimeMs);
	const page = pages[activePageIndex];

	return (
		<div
			style={{
				width,
				...style,
			}}
		>
			{fontLoaded && page ? (
				<CaptionPage
					key={`${activePageIndex}-${page.startMs}`}
					captionAreaWidth={width ?? null}
					currentTimeMs={currentTimeMs}
					fps={fps}
					page={page}
					pageIndex={activePageIndex}
				/>
			) : null}
		</div>
	);
};

export const MovingPillCaptions = Interactive.create({
	Component: MovingPillCaptionsContent,
	componentName: '<MovingPillCaptions>',
	schema: movingPillCaptionsSchema,
});
```

