Video manipulation

You can draw frames of a <OffthreadVideo>, <Video> or a <Html5Video> onto a <canvas> element using the drawImage() API.

note

During preview, makes use of the requestVideoFrameCallback() API.
Browser support: Firefox 130 (August 2024), Chrome 83, Safari 15.4.

Basic example

In this example, an <OffthreadVideo> is rendered and made invisible.
Every frame that is emitted is drawn to a Canvas and a grayscale filter is applied.

tsx
export const VideoOnCanvas: React.FC = () => {
  const video = useRef<HTMLVideoElement>(null);
  const canvas = useRef<HTMLCanvasElement>(null);
  const {width, height} = useVideoConfig();
 
  // Process a frame
  const onVideoFrame = useCallback(
    (frame: CanvasImageSource) => {
      if (!canvas.current) {
        return;
      }
      const context = canvas.current.getContext('2d');
 
      if (!context) {
        return;
      }
 
      context.filter = 'grayscale(100%)';
      context.drawImage(frame, 0, 0, width, height);
    },
    [height, width],
  );
 
  return (
    <AbsoluteFill>
      <AbsoluteFill>
        <OffthreadVideo
          // Hide the original video tag
          style={{opacity: 0}}
          onVideoFrame={onVideoFrame}
          src="http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"
        />
      </AbsoluteFill>
      <AbsoluteFill>
        <canvas ref={canvas} width={width} height={height} />
      </AbsoluteFill>
    </AbsoluteFill>
  );
};

Greenscreen example

In this example, we loop over each pixel in the image buffer and if it's green, we transparentize it. Drag the slider below to turn the video transparent.

Slide to adjust transparency:

tsx
export const Greenscreen: React.FC<{
  opacity: number;
}> = ({opacity}) => {
  const canvas = useRef<HTMLCanvasElement>(null);
  const {width, height} = useVideoConfig();
 
  // Process a frame
  const onVideoFrame = useCallback(
    (frame: CanvasImageSource) => {
      if (!canvas.current) {
        return;
      }
      const context = canvas.current.getContext('2d');
 
      if (!context) {
        return;
      }
 
      context.drawImage(frame, 0, 0, width, height);
      const imageFrame = context.getImageData(0, 0, width, height);
      const {length} = imageFrame.data;
 
      // If the pixel is very green, reduce the alpha channel
      for (let i = 0; i < length; i += 4) {
        const red = imageFrame.data[i + 0];
        const green = imageFrame.data[i + 1];
        const blue = imageFrame.data[i + 2];
        if (green > 100 && red < 100 && blue < 100) {
          imageFrame.data[i + 3] = opacity * 255;
        }
      }
      context.putImageData(imageFrame, 0, 0);
    },
    [height, width],
  );
 
  return (
    <AbsoluteFill>
      <AbsoluteFill>
        <OffthreadVideo style={{opacity: 0}} onVideoFrame={onVideoFrame} src="https://remotion.media/greenscreen.mp4" />
      </AbsoluteFill>
      <AbsoluteFill>
        <canvas ref={canvas} width={width} height={height} />
      </AbsoluteFill>
    </AbsoluteFill>
  );
};

Before v4.0.190

Before v4.0.190, the onVideoFrame prop of <OffthreadVideo> and <Html5Video> was not supported.
You could only manipulate a <Html5Video> using the requestVideoFrameCallback API.
Click here to see the old version of this page.