Merge pull request #5324 from remotion-dev/extract-frames

Author: JonnyBurger

packages/bundler/package.json

@@ -31,6 +31,7 @@
 		"remotion": "workspace:*",
 		"@remotion/studio": "workspace:*",
 		"@remotion/studio-shared": "workspace:*",
+		"@remotion/media-parser": "workspace:*",
 		"style-loader": "4.0.0",
 		"source-map": "0.7.3",
 		"webpack": "5.96.1"

packages/bundler/src/webpack-config.ts

@@ -170,6 +170,12 @@ export const webpackConfig = async ({
 					'esm',
 					'index.mjs',
 				),
+				'@remotion/media-parser/worker': path.resolve(
+					require.resolve('@remotion/media-parser'),
+					'..',
+					'esm',
+					'worker.mjs',
+				),
 				// test visual controls before removing this
 				'@remotion/studio': require.resolve('@remotion/studio'),
 				'react-dom/client': shouldUseReactDomClient

packages/convert/app/components/ConvertUi.tsx

@@ -413,7 +413,7 @@ const ConvertUI = ({
 			...unrotatedDimensions,
 			rotation: userRotation - (rotation ?? 0),
 			resizeOperation,
-			videoCodec: isH264Reencode ? 'h264' : 'vp8',
+			needsToBeMultipleOfTwo: isH264Reencode ?? false,
 		});
 	}, [
 		unrotatedDimensions,

packages/docs/docs/webcodecs/TableOfContents.tsx

@@ -122,6 +122,10 @@ export const TableOfContents: React.FC = () => {
 						Create a <code>VideoDecoder</code> object.
 					</div>
 				</TOCItem>
+				<TOCItem link="/docs/webcodecs/extract-frames">
+					<strong>{'extractFrames()'}</strong>
+					<div>Extract frames from a video at specific timestamps.</div>
+				</TOCItem>
 			</Grid>
 		</div>
 	);

packages/docs/docs/webcodecs/extract-frames.mdx

@@ -0,0 +1,96 @@
+---
+image: /generated/articles-docs-webcodecs-extract-frames.png
+id: extract-frames
+title: extractFrames()
+slug: /webcodecs/extract-frames
+crumb: '@remotion/webcodecs'
+---
+
+# extractFrames()<AvailableFrom v="4.0.311"/>
+
+_Part of the [`@remotion/webcodecs`](/docs/webcodecs) package._
+
+Extracts frames from a video at specific timestamps.
+
+```tsx twoslash title="Extracting frames"
+import {extractFrames} from '@remotion/webcodecs';
+
+await extractFrames({
+  src: 'https://parser.media/video.mp4',
+  timestampsInSeconds: [0, 1, 2, 3, 4],
+  onFrame: (frame) => {
+    console.log(frame);
+    //           ^?
+  },
+});
+```
+
+## API
+
+### `src`
+
+A URL or `File`/`Blob`.
+
+If it is a remote URL, it must support CORS.
+
+### `timestampsInSeconds`
+
+An array of timestamps in seconds, or a function that returns a promise resolving to an array of timestamps in seconds based on the video track.
+
+Consider you wanting you to create a filmstrip of a video. You can do this by extracting as many frames as fit in a canvas.
+
+```tsx twoslash title="Extracting as many frames as fit in a canvas"
+import type {ExtractFramesTimestampsInSecondsFn} from '@remotion/webcodecs';
+
+const toSeconds = 10;
+const fromSeconds = 0;
+const canvasWidth = 500;
+const canvasHeight = 80;
+
+const timestamps: ExtractFramesTimestampsInSecondsFn = async ({track}) => {
+  const aspectRatio = track.width / track.height;
+  const amountOfFramesFit = Math.ceil(
+    canvasWidth / (canvasHeight * aspectRatio),
+  );
+  const timestampsInSeconds: number[] = [];
+  const segmentDuration = toSeconds - fromSeconds;
+
+  for (let i = 0; i < amountOfFramesFit; i++) {
+    timestampsInSeconds.push(
+      fromSeconds + (segmentDuration / amountOfFramesFit) * (i + 0.5),
+    );
+  }
+
+  return timestampsInSeconds;
+};
+```
+
+Note that currently, you can not get the duration of the video in seconds before the extraction.  
+For this you need currently to make another [`parseMedia()`](/docs/media-parser/parse-media) call beforehand.
+
+### `onFrame`
+
+A callback that will be called with the frame at the given timestamp.  
+Each frame is a [`VideoFrame`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame) object that can for example be drawn to a canvas.
+
+
+### `acknowledgeRemotionLicense?`
+
+Acknowledge the [Remotion License](/docs/license) to make the console message disappear.
+
+### `signal?`
+
+An optional [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) to abort the extraction.
+
+### `logLevel?`
+
+_string_ <TsType type="LogLevel" source="@remotion/media-parser"/>
+
+One of `"error"`, `"warn"`, `"info"`, `"debug"`, `"trace"`.  
+Default value: `"info"`, which logs only important information.
+
+## See also
+
+- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/webcodecs/src/extract-frames.ts)
+- [`@remotion/webcodecs`](/docs/webcodecs)
+- [`parseMedia()`](/docs/media-parser/parse-media)

packages/docs/docs/webcodecs/index.mdx

@@ -16,12 +16,13 @@ It leverages [`@remotion/media-parser`](/docs/media-parser) to parse the video a
 import {LicenseDisclaimer} from './LicenseDisclaimer';
 import {UnstableDisclaimer} from './UnstableDisclaimer';
 
-## What can you to with this package?
+## What can you do with this package?
 
 In browsers that implement WebCodecs, you can use this package to:
 
 - [Convert videos from one format to another](/docs/webcodecs/convert-a-video) (From .mp4, .webm, .mov, .mkv, .m3u8, .ts, .avi, .mp3, .flac, .wav, .m4a, .aac to .mp4, .webm, .wav)
 - [Rotate videos](/docs/webcodecs/rotate-a-video)
+- [Efficiently extract frames from a video](/docs/webcodecs/extract-frames)
 - Extract audio from a video
 - Manipulate the pixels of a video
 - [Fix videos that were recorded with `MediaRecorder`](/docs/webcodecs/fix-mediarecorder-video)

packages/docs/sidebars.js

@@ -685,6 +685,7 @@ module.exports = {
 				'webcodecs/convert-audiodata',
 				'webcodecs/create-audio-decoder',
 				'webcodecs/create-video-decoder',
+				'webcodecs/extract-frames',
 			],
 		},
 		{

packages/docs/src/data/articles.ts

@@ -2971,7 +2971,7 @@ export const articles = [
 	},
 	{
 		id: 'webcodecs',
-		title: 'Decoding a video with WebCodecs and @remotion/media-parser',
+		title: 'Processing video with WebCodecs and @remotion/media-parser',
 		relativePath: 'docs/media-parser/webcodecs.mdx',
 		compId: 'articles-docs-media-parser-webcodecs',
 		crumb: '@remotion/media-parser',
@@ -6045,6 +6045,15 @@ export const articles = [
 		noAi: false,
 		slug: 'webcodecs/default-on-video-track-handler',
 	},
+	{
+		id: 'extract-frames',
+		title: 'extractFrames()',
+		relativePath: 'docs/webcodecs/extract-frames.mdx',
+		compId: 'articles-docs-webcodecs-extract-frames',
+		crumb: '@remotion/webcodecs',
+		noAi: false,
+		slug: 'webcodecs/extract-frames',
+	},
 	{
 		id: 'fix-mediarecorder-video',
 		title: 'Fixing a MediaRecorder video',

packages/docs/static/generated/articles-docs-webcodecs-extract-frames.png

@@ -93,7 +93,8 @@ const convertToWorkerPayload = (
 		postM3uAssociatedPlaylistsSelection: Boolean(selectM3uAssociatedPlaylists),
 		postOnAudioTrack: Boolean(onAudioTrack),
 		postOnVideoTrack: Boolean(onVideoTrack),
-		src,
+		// URL cannot be serialized, so we convert it to a string
+		src: src instanceof URL ? src.toString() : src,
 	};
 };
 

packages/media-parser/src/parse-media-on-worker-entry.ts

@@ -54,6 +54,15 @@ export const serializeError = ({
 		};
 	}
 
+	if (error instanceof TypeError) {
+		return {
+			type: 'response-error',
+			errorName: 'TypeError',
+			errorMessage: error.message,
+			errorStack: error.stack ?? '',
+		};
+	}
+
 	if (error.name === 'AbortError') {
 		return {
 			type: 'response-error',
@@ -121,6 +130,8 @@ export const deserializeError = (error: ResponseError): Error => {
 		// TODO: Document 2GB limit
 		case 'NotReadableError':
 			return new Error(error.errorMessage);
+		case 'TypeError':
+			return new TypeError(error.errorMessage);
 		default:
 			throw new Error(`Unknown error name: ${error satisfies never}`);
 	}

packages/media-parser/src/worker/serialize-error.ts

@@ -125,6 +125,10 @@ type NotReadableError = BaseError & {
 	errorName: 'NotReadableError';
 };
 
+type TypeError = BaseError & {
+	errorName: 'TypeError';
+};
+
 type IsAnImageError = BaseError & {
 	errorName: 'IsAnImageError';
 	imageType: ImageType;
@@ -161,7 +165,8 @@ type AnyError =
 	| MediaParserAbortError
 	// browser native errors
 	| AbortError
-	| NotReadableError;
+	| NotReadableError
+	| TypeError;
 
 export type ResponseError = {
 	type: 'response-error';

packages/media-parser/src/worker/worker-types.ts

@@ -13,6 +13,8 @@ const external = [
 	'@remotion/renderer/client',
 	'@remotion/renderer/pure',
 	'@remotion/renderer/error-handling',
+	'@remotion/media-parser/worker',
+	'@remotion/webcodecs',
 	'source-map',
 	'zod',
 	'remotion/no-react',

packages/studio/bundle.ts

@@ -31,6 +31,7 @@
 		"@remotion/media-parser": "workspace:*",
 		"@remotion/renderer": "workspace:*",
 		"@remotion/studio-shared": "workspace:*",
+		"@remotion/webcodecs": "workspace:*",
 		"@remotion/zod-types": "workspace:*",
 		"memfs": "3.4.3",
 		"source-map": "0.7.3",

packages/studio/package.json

@@ -55,6 +55,8 @@ const container: React.CSSProperties = {
 const style: React.CSSProperties = {
 	width: '100%',
 	height: '100%',
+	userSelect: 'none',
+	WebkitUserSelect: 'none',
 };
 
 const getClientXWithScroll = (x: number) => {

packages/studio/src/components/Timeline/TimelineDragHandler.tsx

@@ -120,7 +120,14 @@ const Inner: React.FC<{
 					playbackRate={s.playbackRate}
 				/>
 			) : null}
-			{s.type === 'video' ? <TimelineVideoInfo src={s.src} /> : null}
+			{s.type === 'video' ? (
+				<TimelineVideoInfo
+					src={s.src}
+					visualizationWidth={width}
+					startFrom={s.startMediaFrom}
+					durationInFrames={s.duration}
+				/>
+			) : null}
 			{s.loopDisplay === undefined ? null : (
 				<LoopedTimelineIndicator loops={s.loopDisplay.numberOfTimes} />
 			)}