Merge pull request #5338 from remotion-dev/dry-seek

Author: JonnyBurger

.cursorignore

@@ -0,0 +1,2 @@
+# Add directories or file patterns to ignore during indexing (e.g. foo/ or *.csv)
+packages/google-fonts

packages/docs/docs/media-parser/seeking.mdx

@@ -58,6 +58,30 @@ The reason for this is that all samples need to be iterated over for these field
 
 An error will be thrown if you attempt to do this.
 
+## Simulate a seek<AvailableFrom v="4.0.312" />
+
+You can simulate what would happen if you seeked to a certain time.
+
+```tsx twoslash title="Basic seeking"
+import {parseMedia, mediaParserController} from '@remotion/media-parser';
+
+const controller = mediaParserController();
+
+await parseMedia({
+  src: 'https://parser.media/video.mp4',
+  controller,
+  onVideoTrack: () => {
+    return async () => {
+      const result = await controller.simulateSeek(3); 
+      console.log(result); // { "type": "do-seek", byte: 5763, timeInSeconds: 0 }
+    };
+  },
+});
+```
+
+This is useful if you are considering a seek and only want to execute it if the outcome is desired.  
+The result will be a [`SeekResolution`](/docs/media-parser/types#seekresolution) object.
+
 ## How smart is seeking?
 
 The efficiency of seeking depends on the container format.

packages/docs/docs/media-parser/types.mdx

@@ -308,3 +308,11 @@ import type {M3uAssociatedPlaylist} from '@remotion/media-parser';
 - `name`: The name of the audio track
 - `groupId`: The group ID of the audio track
 - `channels`: The number of audio channels in the audio track, or `null`.
+
+## `SeekResolution`<AvailableFrom v="4.0.312" />
+
+```tsx twoslash
+import type {SeekResolution} from '@remotion/media-parser';
+//                ^?
+```
+

packages/media-parser/src/containers/aac/get-seeking-byte.ts

@@ -37,7 +37,11 @@ export const getSeekingByteForAac = ({
 	}
 
 	if (bestAudioSample) {
-		return {type: 'do-seek', byte: bestAudioSample.offset};
+		return {
+			type: 'do-seek',
+			byte: bestAudioSample.offset,
+			timeInSeconds: bestAudioSample.timeInSeconds,
+		};
 	}
 
 	return {type: 'valid-but-must-wait'};

packages/media-parser/src/containers/flac/get-seeking-byte.ts

@@ -35,7 +35,7 @@ export const getSeekingByteForFlac = ({
 	}
 
 	if (bestAudioSample) {
-		return bestAudioSample.offset;
+		return bestAudioSample;
 	}
 
 	return null;

packages/media-parser/src/containers/iso-base-media/find-keyframe-before-time.ts

@@ -58,5 +58,5 @@ export const findKeyframeBeforeTime = ({
 		return null;
 	}
 
-	return videoSample.offset;
+	return videoSample;
 };

packages/media-parser/src/containers/iso-base-media/get-seeking-byte-from-fragmented-mp4.ts

@@ -116,7 +116,10 @@ export const getSeekingByteFromFragmentedMp4 = async ({
 			if (kf) {
 				return {
 					type: 'do-seek',
-					byte: kf,
+					byte: kf.offset,
+					timeInSeconds:
+						Math.min(kf.decodingTimestamp, kf.timestamp) /
+						firstTrack.originalTimescale,
 				};
 			}
 		}

packages/media-parser/src/containers/iso-base-media/get-seeking-byte.ts

@@ -86,7 +86,10 @@ export const getSeekingByteFromIsoBaseMedia = ({
 	if (keyframe) {
 		return Promise.resolve({
 			type: 'do-seek',
-			byte: keyframe,
+			byte: keyframe.offset,
+			timeInSeconds:
+				Math.min(keyframe.decodingTimestamp, keyframe.timestamp) /
+				track.originalTimescale,
 		});
 	}
 

packages/media-parser/src/containers/m3u/get-seeking-byte.ts

@@ -47,5 +47,7 @@ export const getSeekingByteForM3u8 = ({
 	return {
 		type: 'do-seek',
 		byte: currentPosition,
+		// TODO: This will be imperfect when seeking in playMedia()
+		timeInSeconds: time,
 	};
 };

packages/media-parser/src/containers/mp3/get-seeking-byte.ts

@@ -53,8 +53,13 @@ export const getSeekingByteForMp3 = ({
 		};
 	}
 
+	const byte = Math.max(...candidates);
+	const timeInSeconds =
+		byte === bestAudioSample?.offset ? bestAudioSample.timeInSeconds : time;
+
 	return {
 		type: 'do-seek',
-		byte: Math.max(...candidates),
+		byte,
+		timeInSeconds,
 	};
 };

packages/media-parser/src/containers/riff/get-seeking-byte.ts

@@ -39,6 +39,10 @@ export const getSeekingByteForRiff = async ({
 		return {
 			type: 'do-seek',
 			byte: lastKeyframe.positionInBytes,
+			timeInSeconds: Math.min(
+				lastKeyframe.decodingTimeInSeconds,
+				lastKeyframe.presentationTimeInSeconds,
+			),
 		};
 	}
 
@@ -85,5 +89,8 @@ export const getSeekingByteForRiff = async ({
 	return {
 		type: 'do-seek',
 		byte: bestEntry.offset + info.moviOffset - 4,
+		timeInSeconds:
+			bestEntry.sampleCounts[idx1Entries.videoTrackIndex] /
+			info.samplesPerSecond,
 	};
 };

packages/media-parser/src/containers/wav/get-seeking-byte.ts

@@ -23,5 +23,6 @@ export const getSeekingByteFromWav = ({
 	return Promise.resolve({
 		type: 'do-seek',
 		byte: byteOffset + info.mediaSection.start,
+		timeInSeconds: timeRoundedDown,
 	});
 };

packages/media-parser/src/containers/webm/seek/get-seeking-byte.ts

@@ -59,7 +59,7 @@ const findKeyframeBeforeTime = ({
 		}
 	}
 
-	return keyframeBeforeTime?.positionInBytes ?? null;
+	return keyframeBeforeTime ?? null;
 };
 
 const getByteFromCues = ({
@@ -92,7 +92,10 @@ const getByteFromCues = ({
 		return null;
 	}
 
-	return biggestCueBeforeTime.clusterPositionInSegment + segmentOffset;
+	return {
+		byte: biggestCueBeforeTime.clusterPositionInSegment + segmentOffset,
+		timeInSeconds: toSeconds(biggestCueBeforeTime.timeInTimescale, info.track!),
+	};
 };
 
 export const getSeekingByteFromMatroska = async ({
@@ -140,8 +143,8 @@ export const getSeekingByteFromMatroska = async ({
 	// Don't seek back, if the last seen time is smaller than the time we want to seek to
 
 	const seekPossibilities = [
-		byteFromCues,
-		byteFromObservedKeyframe,
+		byteFromCues?.byte ?? null,
+		byteFromObservedKeyframe?.positionInBytes ?? null,
 		byteFromFirstMediaSection,
 	].filter((n) => n !== null);
 
@@ -163,8 +166,28 @@ export const getSeekingByteFromMatroska = async ({
 		size: 1,
 	});
 
+	const timeInSeconds = (() => {
+		if (byteToSeekTo === byteFromObservedKeyframe?.positionInBytes) {
+			return Math.min(
+				byteFromObservedKeyframe.decodingTimeInSeconds,
+				byteFromObservedKeyframe.presentationTimeInSeconds,
+			);
+		}
+
+		if (byteToSeekTo === byteFromCues?.byte) {
+			return byteFromCues.timeInSeconds;
+		}
+
+		if (byteToSeekTo === byteFromFirstMediaSection) {
+			return 0;
+		}
+
+		throw new Error('Should not happen');
+	})();
+
 	return {
 		type: 'do-seek',
 		byte: byteToSeekTo,
+		timeInSeconds,
 	};
 };

packages/media-parser/src/controller/media-parser-controller.ts

@@ -1,5 +1,6 @@
 import {MediaParserAbortError} from '../errors';
 import type {SeekingHints} from '../seeking-hints';
+import type {SeekResolution} from '../work-on-seek-request';
 import {MediaParserEmitter} from './emitter';
 import type {PauseSignal} from './pause-signal';
 import {makePauseSignal} from './pause-signal';
@@ -14,6 +15,7 @@ export type MediaParserController = {
 	pause: PauseSignal['pause'];
 	resume: PauseSignal['resume'];
 	seek: SeekSignal['seek'];
+	simulateSeek: (seekInSeconds: number) => Promise<SeekResolution>;
 	addEventListener: MediaParserEmitter['addEventListener'];
 	removeEventListener: MediaParserEmitter['removeEventListener'];
 	getSeekingHints: () => Promise<SeekingHints | null>;
@@ -29,6 +31,9 @@ export type MediaParserController = {
 		attachSeekingHintResolution: (
 			callback: () => Promise<SeekingHints | null>,
 		) => void;
+		attachSimulateSeekResolution: (
+			callback: (seekInSeconds: number) => Promise<SeekResolution>,
+		) => void;
 	};
 };
 
@@ -53,6 +58,9 @@ export const mediaParserController = (): MediaParserController => {
 	};
 
 	let seekingHintResolution: (() => Promise<SeekingHints | null>) | null = null;
+	let simulateSeekResolution:
+		| ((seekInSeconds: number) => Promise<SeekResolution>)
+		| null = null;
 
 	const getSeekingHints = () => {
 		if (!seekingHintResolution) {
@@ -64,6 +72,16 @@ export const mediaParserController = (): MediaParserController => {
 		return seekingHintResolution();
 	};
 
+	const simulateSeek = (seekInSeconds: number) => {
+		if (!simulateSeekResolution) {
+			throw new Error(
+				'The mediaParserController() was not yet used in a parseMedia() call',
+			);
+		}
+
+		return simulateSeekResolution(seekInSeconds);
+	};
+
 	const attachSeekingHintResolution = (
 		callback: () => Promise<SeekingHints | null>,
 	) => {
@@ -76,13 +94,26 @@ export const mediaParserController = (): MediaParserController => {
 		seekingHintResolution = callback;
 	};
 
+	const attachSimulateSeekResolution = (
+		callback: (seekInSeconds: number) => Promise<SeekResolution>,
+	) => {
+		if (simulateSeekResolution) {
+			throw new Error(
+				'The mediaParserController() was used in multiple parseMedia() calls. Create a separate controller for each call.',
+			);
+		}
+
+		simulateSeekResolution = callback;
+	};
+
 	return {
 		// eslint-disable-next-line @typescript-eslint/no-explicit-any
 		abort: (reason?: any) => {
 			abortController.abort(reason);
 			emitter.dispatchAbort(reason);
 		},
 		seek: seekSignal.seek,
+		simulateSeek,
 		pause: pauseSignal.pause,
 		resume: pauseSignal.resume,
 		addEventListener: emitter.addEventListener,
@@ -95,6 +126,7 @@ export const mediaParserController = (): MediaParserController => {
 			markAsReadyToEmitEvents: emitter.markAsReady,
 			performedSeeksSignal,
 			attachSeekingHintResolution,
+			attachSimulateSeekResolution,
 		},
 	};
 };

packages/media-parser/src/get-seeking-byte.ts

@@ -4,6 +4,7 @@ import {getSeekingByteFromIsoBaseMedia} from './containers/iso-base-media/get-se
 import {getSeekingByteForM3u8} from './containers/m3u/get-seeking-byte';
 import {getSeekingByteForMp3} from './containers/mp3/get-seeking-byte';
 import {getSeekingByteForRiff} from './containers/riff/get-seeking-byte';
+import {MPEG_TIMESCALE} from './containers/transport-stream/handle-avc-packet';
 import {getSeekingByteFromWav} from './containers/wav/get-seeking-byte';
 import {getSeekingByteFromMatroska} from './containers/webm/seek/get-seeking-byte';
 import type {MediaParserLogLevel} from './log';
@@ -86,7 +87,8 @@ export const getSeekingByte = ({
 		if (byte) {
 			return Promise.resolve({
 				type: 'do-seek',
-				byte,
+				byte: byte.offset,
+				timeInSeconds: byte.timeInSeconds,
 			});
 		}
 
@@ -102,12 +104,27 @@ export const getSeekingByte = ({
 			ptsStartOffset: info.ptsStartOffset,
 		});
 
-		const byte = lastKeyframeBeforeTimeInSeconds?.offset ?? 0;
+		if (!lastKeyframeBeforeTimeInSeconds) {
+			transportStream.resetBeforeSeek();
+
+			return Promise.resolve({
+				type: 'do-seek',
+				byte: 0,
+				timeInSeconds: 0,
+			});
+		}
+
+		const byte = lastKeyframeBeforeTimeInSeconds.offset;
 
 		transportStream.resetBeforeSeek();
 		return Promise.resolve({
 			type: 'do-seek',
 			byte,
+			timeInSeconds:
+				Math.min(
+					lastKeyframeBeforeTimeInSeconds.pts,
+					lastKeyframeBeforeTimeInSeconds.dts ?? Infinity,
+				) / MPEG_TIMESCALE,
 		});
 	}
 

packages/media-parser/src/index.ts

@@ -124,6 +124,7 @@ export {
 } from './controller/media-parser-controller';
 export {VERSION} from './version';
 export {WEBCODECS_TIMESCALE} from './webcodecs-timescale';
+export type {SeekResolution} from './work-on-seek-request';
 
 export type {MediaParserSampleAspectRatio} from './get-tracks';