fix: OPTIC-1608: Seeking video frame by frame sometimes produces duplicated or skipped frames visually (#7027)

Co-authored-by: yyassi-heartex <[email protected]>
Co-authored-by: bmartel <[email protected]>

Author: bmartel

web/libs/editor/public/files/fps_24_frames_24_video.webm

@@ -1,6 +1,6 @@
 import { forwardRef, memo, type MutableRefObject, useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { Block, Elem } from "../../utils/bem";
-import { FF_LSDV_4711, isFF } from "../../utils/feature-flags";
+import { FF_LSDV_4711, FF_VIDEO_FRAME_SEEK_PRECISION, isFF } from "../../utils/feature-flags";
 import { clamp, isDefined } from "../../utils/utilities";
 import "./VideoCanvas.scss";
 import { MAX_ZOOM, MIN_ZOOM } from "./VideoConstants";
@@ -55,6 +55,10 @@ export const clampZoom = (value: number) => clamp(value, MIN_ZOOM, MAX_ZOOM);
 const zoomRatio = (canvasWidth: number, canvasHeight: number, width: number, height: number) =>
   Math.min(1, Math.min(canvasWidth / width, canvasHeight / height));
 
+// Browsers can only handle time to the nearest 2ms, so we need to round to that precision when seeking by frames
+// https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/currentTime
+const BROWSER_TIME_PRECISION = 0.002;
+
 export interface VideoRef {
   currentFrame: number;
   length: number;
@@ -167,7 +171,9 @@ export const VideoCanvas = memo(
         if (!contextRef.current) return;
 
         const currentTime = videoRef.current?.currentTime ?? 0;
-        const frameNumber = Math.round(currentTime * framerate);
+        const frameNumber = isFF(FF_VIDEO_FRAME_SEEK_PRECISION)
+          ? Math.ceil(currentTime * framerate)
+          : Math.round(currentTime * framerate);
         const frame = clamp(frameNumber, 1, length || 1);
         const onChange = props.onFrameChange ?? (() => {});
 
@@ -415,8 +421,16 @@ export const VideoCanvas = memo(
       goToFrame(frame: number) {
         const frameClamped = clamp(frame, 1, length);
 
-        this.currentTime = frameClamped / framerate;
-        requestAnimationFrame(() => drawVideo());
+        // We need to subtract 1 from the frame number because the frame number is 1-based
+        // and the currentTime is 0-based
+        const frameZeroBased = frameClamped - 1;
+
+        // Calculate exact frame time
+        const exactTime = frameZeroBased / framerate;
+
+        // Round to next closest browser precision frame time
+        this.currentTime =
+          Math.round(exactTime / BROWSER_TIME_PRECISION) * BROWSER_TIME_PRECISION + BROWSER_TIME_PRECISION;
       },
     };
 
@@ -459,7 +473,9 @@ export const VideoCanvas = memo(
           const video = videoRef.current;
 
           loadTimeout = setTimeout(() => {
-            const length = Math.ceil(video.duration * framerate);
+            const length = isFF(FF_VIDEO_FRAME_SEEK_PRECISION)
+              ? Math.round(video.duration * framerate)
+              : Math.ceil(video.duration * framerate);
             const [width, height] = [video.videoWidth, video.videoHeight];
 
             const dimensions = {

web/libs/editor/src/components/VideoCanvas/VideoCanvas.tsx

@@ -6,6 +6,7 @@ import IsReadyMixin from "../../../mixins/IsReadyMixin";
 import ProcessAttrsMixin from "../../../mixins/ProcessAttrs";
 import { SyncableMixin } from "../../../mixins/Syncable";
 import { parseValue } from "../../../utils/data";
+import { FF_VIDEO_FRAME_SEEK_PRECISION, isFF } from "../../../utils/feature-flags";
 import ObjectBase from "../Base";
 
 /**
@@ -196,7 +197,11 @@ const Model = types
       setFrame(frame) {
         if (self.frame !== frame && self.framerate) {
           self.frame = frame;
-          self.ref.current.currentTime = frame / self.framerate;
+          if (isFF(FF_VIDEO_FRAME_SEEK_PRECISION)) {
+            self.ref.current.goToFrame(frame);
+          } else {
+            self.ref.current.currentTime = frame / self.framerate;
+          }
         }
       },
 

web/libs/editor/src/tags/object/Video/Video.js

@@ -208,6 +208,8 @@ export const FF_PER_FIELD_COMMENTS = "fflag_feat_all_leap_1430_per_field_comment
 
 export const FF_IMAGE_MEMORY_USAGE = "fflag_feat_front_optic_1479_improve_image_tag_memory_usage_short";
 
+export const FF_VIDEO_FRAME_SEEK_PRECISION = "fflag_fix_front_optic_1608_improve_video_frame_seek_precision_short";
+
 Object.assign(window, {
   APP_SETTINGS: {
     ...(window.APP_SETTINGS ?? {}),

web/libs/editor/src/utils/feature-flags.ts

@@ -0,0 +1,37 @@
+import { LabelStudio, VideoView } from "@humansignal/frontend-test/helpers/LSF/index";
+import { simpleVideoConfig } from "../../data/video_segmentation/regions";
+import { FF_VIDEO_FRAME_SEEK_PRECISION } from "libs/editor/src/utils/feature-flags";
+
+const simpleVideoData = { video: "/public/files/fps_24_frames_24_video.webm" };
+
+describe("Video Frame Seeking", () => {
+  beforeEach(() => {
+    LabelStudio.addFeatureFlagsOnPageLoad({
+      [FF_VIDEO_FRAME_SEEK_PRECISION]: true,
+    });
+  });
+
+  it("Should be able to seek to a specific frame and see the correct frame without duplicating or skipping frames", () => {
+    LabelStudio.params().config(simpleVideoConfig).data(simpleVideoData).withResult([]).init();
+
+    LabelStudio.waitForObjectsReady();
+
+    VideoView.captureVideoCanvas("video_canvas");
+
+    VideoView.clickAtFrame(2);
+
+    VideoView.videoCanvasShouldChange("video_canvas", 0);
+
+    VideoView.captureVideoCanvas("video_canvas");
+
+    VideoView.clickAtFrame(3);
+
+    VideoView.videoCanvasShouldChange("video_canvas", 0);
+
+    VideoView.captureVideoCanvas("video_canvas");
+
+    VideoView.clickAtFrame(4);
+
+    VideoView.videoCanvasShouldChange("video_canvas", 0);
+  });
+});

web/libs/editor/tests/integration/e2e/video/frame_seeking.cy.ts

@@ -18,6 +18,9 @@ export const VideoView = {
     cy.log("Get VideoView's drawing area");
     return this.root.get(".konvajs-content");
   },
+  get videoCanvas() {
+    return this.root.get(".lsf-video-canvas");
+  },
   get timelineContainer() {
     return this.root.get(".lsf-video-segmentation__timeline");
   },
@@ -134,4 +137,32 @@ export const VideoView = {
   canvasShouldNotChange(name: string, treshold = 0.1) {
     return this.drawingArea.compareScreenshot(name, "shouldNotChange", { withHidden: [".lsf-video-canvas"], treshold });
   },
+
+  /**
+   * Captures a screenshot of the video canvas to compare later
+   * @param {string} name name of the screenshot
+   */
+  captureVideoCanvas(name: string) {
+    return this.videoCanvas.captureScreenshot(name, { withHidden: [".konvajs-content"] });
+  },
+
+  /**
+   * Captures a new screenshot of the video canvas and compares it to already taken one
+   * Fails if screenshots are identical
+   * @param name name of the screenshot
+   * @param treshold to compare image. It's a relation between original number of pixels vs changed number of pixels
+   */
+  videoCanvasShouldChange(name: string, treshold = 0.1) {
+    return this.videoCanvas.compareScreenshot(name, "shouldChange", { withHidden: [".konvajs-content"], treshold });
+  },
+
+  /**
+   * Captures a new screenshot of the video canvas and compares it to already taken one
+   * Fails if screenshots are different
+   * @param name name of the screenshot
+   * @param treshold to compare image. It's a relation between original number of pixels vs changed number of pixels
+   */
+  videoCanvasShouldNotChange(name: string, treshold = 0.1) {
+    return this.videoCanvas.compareScreenshot(name, "shouldNotChange", { withHidden: [".konvajs-content"], treshold });
+  },
 };