recreate filter & encoder on resize instead of choking

cleanup

Author: russelltg

src/avhw.rs

@@ -9,6 +9,9 @@ use ffmpeg::{
     format::Pixel,
     frame,
 };
+use log::error;
+
+use crate::DrmModifier;
 
 pub struct AvHwDevCtx {
     ptr: *mut ffmpeg::sys::AVBufferRef,
@@ -45,6 +48,7 @@ impl AvHwDevCtx {
         pixfmt: Pixel,
         width: i32,
         height: i32,
+        modifier: DrmModifier,
     ) -> Result<AvHwFrameCtx, ffmpeg::Error> {
         unsafe {
             let mut hwframe = av_hwframe_ctx_alloc(self.ptr as *mut _);
@@ -57,6 +61,10 @@ impl AvHwDevCtx {
             (*hwframe_casted).height = height;
             (*hwframe_casted).initial_pool_size = 5;
 
+            if modifier != DrmModifier::LINEAR {
+                error!("unknown how to request non-linear frames in vaapi");
+            }
+
             let sts = av_hwframe_ctx_init(hwframe);
             if sts != 0 {
                 return Err(ffmpeg::Error::from(sts));

src/cap_ext_image_copy.rs

@@ -3,7 +3,7 @@ use std::path::PathBuf;
 use anyhow::Context;
 use drm::{buffer::DrmFourcc, node::DrmNode};
 use libc::dev_t;
-use log::warn;
+use log::{debug, warn};
 use log_once::warn_once;
 use wayland_client::{
     globals::GlobalList, protocol::wl_output::WlOutput, Dispatch, Proxy, QueueHandle,
@@ -20,7 +20,7 @@ use wayland_protocols::ext::{
     },
 };
 
-use crate::{CaptureSource, DmabufFormat, DmabufPotentialFormat, DrmModifier, State};
+use crate::{CaptureSource, DmabufPotentialFormat, DrmModifier, State};
 
 impl Dispatch<ExtImageCopyCaptureManagerV1, ()> for State<CapExtImageCopy> {
     fn event(
@@ -99,11 +99,7 @@ impl Dispatch<ExtImageCopyCaptureSessionV1, ()> for State<CapExtImageCopy> {
                 }
             }
             ext_image_copy_capture_session_v1::Event::Done => {
-                let mut constraints = BufferConstraints {
-                    dmabuf_formats: Vec::new(),
-                    buffer_size: None,
-                    dmabuf_device: None,
-                };
+                let mut constraints = BufferConstraints::default();
                 // All buffer constraint events will be resent on every change, so reset
                 // accumulated state
                 std::mem::swap(
@@ -114,28 +110,17 @@ impl Dispatch<ExtImageCopyCaptureSessionV1, ()> for State<CapExtImageCopy> {
                 let size = constraints
                     .buffer_size
                     .expect("Done received before BufferSize...");
-                let fmt = state.negotiate_format(
+                state.negotiate_format(
                     &constraints.dmabuf_formats,
                     size,
                     constraints.dmabuf_device.as_deref(),
+                    qhandle,
                 );
-                let Some(fmt) = fmt else {
-                    // error, it's already reported so we just have to cleanup & exit
-                    return;
-                };
-
-                let cap = state.enc.unwrap_cap();
-                cap.current_config = Some((fmt, size));
-
-                let (width, height, format, frame) = cap
-                    .queue_capture_frame(qhandle)
-                    .expect("Done without size/format!");
-                state.on_copy_src_ready(width, height, format, qhandle, &frame);
             }
             ext_image_copy_capture_session_v1::Event::Stopped => {
                 state.on_copy_fail(qhandle); // untested if this actually works
             }
-            _ => todo!(),
+            _ => {}
         }
     }
 }
@@ -151,8 +136,8 @@ impl Dispatch<ExtImageCopyCaptureFrameV1, ()> for State<CapExtImageCopy> {
     ) {
         use wayland_protocols::ext::image_copy_capture::v1::client::ext_image_copy_capture_frame_v1::Event::*;
         match event {
-            Transform { .. } => {}
-            Damage { .. } => {} // TODO: maybe this is how you implement damage
+            Transform { .. } => {} // TODO: use this
+            Damage { .. } => {}    // TODO: maybe this is how you implement damage
             PresentationTime {
                 tv_sec_hi,
                 tv_sec_lo,
@@ -162,15 +147,17 @@ impl Dispatch<ExtImageCopyCaptureFrameV1, ()> for State<CapExtImageCopy> {
                 let (hi, lo, n) = state.enc.unwrap().cap.time.take().unwrap();
                 state.on_copy_complete(qhandle, hi, lo, n);
             }
-            Failed { .. } => {
+            Failed { reason } => {
+                debug!("frame copy failed: {reason:?}");
                 state.on_copy_fail(qhandle);
             }
-            _ => {}
+            _ => todo!(),
         }
     }
 }
 
 /** Struct to collect buffer constraint information as the events arrive */
+#[derive(Default)]
 struct BufferConstraints {
     dmabuf_formats: Vec<DmabufPotentialFormat>,
     buffer_size: Option<(u32, u32)>,
@@ -181,7 +168,6 @@ pub struct CapExtImageCopy {
     output_capture_session: ExtImageCopyCaptureSessionV1,
     time: Option<(u32, u32, u32)>,
     in_progress_constraints: BufferConstraints,
-    current_config: Option<(DmabufFormat, (u32, u32))>,
 }
 
 impl CaptureSource for CapExtImageCopy {
@@ -218,28 +204,17 @@ impl CaptureSource for CapExtImageCopy {
         Ok(Self {
             output_capture_session,
             time: None,
-            in_progress_constraints: BufferConstraints {
-                dmabuf_formats: Vec::new(),
-                buffer_size: None,
-                dmabuf_device: None,
-            },
-            current_config: None,
+            in_progress_constraints: BufferConstraints::default(),
         })
     }
 
-    fn queue_capture_frame(
-        &self,
-        eq: &QueueHandle<crate::State<Self>>,
-    ) -> Option<(u32, u32, DrmFourcc, Self::Frame)> {
-        if let Some((fmt, (w, h))) = &self.current_config {
-            let frame = self.output_capture_session.create_frame(eq, ());
-            Some((*w, *h, fmt.fourcc, frame))
-        } else {
-            None
-        }
+    fn alloc_frame(&self, eq: &QueueHandle<crate::State<Self>>) -> Option<Self::Frame> {
+        debug!("ext_image_copy_capture_session_v1::create_frame");
+        let frame = self.output_capture_session.create_frame(eq, ());
+        Some(frame)
     }
 
-    fn queue_copy_frame(
+    fn queue_copy(
         &self,
         damage: bool,
         buf: &wayland_client::protocol::wl_buffer::WlBuffer,
@@ -253,6 +228,7 @@ impl CaptureSource for CapExtImageCopy {
     }
 
     fn on_done_with_frame(&self, f: Self::Frame) {
+        debug!("ext_image_copy_capture_frame_v1::destroy");
         f.destroy();
     }
 }

src/cap_wlr_screencopy.rs

@@ -57,24 +57,18 @@ impl Dispatch<ZwlrScreencopyFrameV1, ()> for State<CapWlrScreencopy> {
             } => {
                 let fourcc = DrmFourcc::try_from(format).unwrap();
                 let cap = state.enc.unwrap_cap();
-                if !cap.sent_format {
-                    cap.sent_format = true;
-                    let device = cap.drm_device.clone();
-                    if state
-                        .negotiate_format(
-                            &[DmabufPotentialFormat {
-                                fourcc,
-                                modifiers: vec![DrmModifier::LINEAR],
-                            }],
-                            (dmabuf_width, dmabuf_height),
-                            device.as_deref(),
-                        )
-                        .is_none()
-                    {
-                        return; // error, which has already been reported
-                    }
-                }
-                state.on_copy_src_ready(dmabuf_width, dmabuf_height, fourcc, qhandle, capture);
+
+                let device = cap.drm_device.clone();
+                state.negotiate_format(
+                    &[DmabufPotentialFormat {
+                        fourcc,
+                        modifiers: vec![DrmModifier::LINEAR],
+                    }],
+                    (dmabuf_width, dmabuf_height),
+                    device.as_deref(),
+                    qhandle,
+                );
+                state.on_frame_allocd(qhandle, capture);
             }
             zwlr_screencopy_frame_v1::Event::Damage { .. } => {}
             zwlr_screencopy_frame_v1::Event::Buffer { .. } => {}
@@ -114,7 +108,6 @@ impl Dispatch<ZwpLinuxDmabufFeedbackV1, ()> for State<CapWlrScreencopy> {
 pub struct CapWlrScreencopy {
     screencopy_manager: ZwlrScreencopyManagerV1,
     output: WlOutput,
-    sent_format: bool,
     drm_device: Option<PathBuf>,
 }
 impl CaptureSource for CapWlrScreencopy {
@@ -134,23 +127,19 @@ impl CaptureSource for CapWlrScreencopy {
         Ok(Self {
             screencopy_manager: man,
             output,
-            sent_format: false,
             drm_device: None,
         })
     }
 
-    fn queue_copy_frame(&self, damage: bool, buf: &WlBuffer, capture: &Self::Frame) {
+    fn queue_copy(&self, damage: bool, buf: &WlBuffer, capture: &Self::Frame) {
         if damage {
             capture.copy_with_damage(buf);
         } else {
             capture.copy(buf);
         }
     }
 
-    fn queue_capture_frame(
-        &self,
-        eq: &QueueHandle<State<Self>>,
-    ) -> Option<(u32, u32, DrmFourcc, Self::Frame)> {
+    fn alloc_frame(&self, eq: &QueueHandle<State<Self>>) -> Option<Self::Frame> {
         // creating this triggers the linux_dmabuf event, which is where we allocate etc
 
         let _capture = self