diff --git a/gst-libs/gst/video/gstvideoencoder.c b/gst-libs/gst/video/gstvideoencoder.c
index b9dfc4607..0d85ba232 100644
--- a/gst-libs/gst/video/gstvideoencoder.c
+++ b/gst-libs/gst/video/gstvideoencoder.c
@@ -2563,6 +2563,9 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder,
   frame->distance_from_sync = priv->distance_from_sync;
   priv->distance_from_sync++;
 
+  /* We need a writable buffer for the metadata changes below */
+  frame->output_buffer = gst_buffer_make_writable (frame->output_buffer);
+
   GST_BUFFER_PTS (frame->output_buffer) = frame->pts;
   GST_BUFFER_DTS (frame->output_buffer) = frame->dts;
   GST_BUFFER_DURATION (frame->output_buffer) = frame->duration;
@@ -2709,6 +2712,9 @@ gst_video_encoder_finish_subframe (GstVideoEncoder * encoder,
 
   gst_video_encoder_infer_dts_unlocked (encoder, frame);
 
+  /* We need a writable buffer for the metadata changes below */
+  subframe_buffer = gst_buffer_make_writable (subframe_buffer);
+
   GST_BUFFER_PTS (subframe_buffer) = frame->pts;
   GST_BUFFER_DTS (subframe_buffer) = frame->dts;
   GST_BUFFER_DURATION (subframe_buffer) = frame->duration;