From f3b66928a194b32b27fac3c3379d3d20e5966442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sun, 14 Jun 2026 22:03:25 +0300 Subject: [PATCH] librfb: Validate framebuffer update rectangles against the framebuffer size Patch provided by Junyi Liu, who also reported this. Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/work_items/5105 Part-of: --- .../gst-plugins-bad/gst/librfb/rfbdecoder.c | 136 +++++++++++++++--- 1 file changed, 115 insertions(+), 21 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst/librfb/rfbdecoder.c b/subprojects/gst-plugins-bad/gst/librfb/rfbdecoder.c index f0507a36f46..7f94a28d319 100644 --- gst/librfb/rfbdecoder.c.orig +++ gst/librfb/rfbdecoder.c @@ -45,6 +45,10 @@ static gboolean rfb_decoder_corre_encoding (RfbDecoder * decoder, gint start_x, gint start_y, gint rect_w, gint rect_h); static gboolean rfb_decoder_hextile_encoding (RfbDecoder * decoder, gint start_x, gint start_y, gint rect_w, gint rect_h); +static gboolean rfb_decoder_clip_rectangle (RfbDecoder * decoder, gint * x, + gint * y, gint * w, gint * h, gint * skip_x, gint * skip_y); +static gboolean rfb_decoder_clip_copyrect (RfbDecoder * decoder, gint * dst_x, + gint * dst_y, gint * src_x, gint * src_y, gint * w, gint * h); RfbDecoder * rfb_decoder_new (void) @@ -802,23 +806,20 @@ rfb_decoder_state_framebuffer_update_rectangle (RfbDecoder * decoder) if (!rfb_decoder_read (decoder, 12)) return FALSE; - x = RFB_GET_UINT16 (decoder->data + 0) - decoder->offset_x; - y = RFB_GET_UINT16 (decoder->data + 2) - decoder->offset_y; + x = RFB_GET_UINT16 (decoder->data + 0); + y = RFB_GET_UINT16 (decoder->data + 2); w = RFB_GET_UINT16 (decoder->data + 4); h = RFB_GET_UINT16 (decoder->data + 6); encoding = RFB_GET_UINT32 (decoder->data + 8); + x -= (gint) decoder->offset_x; + y -= (gint) decoder->offset_y; + GST_DEBUG ("update received"); GST_DEBUG ("x:%d y:%d", x, y); GST_DEBUG ("w:%d h:%d", w, h); GST_DEBUG ("encoding: %d", encoding); - if (((w * h) + (x * y)) > (decoder->width * decoder->height)) { - GST_ERROR ("Desktop resize is unsupported."); - decoder->state = NULL; - return TRUE; - } - switch (encoding) { case ENCODING_TYPE_RAW: ret = rfb_decoder_raw_encoding (decoder, x, y, w, h); @@ -857,11 +858,20 @@ static gboolean rfb_decoder_raw_encoding (RfbDecoder * decoder, gint start_x, gint start_y, gint rect_w, gint rect_h) { - gint size; + gint copy_x, copy_y, copy_w, copy_h; + gint skip_x = 0, skip_y = 0; + guint32 size; guint8 *frame, *p; guint32 raw_line_size; + guint32 copy_line_size; + + if (rect_w <= 0 || rect_h <= 0) + return TRUE; raw_line_size = rect_w * decoder->bytespp; + if (rect_h > 0 && raw_line_size > G_MAXUINT32 / rect_h) + return FALSE; + size = rect_h * raw_line_size; GST_DEBUG ("Reading %d bytes (%dx%d)", size, rect_w, rect_h); @@ -869,13 +879,23 @@ rfb_decoder_raw_encoding (RfbDecoder * decoder, gint start_x, gint start_y, if (!rfb_decoder_read (decoder, size)) return FALSE; + copy_x = start_x; + copy_y = start_y; + copy_w = rect_w; + copy_h = rect_h; + + if (!rfb_decoder_clip_rectangle (decoder, ©_x, ©_y, ©_w, ©_h, + &skip_x, &skip_y)) + return TRUE; + frame = - decoder->frame + (((start_y * decoder->rect_width) + - start_x) * decoder->bytespp); - p = decoder->data; + decoder->frame + (((copy_y * decoder->rect_width) + + copy_x) * decoder->bytespp); + p = decoder->data + (skip_y * raw_line_size) + (skip_x * decoder->bytespp); + copy_line_size = copy_w * decoder->bytespp; - while (rect_h--) { - memcpy (frame, p, raw_line_size); + while (copy_h--) { + memcpy (frame, p, copy_line_size); p += raw_line_size; frame += decoder->line_size; } @@ -887,7 +907,8 @@ static gboolean rfb_decoder_copyrect_encoding (RfbDecoder * decoder, gint start_x, gint start_y, gint rect_w, gint rect_h) { - guint16 src_x, src_y; + gint src_x, src_y; + gint copy_x, copy_y, copy_w, copy_h; gint line_width, copyrect_width; guint8 *src, *dst; @@ -895,20 +916,28 @@ rfb_decoder_copyrect_encoding (RfbDecoder * decoder, gint start_x, gint start_y, return FALSE; /* don't forget the offset */ - src_x = RFB_GET_UINT16 (decoder->data) - decoder->offset_x; - src_y = RFB_GET_UINT16 (decoder->data + 2) - decoder->offset_y; + src_x = RFB_GET_UINT16 (decoder->data) - (gint) decoder->offset_x; + src_y = RFB_GET_UINT16 (decoder->data + 2) - (gint) decoder->offset_y; GST_DEBUG ("Copyrect from %d %d", src_x, src_y); - copyrect_width = rect_w * decoder->bytespp; + copy_x = start_x; + copy_y = start_y; + copy_w = rect_w; + copy_h = rect_h; + if (!rfb_decoder_clip_copyrect (decoder, ©_x, ©_y, &src_x, &src_y, + ©_w, ©_h)) + return TRUE; + + copyrect_width = copy_w * decoder->bytespp; line_width = decoder->line_size; src = decoder->prev_frame + ((src_y * decoder->rect_width) + src_x) * decoder->bytespp; dst = - decoder->frame + ((start_y * decoder->rect_width) + - start_x) * decoder->bytespp; + decoder->frame + ((copy_y * decoder->rect_width) + + copy_x) * decoder->bytespp; - while (rect_h--) { + while (copy_h--) { memcpy (dst, src, copyrect_width); src += line_width; dst += line_width; @@ -926,6 +955,9 @@ rfb_decoder_fill_rectangle (RfbDecoder * decoder, gint x, gint y, gint w, guint32 *offset; gint i, j; + if (!rfb_decoder_clip_rectangle (decoder, &x, &y, &w, &h, NULL, NULL)) + return; + for (i = 0; i < h; i++) { offset = (guint32 *) (decoder->frame + ((x + (y + @@ -1101,6 +1133,68 @@ rfb_decoder_hextile_encoding (RfbDecoder * decoder, gint start_x, gint start_y, return TRUE; } +static gboolean +rfb_decoder_clip_rectangle (RfbDecoder * decoder, gint * x, gint * y, gint * w, + gint * h, gint * skip_x, gint * skip_y) +{ + gint x1, y1, x2, y2; + gint orig_x, orig_y; + + if (*w <= 0 || *h <= 0) + return FALSE; + + orig_x = *x; + orig_y = *y; + x1 = MAX (orig_x, 0); + y1 = MAX (orig_y, 0); + x2 = MIN (orig_x + *w, (gint) decoder->rect_width); + y2 = MIN (orig_y + *h, (gint) decoder->rect_height); + + if (x2 <= x1 || y2 <= y1) + return FALSE; + + *x = x1; + *y = y1; + *w = x2 - x1; + *h = y2 - y1; + + if (skip_x) + *skip_x = x1 - orig_x; + if (skip_y) + *skip_y = y1 - orig_y; + + return TRUE; +} + +static gboolean +rfb_decoder_clip_copyrect (RfbDecoder * decoder, gint * dst_x, gint * dst_y, + gint * src_x, gint * src_y, gint * w, gint * h) +{ + gint left, top, right, bottom; + + if (*w <= 0 || *h <= 0) + return FALSE; + + left = MAX (0, MAX (-*dst_x, -*src_x)); + top = MAX (0, MAX (-*dst_y, -*src_y)); + right = MIN (*w, MIN ((gint) decoder->rect_width - *dst_x, + (gint) decoder->rect_width - *src_x)); + bottom = MIN (*h, MIN ((gint) decoder->rect_height - *dst_y, + (gint) decoder->rect_height - *src_y)); + + if (right <= left || bottom <= top) + return FALSE; + + *dst_x += left; + *dst_y += top; + *src_x += left; + *src_y += top; + *w = right - left; + *h = bottom - top; + + return TRUE; +} + static gboolean rfb_decoder_state_set_colour_map_entries (RfbDecoder * decoder) { -- GitLab