diff --git a/src/xdisp.c b/src/xdisp.c
index 6b0532d95f5..85b9eae36d1 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -17655,6 +17655,9 @@ try_window_reusing_current_matrix (struct window *w)
   if (w->vscroll || MATRIX_ROW_PARTIALLY_VISIBLE_P (w, start_row))
     return false;
 
+  /* Clear the desired matrix for the display below.  */
+  clear_glyph_matrix (w->desired_matrix);
+
   /* Give up if line numbers are being displayed, because reusing the
      current matrix might use the wrong width for line-number
      display.  */
@@ -17667,9 +17670,6 @@ try_window_reusing_current_matrix (struct window *w)
   start = start_row->minpos;
   start_vpos = MATRIX_ROW_VPOS (start_row, w->current_matrix);
 
-  /* Clear the desired matrix for the display below.  */
-  clear_glyph_matrix (w->desired_matrix);
-
   if (CHARPOS (new_start) <= CHARPOS (start))
     {
       /* Don't use this method if the display starts with an ellipsis