diff --git a/lisp/progmodes/xref.el b/lisp/progmodes/xref.el
index 13a1600594f..bbd3940be4f 100644
--- a/lisp/progmodes/xref.el
+++ b/lisp/progmodes/xref.el
@@ -1291,8 +1291,11 @@ Such as the current syntax table and the applied syntax properties."
 
 (defun xref--collect-matches (hit regexp tmp-buffer)
   (pcase-let* ((`(,line ,file ,text) hit)
-               (file (and file (concat (file-remote-p default-directory) file)))
-               (buf (xref--find-buffer-visiting file))
+               (remote-id (file-remote-p default-directory))
+               (file (and file (concat remote-id file)))
+               (buf (unless remote-id
+                      ;; find-buffer-visiting is slow on remote.
+                      (xref--find-buffer-visiting file)))
                (syntax-needed (xref--regexp-syntax-dependent-p regexp)))
     (if buf
         (with-current-buffer buf