diff --git a/lisp/mail/smtpmail.el b/lisp/mail/smtpmail.el
index dcec0b1d9e9..dc3293fb2f4 100644
--- a/lisp/mail/smtpmail.el
+++ b/lisp/mail/smtpmail.el
@@ -639,11 +639,21 @@ USER and PASSWORD should be non-nil."
 
 (cl-defmethod smtpmail-try-auth-method
   (process (_mech (eql 'xoauth2)) user password)
-  (smtpmail-command-or-throw
-   process
-   (concat "AUTH XOAUTH2 "
-           (base64-encode-string
-            (concat "user=" user "\1auth=Bearer " password "\1\1") t))))
+  (let ((ret (smtpmail-command-or-throw
+              process
+              (concat "AUTH XOAUTH2 "
+                      (base64-encode-string
+                       (concat "user=" user "\1auth=Bearer " password "\1\1")
+                       t)))))
+    (if (eq (car ret) 334)
+        ;; When a server returns 334 server challenge, it usually means
+        ;; the credentials it received were wrong (e.g. was an actual
+        ;; password instead of an access token).  In such a case, we
+        ;; should return a string with 535 to indicate a failure so that
+        ;; smtpmail will try other authentication mechanisms.  See also
+        ;; https://debbugs.gnu.org/78366.
+        (throw 'done "535 5.7.8 Authentication credentials invalid")
+      ret)))
 
 (defun smtpmail-response-code (string)
   (when string