diff --git a/lisp/vc/diff-mode.el b/lisp/vc/diff-mode.el
index 77807fc4f35..8d93b31ea35 100644
--- a/lisp/vc/diff-mode.el
+++ b/lisp/vc/diff-mode.el
@@ -1620,15 +1620,16 @@ a diff with \\[diff-reverse-direction].
 \\{diff-minor-mode-map}"
   :group 'diff-mode :lighter " Diff"
   ;; FIXME: setup font-lock
-  (when diff--track-changes (track-changes-unregister diff--track-changes))
+  (when diff--track-changes
+    (track-changes-unregister diff--track-changes)
+    (setq diff--track-changes nil))
   (remove-hook 'write-contents-functions #'diff-write-contents-hooks t)
   (when diff-minor-mode
     (if (not diff-update-on-the-fly)
         (add-hook 'write-contents-functions #'diff-write-contents-hooks nil t)
-      (unless diff--track-changes
-        (setq diff--track-changes
-              (track-changes-register #'diff--track-changes-signal
-                                      :nobefore t))))))
+      (setq diff--track-changes
+            (track-changes-register #'diff--track-changes-signal
+                                    :nobefore t)))))
 
 ;;; Handy hook functions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;