From 4696869d3d57e0b28b0450515f2f3322607d845e Mon Sep 17 00:00:00 2001
From: Denis Zubarev <dvzubarev@yandex.ru>
Date: Sat, 11 Nov 2023 04:55:44 +0300
Subject: [PATCH 1/6] Improve syntax highlighting for python-ts-mode

Fix fontification of strings inside of f-strings interpolation, e.g. for
f"beg {'nested'}" - 'nested' was not fontified as string.  Do not
override the face of builtin functions (all, bytes etc.) with the
function call face.  Add missing assignment expressions (:= *=).
Fontify built-ins (dict,list,etc.) as types when they are used in type
hints.  Highlight union types (type1|type2).  Highlight base class names
in the class definition.  Fontify class patterns in case statements.
Highlight the second argument as a type in isinstance/issubclass call.
Highlight dotted decorator names.

* lisp/progmodes/python.el (python--treesit-keywords): Add compound
keyword "is not".
(python--treesit-builtin-types): New variable that stores all python
built-in types.
(python--treesit-type-regex): New variable.  Regex matches if text is
either built-in type or text starts with capital letter.
(python--treesit-builtins): Extract built-in types to other variable.
(python--treesit-fontify-string): fix f-string interpolation.  Enable
interpolation highlighting only if string-interpolation is presented
on the enabled levels of treesit-font-lock-feature-list.
(python--treesit-fontify-string-interpolation): Remove function.
(python--treesit-fontify-union-types): Fontify nested union types.
(python--treesit-fontify-union-types-strict): Fontify nested union
types, only if type identifier matches against
python--treesit-type-regex.
(python--treesit-fontify-dotted-decorator): Fontify all parts of
dotted decorator name.
(python--treesit-settings): Change/add rules.  (Bug#67061)

* test/lisp/progmodes/python-tests.el
(python-ts-tests-with-temp-buffer): Function for setting up test
buffer.
(python-ts-mode-compound-keywords-face)
(python-ts-mode-named-assignement-face-1)
(python-ts-mode-assignement-face-2)
(python-ts-mode-nested-types-face-1)
(python-ts-mode-union-types-face-1)
(python-ts-mode-union-types-face-2)
(python-ts-mode-types-face-1)
(python-ts-mode-types-face-2)
(python-ts-mode-types-face-3)
(python-ts-mode-isinstance-type-face-1)
(python-ts-mode-isinstance-type-face-2)
(python-ts-mode-isinstance-type-face-3)
(python-ts-mode-superclass-type-face)
(python-ts-mode-class-patterns-face)
(python-ts-mode-dotted-decorator-face-1)
(python-ts-mode-dotted-decorator-face-2)
(python-ts-mode-builtin-call-face)
(python-ts-mode-interpolation-nested-string)
(python-ts-mode-disabled-string-interpolation)
(python-ts-mode-interpolation-doc-string): Add tests.
---
 lisp/progmodes/python.el            | 238 ++++++++++++++++------
 test/lisp/progmodes/python-tests.el | 302 ++++++++++++++++++++++++++++
 2 files changed, 479 insertions(+), 61 deletions(-)

diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index d7250148fad..f5f89ad552c 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -969,19 +969,30 @@ It makes underscores and dots word constituent chars.")
     "raise" "return" "try" "while" "with" "yield"
     ;; These are technically operators, but we fontify them as
     ;; keywords.
-    "and" "in" "is" "not" "or" "not in"))
+    "and" "in" "is" "not" "or" "not in" "is not"))
+
+(defvar python--treesit-builtin-types
+  '("int" "float" "complex" "bool" "list" "tuple" "range" "str"
+    "bytes" "bytearray" "memoryview" "set" "frozenset" "dict"))
+
+(defvar python--treesit-type-regex
+  (rx-to-string `(seq bol (or
+                           ,@python--treesit-builtin-types
+                           (seq (?  "_") (any "A-Z") (+ (any "a-zA-Z_0-9"))))
+                  eol)))
 
 (defvar python--treesit-builtins
-  '("abs" "all" "any" "ascii" "bin" "bool" "breakpoint" "bytearray"
-    "bytes" "callable" "chr" "classmethod" "compile" "complex"
-    "delattr" "dict" "dir" "divmod" "enumerate" "eval" "exec"
-    "filter" "float" "format" "frozenset" "getattr" "globals"
-    "hasattr" "hash" "help" "hex" "id" "input" "int" "isinstance"
-    "issubclass" "iter" "len" "list" "locals" "map" "max"
-    "memoryview" "min" "next" "object" "oct" "open" "ord" "pow"
-    "print" "property" "range" "repr" "reversed" "round" "set"
-    "setattr" "slice" "sorted" "staticmethod" "str" "sum" "super"
-    "tuple" "type" "vars" "zip" "__import__"))
+  (append python--treesit-builtin-types
+          '("abs" "all" "any" "ascii" "bin" "breakpoint"
+            "callable" "chr" "classmethod" "compile"
+            "delattr" "dir" "divmod" "enumerate" "eval" "exec"
+            "filter" "format" "getattr" "globals"
+            "hasattr" "hash" "help" "hex" "id" "input" "isinstance"
+            "issubclass" "iter" "len" "locals" "map" "max"
+            "min" "next" "object" "oct" "open" "ord" "pow"
+            "print" "property" "repr" "reversed" "round"
+            "setattr" "slice" "sorted" "staticmethod" "sum" "super"
+            "type" "vars" "zip" "__import__")))
 
 (defvar python--treesit-constants
   '("Ellipsis" "False" "None" "NotImplemented" "True" "__debug__"
@@ -1032,9 +1043,7 @@ NODE is the string node.  Do not fontify the initial f for
 f-strings.  OVERRIDE is the override flag described in
 `treesit-font-lock-rules'.  START and END mark the region to be
 fontified."
-  (let* ((string-beg (treesit-node-start node))
-         (string-end (treesit-node-end node))
-         (maybe-expression (treesit-node-parent node))
+  (let* ((maybe-expression (treesit-node-parent node))
          (grandparent (treesit-node-parent
                        (treesit-node-parent
                         maybe-expression)))
@@ -1062,28 +1071,92 @@ fontified."
                         (equal (treesit-node-type maybe-expression)
                                "expression_statement"))
                    'font-lock-doc-face
-                 'font-lock-string-face)))
-    ;; Don't highlight string prefixes like f/r/b.
-    (save-excursion
-      (goto-char string-beg)
-      (when (re-search-forward "[\"']" string-end t)
-        (setq string-beg (match-beginning 0))))
-    (treesit-fontify-with-override
-     string-beg string-end face override start end)))
+                 'font-lock-string-face))
 
-(defun python--treesit-fontify-string-interpolation
-    (node _ start end &rest _)
-  "Fontify string interpolation.
-NODE is the string node.  Do not fontify the initial f for
-f-strings.  START and END mark the region to be
+         (ignore-interpolation (not
+                                (seq-some
+                                 (lambda (feats) (memq 'string-interpolation feats))
+                                 (seq-take treesit-font-lock-feature-list treesit-font-lock-level))))
+         ;; If interpolation is enabled, highlight only
+         ;; string_start/string_content/string_end children.  Do not
+         ;; touch interpolation node that can occur inside of the
+         ;; string.
+         (string-nodes (if ignore-interpolation
+                           (list node)
+                         (treesit-filter-child
+                          node
+                          (lambda (ch) (member (treesit-node-type ch)
+                                               '("string_start"
+                                                 "string_content"
+                                                 "string_end")))
+                          t))))
+
+    (dolist (string-node string-nodes)
+      (let ((string-beg (treesit-node-start string-node))
+            (string-end (treesit-node-end string-node)))
+        (when (or ignore-interpolation
+                  (equal (treesit-node-type string-node) "string_start"))
+          ;; Don't highlight string prefixes like f/r/b.
+          (save-excursion
+            (goto-char string-beg)
+            (when (re-search-forward "[\"']" string-end t)
+              (setq string-beg (match-beginning 0)))))
+
+        (treesit-fontify-with-override
+         string-beg string-end face override start end)))))
+
+(defun python--treesit-fontify-union-types (node override start end &optional type-regex &rest _)
+  "Fontify nested union types in the type hints.
+For examlpe, Lvl1 | Lvl2[Lvl3[Lvl4[Lvl5 | None]], Lvl2].  This
+structure is represented via nesting binary_operator and
+subscript nodes.  This function iterates over all levels and
+highlight identifier nodes. If TYPE-REGEX is not nil fontify type
+identifier only if it matches against TYPE-REGEX. NODE is the
+binary_operator node.  OVERRIDE is the override flag described in
+`treesit-font-lock-rules'.  START and END mark the region to be
 fontified."
-  ;; This is kind of a hack, it basically removes the face applied by
-  ;; the string feature, so that following features can apply their
-  ;; face.
-  (let ((n-start (treesit-node-start node))
-        (n-end (treesit-node-end node)))
-    (remove-text-properties
-     (max start n-start) (min end n-end) '(face))))
+  (dolist (child (treesit-node-children node t))
+    (let (font-node)
+      (pcase (treesit-node-type child)
+        ((or "identifier" "none")
+         (setq font-node child))
+        ("attribute"
+         (when-let ((type-node (treesit-node-child-by-field-name child "attribute")))
+           (setq font-node type-node)))
+        ((or "binary_operator" "subscript")
+         (python--treesit-fontify-union-types child override start end type-regex)))
+
+      (when (and font-node
+                 (or (null type-regex)
+                     (let ((case-fold-search nil))
+                       (string-match-p type-regex (treesit-node-text font-node)))))
+        (treesit-fontify-with-override
+         (treesit-node-start font-node) (treesit-node-end font-node)
+         'font-lock-type-face override start end)))))
+
+(defun python--treesit-fontify-union-types-strict (node override start end &rest _)
+  "Fontify nested union types.
+Same as `python--treesit-fontify-union-types' but type identifier
+should match against `python--treesit-type-regex'.  For NODE,
+OVERRIDE, START and END description see
+`python--treesit-fontify-union-types'."
+  (python--treesit-fontify-union-types node override start end python--treesit-type-regex))
+
+(defun python--treesit-fontify-dotted-decorator (node override start end &rest _)
+  "Fontify dotted decorators.
+For example @pytes.mark.skip.  Iterate over all nested attribute
+nodes and highlight identifier nodes.  NODE is the first attribute
+node.  OVERRIDE is the override flag described in
+`treesit-font-lock-rules'.  START and END mark the region to be
+fontified."
+  (dolist (child (treesit-node-children node t))
+    (pcase (treesit-node-type child)
+      ("identifier"
+       (treesit-fontify-with-override
+        (treesit-node-start child) (treesit-node-end child)
+        'font-lock-type-face override start end))
+      ("attribute"
+       (python--treesit-fontify-dotted-decorator child override start end)))))
 
 (defvar python--treesit-settings
   (treesit-font-lock-rules
@@ -1093,14 +1166,9 @@ fontified."
 
    :feature 'string
    :language 'python
-   '((string) @python--treesit-fontify-string)
+   '((string) @python--treesit-fontify-string
+     (interpolation ["{" "}"] @font-lock-misc-punctuation-face))
 
-   ;; HACK: This feature must come after the string feature and before
-   ;; other features.  Maybe we should make string-interpolation an
-   ;; option rather than a feature.
-   :feature 'string-interpolation
-   :language 'python
-   '((interpolation) @python--treesit-fontify-string-interpolation)
 
    :feature 'keyword
    :language 'python
@@ -1117,12 +1185,6 @@ fontified."
      (parameters (identifier) @font-lock-variable-name-face)
      (parameters (default_parameter name: (identifier) @font-lock-variable-name-face)))
 
-   :feature 'function
-   :language 'python
-   '((call function: (identifier) @font-lock-function-call-face)
-     (call function: (attribute
-                      attribute: (identifier) @font-lock-function-call-face)))
-
    :feature 'builtin
    :language 'python
    `(((identifier) @font-lock-builtin-face
@@ -1133,6 +1195,19 @@ fontified."
                       eol))
               @font-lock-builtin-face)))
 
+   :feature 'decorator
+   :language 'python
+   '((decorator "@" @font-lock-type-face)
+     (decorator (call function: (identifier) @font-lock-type-face))
+     (decorator (identifier) @font-lock-type-face)
+     (decorator [(attribute) (call (attribute))] @python--treesit-fontify-dotted-decorator))
+
+   :feature 'function
+   :language 'python
+   '((call function: (identifier) @font-lock-function-call-face)
+     (call function: (attribute
+                      attribute: (identifier) @font-lock-function-call-face)))
+
    :feature 'constant
    :language 'python
    '([(true) (false) (none)] @font-lock-constant-face)
@@ -1144,30 +1219,71 @@ fontified."
                  @font-lock-variable-name-face)
      (assignment left: (attribute
                         attribute: (identifier)
-                        @font-lock-property-use-face))
-     (pattern_list (identifier)
+                        @font-lock-variable-name-face))
+     (augmented_assignment left: (identifier)
+                           @font-lock-variable-name-face)
+     (named_expression name: (identifier)
+                       @font-lock-variable-name-face)
+     (pattern_list [(identifier)
+                    (list_splat_pattern (identifier))]
                    @font-lock-variable-name-face)
-     (tuple_pattern (identifier)
+     (tuple_pattern [(identifier)
+                     (list_splat_pattern (identifier))]
                     @font-lock-variable-name-face)
-     (list_pattern (identifier)
-                   @font-lock-variable-name-face)
-     (list_splat_pattern (identifier)
-                         @font-lock-variable-name-face))
+     (list_pattern [(identifier)
+                    (list_splat_pattern (identifier))]
+                   @font-lock-variable-name-face))
 
-   :feature 'decorator
-   :language 'python
-   '((decorator "@" @font-lock-type-face)
-     (decorator (call function: (identifier) @font-lock-type-face))
-     (decorator (identifier) @font-lock-type-face))
 
    :feature 'type
    :language 'python
+   ;; Override built-in faces when dict/list are used for type hints.
+   :override t
    `(((identifier) @font-lock-type-face
       (:match ,(rx-to-string
                 `(seq bol (or ,@python--treesit-exceptions)
-                      eol))
+                  eol))
               @font-lock-type-face))
-     (type (identifier) @font-lock-type-face))
+     (type [(identifier) (none)] @font-lock-type-face)
+     (type (attribute attribute: (identifier) @font-lock-type-face))
+     ;; We don't want to highlight a package of the type
+     ;; (e.g. pack.ClassName).  So explicitly exclude patterns with
+     ;; attribute, since we handle dotted type name in the previous
+     ;; rule.  The following rule handle
+     ;; generic_type/list/tuple/splat_type nodes.
+     (type (_ !attribute [[(identifier) (none)] @font-lock-type-face
+                          (attribute attribute: (identifier) @font-lock-type-face) ]))
+     ;; collections.abc.Iterator[T] case.
+     (type (subscript (attribute attribute: (identifier) @font-lock-type-face)))
+     ;; Nested optional type hints, e.g. val: Lvl1 | Lvl2[Lvl3[Lvl4]].
+     (type (binary_operator) @python--treesit-fontify-union-types)
+     ;;class Type(Base1, Sequence[T]).
+     (class_definition
+      superclasses:
+      (argument_list [(identifier) @font-lock-type-face
+                      (attribute attribute: (identifier) @font-lock-type-face)
+                      (subscript (identifier) @font-lock-type-face)
+                      (subscript (attribute attribute: (identifier) @font-lock-type-face))]))
+
+     ;; Patern matching: case [str(), pack0.Type0()].  Take only the
+     ;; last identifier.
+     (class_pattern (dotted_name (identifier) @font-lock-type-face :anchor))
+
+     ;; Highlight the second argument as a type in isinstance/issubclass.
+     ((call function: (identifier) @func-name
+            (argument_list :anchor (_)
+                           [(identifier) @font-lock-type-face
+                            (attribute attribute: (identifier) @font-lock-type-face)
+                            (tuple (identifier) @font-lock-type-face)
+                            (tuple (attribute attribute: (identifier) @font-lock-type-face))]
+                           (:match ,python--treesit-type-regex @font-lock-type-face)))
+      (:match "^is\\(?:instance\\|subclass\\)$" @func-name))
+
+     ;; isinstance(t, int|float).
+     ((call function: (identifier) @func-name
+            (argument_list :anchor (_)
+                           (binary_operator) @python--treesit-fontify-union-types-strict))
+      (:match "^is\\(?:instance\\|subclass\\)$" @func-name)))
 
    :feature 'escape-sequence
    :language 'python
diff --git a/test/lisp/progmodes/python-tests.el b/test/lisp/progmodes/python-tests.el
index e1b4c0a74c0..59287970ca0 100644
--- a/test/lisp/progmodes/python-tests.el
+++ b/test/lisp/progmodes/python-tests.el
@@ -7122,6 +7122,308 @@ buffer with overlapping strings."
                          "Unused import a.b.c (unused-import)"
                        "W0611: Unused import a.b.c (unused-import)"))))))
 
+;;; python-ts-mode font-lock tests
+
+(defmacro python-ts-tests-with-temp-buffer (contents &rest body)
+  "Create a `python-ts-mode' enabled temp buffer with CONTENTS.
+BODY is code to be executed within the temp buffer.  Point is
+always located at the beginning of buffer."
+  (declare (indent 1) (debug t))
+  `(with-temp-buffer
+     (skip-unless (treesit-ready-p 'python))
+     (require 'python)
+     (let ((python-indent-guess-indent-offset nil))
+       (python-ts-mode)
+       (setopt treesit-font-lock-level 3)
+       (insert ,contents)
+       (font-lock-ensure)
+       (goto-char (point-min))
+       ,@body)))
+
+(ert-deftest python-ts-mode-compound-keywords-face ()
+  (dolist (test '("is not" "not in"))
+    (python-ts-tests-with-temp-buffer
+     (concat "t " test " t")
+     (forward-to-word 1)
+     (should (eq (face-at-point) font-lock-keyword-face))
+     (forward-to-word 1)
+     (should (eq (face-at-point) font-lock-keyword-face)))))
+
+(ert-deftest python-ts-mode-named-assignement-face-1 ()
+  (python-ts-tests-with-temp-buffer
+   "var := 3"
+   (should (eq (face-at-point) font-lock-variable-name-face))))
+
+(ert-deftest python-ts-mode-assignement-face-2 ()
+  (python-ts-tests-with-temp-buffer
+   "var, *rest = call()"
+   (dolist (test '("var" "rest"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-variable-name-face))))
+
+  (python-ts-tests-with-temp-buffer
+   "def func(*args):"
+   (dolist (test '("args"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-variable-name-face))))))
+
+(ert-deftest python-ts-mode-nested-types-face-1 ()
+  (python-ts-tests-with-temp-buffer
+   "def func(v:dict[ list[ tuple[str] ], int | None] | None):"
+   (dolist (test '("dict" "list" "tuple" "str" "int" "None" "None"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))))
+
+(ert-deftest python-ts-mode-union-types-face-1 ()
+  (python-ts-tests-with-temp-buffer
+   "def f(val: tuple[tuple, list[Lvl1 | Lvl2[Lvl3[Lvl4[Lvl5 | None]], Lvl2]]]):"
+   (dolist (test '("tuple" "tuple" "list" "Lvl1" "Lvl2" "Lvl3" "Lvl4" "Lvl5" "None" "Lvl2"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))))
+
+(ert-deftest python-ts-mode-union-types-face-2 ()
+  (python-ts-tests-with-temp-buffer
+   "def f(val: Type0 | Type1[Type2, pack0.Type3] | pack1.pack2.Type4 | None):"
+   (dolist (test '("Type0" "Type1" "Type2" "Type3" "Type4" "None"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))
+
+   (goto-char (point-min))
+   (dolist (test '("pack0" "pack1" "pack2"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-type-face))))))
+
+(ert-deftest python-ts-mode-types-face-1 ()
+  (python-ts-tests-with-temp-buffer
+   "def f(val: Callable[[Type0], (Type1, Type2)]):"
+   (dolist (test '("Callable" "Type0" "Type1" "Type2"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))))
+
+(ert-deftest python-ts-mode-types-face-2 ()
+  (python-ts-tests-with-temp-buffer
+   "def annot3(val:pack0.Type0)->pack1.pack2.pack3.Type1:"
+   (dolist (test '("Type0" "Type1"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))
+   (goto-char (point-min))
+   (dolist (test '("pack0" "pack1" "pack2" "pack3"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-type-face))))))
+
+(ert-deftest python-ts-mode-types-face-3 ()
+  (python-ts-tests-with-temp-buffer
+   "def annot3(val:collections.abc.Iterator[Type0]):"
+   (dolist (test '("Iterator" "Type0"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))
+   (goto-char (point-min))
+   (dolist (test '("collections" "abc"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-type-face))))))
+
+(ert-deftest python-ts-mode-isinstance-type-face-1 ()
+  (python-ts-tests-with-temp-buffer
+   "isinstance(var1, pkg.Type0)
+    isinstance(var2, (str, dict, Type1, type(None)))
+    isinstance(var3, my_type())"
+
+   (dolist (test '("var1" "pkg" "var2" "type" "None" "var3" "my_type"))
+     (let ((case-fold-search nil))
+       (search-forward test))
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-type-face))))
+
+   (goto-char (point-min))
+   (dolist (test '("Type0" "str" "dict" "Type1"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))))
+
+(ert-deftest python-ts-mode-isinstance-type-face-2 ()
+  (python-ts-tests-with-temp-buffer
+   "issubclass(mytype, int|list|collections.abc.Iterable)"
+   (dolist (test '("int" "list" "Iterable"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))))
+
+(ert-deftest python-ts-mode-isinstance-type-face-3 ()
+  (python-ts-tests-with-temp-buffer
+   "issubclass(mytype, typevar1)
+    isinstance(mytype, (Type1, typevar2, tuple, abc.Coll))
+    isinstance(mytype, pkg0.Type2|self.typevar3|typevar4)"
+
+   (dolist (test '("typevar1" "typevar2" "pkg0" "self" "typevar3" "typevar4"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-type-face))))
+
+   (goto-char (point-min))
+   (dolist (test '("Type1" "tuple" "Coll" "Type2"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))))
+
+(ert-deftest python-ts-mode-superclass-type-face ()
+  (python-ts-tests-with-temp-buffer
+   "class Temp(Base1, pack0.Base2,  Sequence[T1, T2]):"
+
+   (dolist (test '("Base1" "Base2" "Sequence" "T1" "T2"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))
+
+   (goto-char (point-min))
+   (dolist (test '("pack0"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-type-face))))))
+
+(ert-deftest python-ts-mode-class-patterns-face ()
+  (python-ts-tests-with-temp-buffer
+   "match tt:
+        case str():
+            pass
+        case [Type0() | bytes(b) | pack0.pack1.Type1()]:
+            pass
+        case {'i': int(i), 'f': float() as f}:
+            pass"
+
+   (dolist (test '("str" "Type0" "bytes" "Type1" "int" "float"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))
+
+   (goto-char (point-min))
+   (dolist (test '("pack0" "pack1"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-type-face))))))
+
+(ert-deftest python-ts-mode-dotted-decorator-face-1 ()
+  (python-ts-tests-with-temp-buffer
+   "@pytest.mark.skip
+    @pytest.mark.skip(reason='msg')
+    def test():"
+
+   (dolist (test '("pytest" "mark" "skip" "pytest" "mark" "skip"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))))
+
+(ert-deftest python-ts-mode-dotted-decorator-face-2 ()
+  (python-ts-tests-with-temp-buffer
+   "@pytest.mark.skip(reason='msg')
+    def test():"
+
+   (setopt treesit-font-lock-level 4)
+   (dolist (test '("pytest" "mark" "skip"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-type-face)))))
+
+(ert-deftest python-ts-mode-builtin-call-face ()
+  (python-ts-tests-with-temp-buffer
+   "all()"
+   ;; enable 'function' feature from 4th level
+   (setopt treesit-font-lock-level 4)
+   (should (eq (face-at-point) font-lock-builtin-face))))
+
+(ert-deftest python-ts-mode-interpolation-nested-string ()
+  (python-ts-tests-with-temp-buffer
+   "t = f\"beg {True + 'string'}\""
+
+   (search-forward "True")
+   (goto-char (match-beginning 0))
+   (should (eq (face-at-point) font-lock-constant-face))
+
+   (goto-char (point-min))
+   (dolist (test '("f" "{" "+" "}"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-string-face))))
+
+
+   (goto-char (point-min))
+   (dolist (test '("beg" "'string'" "\""))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-string-face)))))
+
+(ert-deftest python-ts-mode-level-fontification-wo-interpolation ()
+  (python-ts-tests-with-temp-buffer
+   "t = f\"beg {True + var}\""
+
+   (setopt treesit-font-lock-level 2)
+   (search-forward "f")
+   (goto-char (match-beginning 0))
+   (should (not (eq (face-at-point) font-lock-string-face)))
+
+   (dolist (test '("\"" "beg" "{" "True" "var" "}" "\""))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-string-face)))))
+
+(ert-deftest python-ts-mode-disabled-string-interpolation ()
+  (python-ts-tests-with-temp-buffer
+   "t = f\"beg {True + var}\""
+
+   (unwind-protect
+       (progn
+         (setf (nth 2 treesit-font-lock-feature-list)
+               (remq 'string-interpolation (nth 2 treesit-font-lock-feature-list)))
+         (setopt treesit-font-lock-level 3)
+
+         (search-forward "f")
+         (goto-char (match-beginning 0))
+         (should (not (eq (face-at-point) font-lock-string-face)))
+
+         (dolist (test '("\"" "beg" "{" "True" "var" "}" "\""))
+           (search-forward test)
+           (goto-char (match-beginning 0))
+           (should (eq (face-at-point) font-lock-string-face))))
+
+    (setf (nth 2 treesit-font-lock-feature-list)
+          (append (nth 2 treesit-font-lock-feature-list) '(string-interpolation))))))
+
+(ert-deftest python-ts-mode-interpolation-doc-string ()
+  (python-ts-tests-with-temp-buffer
+   "f\"\"\"beg {'s1' + True + 's2'} end\"\"\""
+
+   (search-forward "True")
+   (goto-char (match-beginning 0))
+   (should (eq (face-at-point) font-lock-constant-face))
+
+   (goto-char (point-min))
+   (dolist (test '("f" "{" "+" "}"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (not (eq (face-at-point) font-lock-string-face))))
+
+   (goto-char (point-min))
+   (dolist (test '("\"\"\"" "beg" "end" "\"\"\""))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-doc-face)))
+
+   (goto-char (point-min))
+   (dolist (test '("'s1'" "'s2'"))
+     (search-forward test)
+     (goto-char (match-beginning 0))
+     (should (eq (face-at-point) font-lock-string-face)))))
+
 (provide 'python-tests)
 
 ;;; python-tests.el ends here

From 55555a6a0d1d76468f8327972b3cb067b9e35f24 Mon Sep 17 00:00:00 2001
From: Stefan Kangas <stefankangas@gmail.com>
Date: Sat, 30 Dec 2023 17:53:26 +0100
Subject: [PATCH 2/6] org-protocol: Minor copy-edits to Commentary

* lisp/org/org-protocol.el: Minor copy-edits to Commentary.
---
 lisp/org/org-protocol.el | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/lisp/org/org-protocol.el b/lisp/org/org-protocol.el
index 2b07a377e2a..15132196290 100644
--- a/lisp/org/org-protocol.el
+++ b/lisp/org/org-protocol.el
@@ -34,7 +34,10 @@
 ;; `org-protocol-protocol-alist' and `org-protocol-protocol-alist-default'.
 ;;
 ;; Any application that supports calling external programs with an URL
-;; as argument may be used with this functionality.
+;; as argument could use this functionality.  For example, you can
+;; configure bookmarks in your web browser to send a link to the
+;; current page to Org and create a note from it using `org-capture'.
+;; See Info node `(org) Protocols' for more information.
 ;;
 ;;
 ;; Usage:
@@ -44,13 +47,13 @@
 ;;
 ;;       (require 'org-protocol)
 ;;
-;;   3.) Ensure emacs-server is up and running.
-;;   4.) Try this from the command line (adjust the URL as needed):
+;;   2.) Ensure emacs-server is up and running.
+;;   3.) Try this from the command line (adjust the URL as needed):
 ;;
 ;;       $ emacsclient \
 ;;         "org-protocol://store-link?url=http:%2F%2Flocalhost%2Findex.html&title=The%20title"
 ;;
-;;   5.) Optionally add custom sub-protocols and handlers:
+;;   4.) Optionally, add custom sub-protocols and handlers:
 ;;
 ;;       (setq org-protocol-protocol-alist
 ;;             '(("my-protocol"
@@ -64,10 +67,11 @@
 ;; If it works, you can now setup other applications for using this feature.
 ;;
 ;;
-;; As of March 2009 Firefox users follow the steps documented on
-;; https://kb.mozillazine.org/Register_protocol, Opera setup is described here:
-;; http://www.opera.com/support/kb/view/535/
+;; Firefox users follow the steps documented on
+;; https://kb.mozillazine.org/Register_protocol, Opera setup is
+;; described here: http://www.opera.com/support/kb/view/535/
 ;;
+;; See also: https://orgmode.org/worg/org-contrib/org-protocol.html
 ;;
 ;; Documentation
 ;; -------------
@@ -123,9 +127,6 @@
 ;; Note that using double slashes is optional from org-protocol.el's point of
 ;; view because emacsclient squashes the slashes to one.
 ;;
-;;
-;; provides: 'org-protocol
-;;
 ;;; Code:
 
 (require 'org-macs)

From 01be4fe39d7029295d09b0e5e46b9239ab4600bc Mon Sep 17 00:00:00 2001
From: Eli Zaretskii <eliz@gnu.org>
Date: Sun, 31 Dec 2023 09:44:32 +0200
Subject: [PATCH 3/6] * doc/emacs/custom.texi (Modifier Keys): Fix markup
 (bug#68164).

Suggested by Jens Quade <jq@qdevelop.de>.
---
 doc/emacs/custom.texi | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/emacs/custom.texi b/doc/emacs/custom.texi
index ba15e205b0f..221da5c486f 100644
--- a/doc/emacs/custom.texi
+++ b/doc/emacs/custom.texi
@@ -2035,7 +2035,7 @@ C-a} is a way to enter @kbd{Hyper-Control-a}.  (Unfortunately, there
 is no way to add two modifiers by using @kbd{C-x @@} twice for the
 same character, because the first one goes to work on the @kbd{C-x}.)
 You can similarly enter the Shift, Control, and Meta modifiers by
-using @kbd{C-x @ S}, @kbd{C-x @ c}, and @kbd{C-x @ m}, respectively,
+using @kbd{C-x @@ S}, @kbd{C-x @@ c}, and @kbd{C-x @@ m}, respectively,
 although this is rarely needed.
 
 @node Function Keys

From 240b4594f11ee14c91f4a37d0b3ff4625e79f19c Mon Sep 17 00:00:00 2001
From: Eli Zaretskii <eliz@gnu.org>
Date: Sun, 31 Dec 2023 15:17:18 +0200
Subject: [PATCH 4/6] ; * etc/TODO: Add an item about
 'Info-hide-note-references'.

---
 etc/TODO | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/etc/TODO b/etc/TODO
index f1b3373e9e5..ff742b79bec 100644
--- a/etc/TODO
+++ b/etc/TODO
@@ -133,6 +133,20 @@ This should use a heuristic of some kind?
 ** In Emacs Info, examples of using Customize should be clickable
 They should create Custom buffers when clicked.
 
+** Replacements under 'Info-hide-note-references' should be language-sensitive
+Currently, we replace the "*note" cross-reference indicators with a
+hard-coded "see", which is English-centric and doesn't look well in
+manuals written in languages other than English.  To fix this, we need
+a change in the Texinfo's 'makeinfo' program so that it records the
+document's language (specified via the @documentlanguage directive in
+Texinfo) in a variable in the Local Variables section of the produced
+Info file.  Then 'Info-fontify-node' should be modified to look up the
+translation of "see" to that language in a database (which should be
+added), and should use that translation instead of "see".  See this
+discussion on the Texinfo mailing list for more details:
+
+  https://lists.gnu.org/archive/html/help-texinfo/2023-12/msg00011.html
+
 ** Add function to redraw the tool bar
 
 ** Redesign the load-history data structure

From 7591acfe38e3f5f3fb9b89e2b1ed08723b0298e6 Mon Sep 17 00:00:00 2001
From: Kyle Meyer <kyle@kyleam.com>
Date: Mon, 1 Jan 2024 12:47:27 -0500
Subject: [PATCH 5/6] Update to Org 9.6.15

---
 doc/misc/org.org         |  4 ++--
 etc/refcards/orgcard.tex |  2 +-
 lisp/org/org-entities.el | 21 +++++++++++++--------
 lisp/org/org-macs.el     | 15 ++++++++++-----
 lisp/org/org-version.el  |  4 ++--
 lisp/org/org.el          |  2 +-
 lisp/org/ox-md.el        |  2 +-
 7 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/doc/misc/org.org b/doc/misc/org.org
index 9f6cda17da0..ce83f57793d 100644
--- a/doc/misc/org.org
+++ b/doc/misc/org.org
@@ -4606,7 +4606,7 @@ checked.
 #+cindex: statistics, for checkboxes
 #+cindex: checkbox statistics
 #+cindex: @samp{COOKIE_DATA}, property
-#+vindex: org-hierarchical-checkbox-statistics
+#+vindex: org-checkbox-hierarchical-statistics
 The =[2/4]= and =[1/3]= in the first and second line are cookies
 indicating how many checkboxes present in this entry have been checked
 off, and the total number of checkboxes present.  This can give you an
@@ -4614,7 +4614,7 @@ idea on how many checkboxes remain, even without opening a folded
 entry.  The cookies can be placed into a headline or into (the first
 line of) a plain list item.  Each cookie covers checkboxes of direct
 children structurally below the headline/item on which the cookie
-appears[fn:: Set the variable ~org-hierarchical-checkbox-statistics~
+appears[fn:: Set the variable ~org-checkbox-hierarchical-statistics~
 if you want such cookies to count all checkboxes below the cookie, not
 just those belonging to direct children.].  You have to insert the
 cookie yourself by typing either =[/]= or =[%]=.  With =[/]= you get
diff --git a/etc/refcards/orgcard.tex b/etc/refcards/orgcard.tex
index 11e046fc0dd..c757d343bc8 100644
--- a/etc/refcards/orgcard.tex
+++ b/etc/refcards/orgcard.tex
@@ -1,5 +1,5 @@
 % Reference Card for Org Mode
-\def\orgversionnumber{9.6.13}
+\def\orgversionnumber{9.6.15}
 \def\versionyear{2023}          % latest update
 \input emacsver.tex
 
diff --git a/lisp/org/org-entities.el b/lisp/org/org-entities.el
index 61083022b82..195374e9fff 100644
--- a/lisp/org/org-entities.el
+++ b/lisp/org/org-entities.el
@@ -41,14 +41,19 @@
 
 (defun org-entities--user-safe-p (v)
   "Non-nil if V is a safe value for `org-entities-user'."
-  (pcase v
-    (`nil t)
-    (`(,(and (pred stringp)
-	     (pred (string-match-p "\\`[a-zA-Z][a-zA-Z0-9]*\\'")))
-       ,(pred stringp) ,(pred booleanp) ,(pred stringp)
-       ,(pred stringp) ,(pred stringp) ,(pred stringp))
-     t)
-    (_ nil)))
+  (cond
+   ((not v) t)
+   ((listp v)
+    (seq-every-p
+     (lambda (e)
+       (pcase e
+         (`(,(and (pred stringp)
+	              (pred (string-match-p "\\`[a-zA-Z][a-zA-Z0-9]*\\'")))
+            ,(pred stringp) ,(pred booleanp) ,(pred stringp)
+            ,(pred stringp) ,(pred stringp) ,(pred stringp))
+          t)
+         (_ nil)))
+     v))))
 
 (defcustom org-entities-user nil
   "User-defined entities used in Org to produce special characters.
diff --git a/lisp/org/org-macs.el b/lisp/org/org-macs.el
index 6ed901b7397..fd083ca6250 100644
--- a/lisp/org/org-macs.el
+++ b/lisp/org/org-macs.el
@@ -56,8 +56,8 @@ by `package-activate-all').")
   ;; `org-assert-version' calls would fail using strict
   ;; `org-git-version' check because the generated Org version strings
   ;; will not match.
-  `(unless (or org--inhibit-version-check (equal (org-release) ,(org-release)))
-     (warn "Org version mismatch.  Org loading aborted.
+  `(unless (or ,org--inhibit-version-check (equal (org-release) ,(org-release)))
+     (warn "Org version mismatch.
 This warning usually appears when a built-in Org version is loaded
 prior to the more recent Org version.
 
@@ -91,10 +91,15 @@ Version mismatch is commonly encountered in the following situations:
    early in the config.  Ideally, right after the straight.el
    bootstrap.  Moving `use-package' :straight declaration may not be
    sufficient if the corresponding `use-package' statement is
-   deferring the loading."
+   deferring the loading.
+
+4. A new Org version is synchronized with Emacs git repository and
+   stale .elc files are still left from the previous build.
+
+   It is recommended to remove .elc files from lisp/org directory and
+   re-compile."
            ;; Avoid `warn' replacing "'" with "’" (see `format-message').
-           "(straight-use-package 'org)")
-     (error "Org version mismatch.  Make sure that correct `load-path' is set early in init.el")))
+           "(straight-use-package 'org)")))
 
 ;; We rely on org-macs when generating Org version.  Checking Org
 ;; version here will interfere with Org build process.
diff --git a/lisp/org/org-version.el b/lisp/org/org-version.el
index 8eebdbe09b2..b41756ac53e 100644
--- a/lisp/org/org-version.el
+++ b/lisp/org/org-version.el
@@ -5,13 +5,13 @@
 (defun org-release ()
   "The release version of Org.
 Inserted by installing Org mode or when a release is made."
-   (let ((org-release "9.6.13"))
+   (let ((org-release "9.6.15"))
      org-release))
 ;;;###autoload
 (defun org-git-version ()
   "The Git version of Org mode.
 Inserted by installing Org or when a release is made."
-   (let ((org-git-version "release_9.6.13"))
+   (let ((org-git-version "release_9.6.15"))
      org-git-version))
 
 (provide 'org-version)
diff --git a/lisp/org/org.el b/lisp/org/org.el
index b94dcd07b9a..7917baf4c46 100644
--- a/lisp/org/org.el
+++ b/lisp/org/org.el
@@ -9,7 +9,7 @@
 ;; URL: https://orgmode.org
 ;; Package-Requires: ((emacs "26.1"))
 
-;; Version: 9.6.13
+;; Version: 9.6.15
 
 ;; This file is part of GNU Emacs.
 ;;
diff --git a/lisp/org/ox-md.el b/lisp/org/ox-md.el
index 5be0ca22e07..ec8e3c53ec0 100644
--- a/lisp/org/ox-md.el
+++ b/lisp/org/ox-md.el
@@ -305,7 +305,7 @@ INFO is a plist used as a communication channel."
          (section-title (org-html--translate "Footnotes" info)))
     (when fn-alist
       (format (plist-get info :md-footnotes-section)
-              (org-md--headline-title headline-style 1 section-title)
+              (org-md--headline-title headline-style (plist-get info :md-toplevel-hlevel) section-title)
               (mapconcat (lambda (fn) (org-md--footnote-formatted fn info))
                          fn-alist
                          "\n")))))

From 3204825f56040df0a783de4fc99052eabb62b84b Mon Sep 17 00:00:00 2001
From: Mike Kupfer <kupfer@rawbw.com>
Date: Sun, 31 Dec 2023 09:11:23 -0800
Subject: [PATCH 6/6] Fix mangled Subject header field when forwarding
 (Bug#67360)

* lisp/mh-e/mh-comp.el (mh-forward): Overwrite subject when
forwarding.
---
 lisp/mh-e/mh-comp.el | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lisp/mh-e/mh-comp.el b/lisp/mh-e/mh-comp.el
index 92d31bf1826..0d1dcdf626a 100644
--- a/lisp/mh-e/mh-comp.el
+++ b/lisp/mh-e/mh-comp.el
@@ -574,7 +574,7 @@ See also `mh-compose-forward-as-mime-flag',
         (setq orig-subject (mh-get-header-field "Subject:")))
       (let ((forw-subject
              (mh-forwarded-letter-subject orig-from orig-subject)))
-        (mh-insert-fields "Subject:" forw-subject)
+        (mh-modify-header-field "Subject" forw-subject t)
         (goto-char (point-min))
         ;; Set the local value of mh-mail-header-separator according to what is
         ;; present in the buffer...