From 85222d4485aaaf5b308859988ac3d06212e6bf3f Mon Sep 17 00:00:00 2001 From: Dmitry Gutov Date: Tue, 24 Apr 2012 11:51:14 -0400 Subject: [PATCH 1/3] * lisp/progmodes/ruby-mode.el: Handle general delimited literals. (ruby-syntax-general-delimiters-goto-beg) (ruby-syntax-propertize-general-delimiters): New functions. (ruby-syntax-propertize-function): Use them to handle GDL. (ruby-font-lock-keywords): Move old handling of GDL... (ruby-font-lock-syntactic-keywords): .. to here. (ruby-calculate-indent): Adjust indentation for GDL. Fixes: debbugs:6286 --- lisp/ChangeLog | 10 +++++++ lisp/progmodes/ruby-mode.el | 52 ++++++++++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/lisp/ChangeLog b/lisp/ChangeLog index b1f1ad99b23..03fbfd83255 100644 --- a/lisp/ChangeLog +++ b/lisp/ChangeLog @@ -1,3 +1,13 @@ +2012-04-24 Dmitry Gutov + + * progmodes/ruby-mode.el: Handle general delimited literals (bug#6286). + (ruby-syntax-general-delimiters-goto-beg) + (ruby-syntax-propertize-general-delimiters): New functions. + (ruby-syntax-propertize-function): Use them to handle GDL. + (ruby-font-lock-keywords): Move old handling of GDL... + (ruby-font-lock-syntactic-keywords): .. to here. + (ruby-calculate-indent): Adjust indentation for GDL. + 2012-04-24 Leo Liu * progmodes/python.el: Move hideshow setup to the end. diff --git a/lisp/progmodes/ruby-mode.el b/lisp/progmodes/ruby-mode.el index 66aa256f947..05a4042b67d 100644 --- a/lisp/progmodes/ruby-mode.el +++ b/lisp/progmodes/ruby-mode.el @@ -794,8 +794,8 @@ and `\\' when preceded by `?'." ;; (not (or (eolp) (looking-at "#") ;; (and (eq (car (nth 1 state)) ?{) ;; (looking-at "|")))))) - (or (not (eq ?/ c)) - (null (nth 0 (ruby-parse-region (or begin parse-start) (point))))) + ;; not a regexp or general delimited literal + (null (nth 0 (ruby-parse-region (or begin parse-start) (point)))) (or (not (eq ?| (char-after (point)))) (save-excursion (or (eolp) (forward-char -1)) @@ -1118,6 +1118,7 @@ See `add-log-current-defun-function'." "Syntactic keywords for Ruby mode. See `syntax-propertize-function'." (goto-char start) (ruby-syntax-propertize-heredoc end) + (ruby-syntax-general-delimiters-goto-beg) (funcall (syntax-propertize-rules ;; #{ }, #$hoge, #@foo are not comments @@ -1137,7 +1138,10 @@ See `add-log-current-defun-function'." ("^\\(=\\)begin\\_>" (1 "!")) ;; Handle here documents. ((concat ruby-here-doc-beg-re ".*\\(\n\\)") - (7 (prog1 "\"" (ruby-syntax-propertize-heredoc end))))) + (7 (prog1 "\"" (ruby-syntax-propertize-heredoc end)))) + ;; Handle percent literals: %w(), %q{}, etc. + ("\\(?:^\\|[[ \t\n<+(,=]\\)\\(%\\)[qQrswWx]?\\([[:punct:]]\\)" + (1 (prog1 "|" (ruby-syntax-propertize-general-delimiters end))))) (point) end)) (defun ruby-syntax-propertize-heredoc (limit) @@ -1163,6 +1167,41 @@ See `add-log-current-defun-function'." ;; Make extra sure we don't move back, lest we could fall into an ;; inf-loop. (if (< (point) start) (goto-char start)))))) + + (defun ruby-syntax-general-delimiters-goto-beg () + (let ((state (syntax-ppss))) + ;; Move to the start of the literal, in case it's multiline. + ;; TODO: determine the literal type more reliably here? + (when (eq t (nth 3 state)) + (goto-char (nth 8 state)) + (beginning-of-line)))) + + (defun ruby-syntax-propertize-general-delimiters (limit) + (goto-char (match-beginning 2)) + (let* ((op (char-after)) + (ops (char-to-string op)) + (cl (or (cdr (aref (syntax-table) op)) + (cdr (assoc op '((?< . ?>)))))) + parse-sexp-lookup-properties) + (ignore-errors + (if cl + (progn ; paired delimiters + ;; Delimiter pairs of the same kind can be nested + ;; inside the literal, as long as they are balanced. + ;; Create syntax table that ignores other characters. + (with-syntax-table (make-char-table 'syntax-table nil) + (modify-syntax-entry op (concat "(" (char-to-string cl))) + (modify-syntax-entry cl (concat ")" ops)) + (modify-syntax-entry ?\\ "\\") + (save-restriction + (narrow-to-region (point) limit) + (forward-list)))) ; skip to the paired character + ;; single character delimiter + (re-search-forward (concat "[^\\]\\(?:\\\\\\\\\\)*" + (regexp-quote ops)) limit nil)) + ;; if we reached here, the closing delimiter was found + (put-text-property (1- (point)) (point) + 'syntax-table (string-to-syntax "|"))))) ) ;; For Emacsen where syntax-propertize-rules is not (yet) available, @@ -1207,6 +1246,10 @@ This should only be called after matching against `ruby-here-doc-end-re'." (4 (7 . ?/)) (6 (7 . ?/))) ("^=en\\(d\\)\\_>" 1 "!") + ;; general delimited string + ("\\(^\\|[[ \t\n<+(,=]\\)\\(%[xrqQwW]?\\([^<[{(a-zA-Z0-9 \n]\\)[^\n\\\\]*\\(\\\\.[^\n\\\\]*\\)*\\(\\3\\)\\)" + (3 "\"") + (5 "\"")) ("^\\(=\\)begin\\_>" 1 (ruby-comment-beg-syntax)) ;; Currently, the following case is highlighted incorrectly: ;; @@ -1415,9 +1458,6 @@ See `font-lock-syntax-table'.") 1 font-lock-variable-name-face) '("\\(\\$\\|@\\|@@\\)\\(\\w\\|_\\)+" 0 font-lock-variable-name-face) - ;; general delimited string - '("\\(^\\|[[ \t\n<+(,=]\\)\\(%[xrqQwW]?\\([^<[{(a-zA-Z0-9 \n]\\)[^\n\\\\]*\\(\\\\.[^\n\\\\]*\\)*\\(\\3\\)\\)" - (2 font-lock-string-face)) ;; constants '("\\(^\\|[^_]\\)\\b\\([A-Z]+\\(\\w\\|_\\)*\\)" 2 font-lock-type-face) From 51a8ea2acf100a3a0ab783632c5fbcdb665a2e14 Mon Sep 17 00:00:00 2001 From: Dmitry Gutov Date: Tue, 24 Apr 2012 12:00:08 -0400 Subject: [PATCH 2/3] * lisp/progmodes/ruby-mode.el: Handle Cucumber defs (bug#6286). (ruby-syntax-propertize-regexp): New function. (ruby-syntax-propertize-function): Use it to handle regexp not preceded by a special keyword. --- lisp/ChangeLog | 5 +++++ lisp/progmodes/ruby-mode.el | 20 +++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lisp/ChangeLog b/lisp/ChangeLog index 03fbfd83255..2328cf99c1a 100644 --- a/lisp/ChangeLog +++ b/lisp/ChangeLog @@ -1,5 +1,10 @@ 2012-04-24 Dmitry Gutov + * progmodes/ruby-mode.el: Handle Cucumber defs (bug#6286). + (ruby-syntax-propertize-regexp): New function. + (ruby-syntax-propertize-function): Use it to handle regexp not preceded + by a special keyword. + * progmodes/ruby-mode.el: Handle general delimited literals (bug#6286). (ruby-syntax-general-delimiters-goto-beg) (ruby-syntax-propertize-general-delimiters): New functions. diff --git a/lisp/progmodes/ruby-mode.el b/lisp/progmodes/ruby-mode.el index 05a4042b67d..8818911159b 100644 --- a/lisp/progmodes/ruby-mode.el +++ b/lisp/progmodes/ruby-mode.el @@ -1131,9 +1131,8 @@ See `add-log-current-defun-function'." (nth 3 (syntax-ppss (match-beginning 0)))) (string-to-syntax "\\")))) ;; regexps - ("\\(^\\|[[=(,~?:;<>]\\|\\(^\\|\\s \\)\\(if\\|elsif\\|unless\\|while\\|until\\|when\\|and\\|or\\|&&\\|||\\)\\|g?sub!?\\|scan\\|split!?\\)\\s *\\(/\\)[^/\n\\\\]*\\(\\\\.[^/\n\\\\]*\\)*\\(/\\)" - (4 "\"/") - (6 "\"/")) + ("\\(^\\|[[=(,~?:;<>]\\|\\(?:^\\|\\s \\)\\(?:if\\|elsif\\|unless\\|while\\|until\\|when\\|and\\|or\\|&&\\|||\\)\\|g?sub!?\\|scan\\|split!?\\)?\\s *\\(/\\)[^/\n\\\\]*\\(?:\\\\.[^/\n\\\\]*\\)*\\(/\\)" + (2 (ruby-syntax-propertize-regexp))) ("^=en\\(d\\)\\_>" (1 "!")) ("^\\(=\\)begin\\_>" (1 "!")) ;; Handle here documents. @@ -1144,6 +1143,21 @@ See `add-log-current-defun-function'." (1 (prog1 "|" (ruby-syntax-propertize-general-delimiters end))))) (point) end)) + (defun ruby-syntax-propertize-regexp () + (let ((syn (string-to-syntax "\"/"))) + (goto-char (match-end 3)) + (if (or + ;; after paren, comma, operator, control flow keyword, + ;; or a method from hardcoded list + (match-beginning 1) + ;; followed by comma or block + (looking-at "[imxo]*\\s *\\(?:,\\|\\\\)")) + (progn + (put-text-property (1- (point)) (point) + 'syntax-table syn) + syn) + (goto-char (match-end 2))))) + (defun ruby-syntax-propertize-heredoc (limit) (let ((ppss (syntax-ppss)) (res '())) From dfbd787fe6a5684d699926d698aaf9166812a81b Mon Sep 17 00:00:00 2001 From: Stefan Monnier Date: Tue, 24 Apr 2012 13:06:12 -0400 Subject: [PATCH 3/3] * lisp/progmodes/ruby-mode.el: Simplify last change, and cleanup code. (ruby-syntax-propertize-regexp): Remove. (ruby-syntax-propertize-function): Split regexp into chunks. Match following code directly. * test/indent/ruby.rb: New file, to test new syntax-propertize code. --- lisp/ChangeLog | 7 +++++ lisp/progmodes/ruby-mode.el | 63 ++++++++++++++++++++----------------- test/ChangeLog | 4 +++ test/indent/ruby.rb | 19 +++++++++++ 4 files changed, 65 insertions(+), 28 deletions(-) create mode 100644 test/indent/ruby.rb diff --git a/lisp/ChangeLog b/lisp/ChangeLog index 2328cf99c1a..6ae22e6948f 100644 --- a/lisp/ChangeLog +++ b/lisp/ChangeLog @@ -1,3 +1,10 @@ +2012-04-24 Stefan Monnier + + * progmodes/ruby-mode.el: Simplify last change, and cleanup code. + (ruby-syntax-propertize-regexp): Remove. + (ruby-syntax-propertize-function): Split regexp into chunks. + Match following code directly. + 2012-04-24 Dmitry Gutov * progmodes/ruby-mode.el: Handle Cucumber defs (bug#6286). diff --git a/lisp/progmodes/ruby-mode.el b/lisp/progmodes/ruby-mode.el index 8818911159b..5d79437c3c2 100644 --- a/lisp/progmodes/ruby-mode.el +++ b/lisp/progmodes/ruby-mode.el @@ -784,7 +784,7 @@ and `\\' when preceded by `?'." (not (looking-at "[a-z_]")))) (and (looking-at ruby-operator-re) (not (ruby-special-char-p)) - ;; operator at the end of line + ;; Operator at the end of line. (let ((c (char-after (point)))) (and ;; (or (null begin) @@ -794,8 +794,9 @@ and `\\' when preceded by `?'." ;; (not (or (eolp) (looking-at "#") ;; (and (eq (car (nth 1 state)) ?{) ;; (looking-at "|")))))) - ;; not a regexp or general delimited literal - (null (nth 0 (ruby-parse-region (or begin parse-start) (point)))) + ;; Not a regexp or general delimited literal. + (null (nth 0 (ruby-parse-region (or begin parse-start) + (point)))) (or (not (eq ?| (char-after (point)))) (save-excursion (or (eolp) (forward-char -1)) @@ -1110,6 +1111,8 @@ See `add-log-current-defun-function'." mlist))))) (declare-function ruby-syntax-propertize-heredoc "ruby-mode" (limit)) +(declare-function ruby-syntax-general-delimiters-goto-beg "ruby-mode" ()) +(declare-function ruby-syntax-propertize-general-delimiters "ruby-mode" (limit)) (if (eval-when-compile (fboundp #'syntax-propertize-rules)) ;; New code that works independently from font-lock. @@ -1121,18 +1124,37 @@ See `add-log-current-defun-function'." (ruby-syntax-general-delimiters-goto-beg) (funcall (syntax-propertize-rules - ;; #{ }, #$hoge, #@foo are not comments + ;; #{ }, #$hoge, #@foo are not comments. ("\\(#\\)[{$@]" (1 ".")) - ;; $' $" $` .... are variables - ;; ?' ?" ?` are ascii codes + ;; $' $" $` .... are variables. + ;; ?' ?" ?` are ascii codes. ("\\([?$]\\)[#\"'`]" (1 (unless (save-excursion ;; Not within a string. (nth 3 (syntax-ppss (match-beginning 0)))) (string-to-syntax "\\")))) - ;; regexps - ("\\(^\\|[[=(,~?:;<>]\\|\\(?:^\\|\\s \\)\\(?:if\\|elsif\\|unless\\|while\\|until\\|when\\|and\\|or\\|&&\\|||\\)\\|g?sub!?\\|scan\\|split!?\\)?\\s *\\(/\\)[^/\n\\\\]*\\(?:\\\\.[^/\n\\\\]*\\)*\\(/\\)" - (2 (ruby-syntax-propertize-regexp))) + ;; Regexps: regexps are distinguished from division either because + ;; of the keyword/symbol before them, or because of the code + ;; following them. + ((concat + ;; Special tokens that can't be followed by a division operator. + "\\(?:\\(^\\|[[=(,~?:;<>]\\|\\(?:^\\|\\s \\)" + (regexp-opt '("if" "elsif" "unless" "while" "until" "when" "and" + "or" "&&" "||" + "gsub" "gsub!" "sub" "sub!" "scan" "split" "split!")) + "\\)\\s *\\)?" + ;; The regular expression itself. + "\\(/\\)[^/\n\\\\]*\\(?:\\\\.[^/\n\\\\]*\\)*\\(/\\)" + ;; Special code that cannot follow a division operator. + ;; FIXME: Just because the second slash of "/foo/ do bar" can't + ;; be a division, doesn't mean it can't *start* a regexp, as in + ;; "x = toto/foo; if /do bar/". + "\\([imxo]*\\s *\\(?:,\\|\\_\\)\\)?") + (2 (when (or (match-beginning 1) (match-beginning 4)) + (string-to-syntax "\"/"))) + (3 (if (or (match-beginning 1) (match-beginning 4)) + (string-to-syntax "\"/") + (goto-char (match-end 2))))) ("^=en\\(d\\)\\_>" (1 "!")) ("^\\(=\\)begin\\_>" (1 "!")) ;; Handle here documents. @@ -1143,21 +1165,6 @@ See `add-log-current-defun-function'." (1 (prog1 "|" (ruby-syntax-propertize-general-delimiters end))))) (point) end)) - (defun ruby-syntax-propertize-regexp () - (let ((syn (string-to-syntax "\"/"))) - (goto-char (match-end 3)) - (if (or - ;; after paren, comma, operator, control flow keyword, - ;; or a method from hardcoded list - (match-beginning 1) - ;; followed by comma or block - (looking-at "[imxo]*\\s *\\(?:,\\|\\\\)")) - (progn - (put-text-property (1- (point)) (point) - 'syntax-table syn) - syn) - (goto-char (match-end 2))))) - (defun ruby-syntax-propertize-heredoc (limit) (let ((ppss (syntax-ppss)) (res '())) @@ -1199,7 +1206,7 @@ See `add-log-current-defun-function'." parse-sexp-lookup-properties) (ignore-errors (if cl - (progn ; paired delimiters + (progn ; Paired delimiters. ;; Delimiter pairs of the same kind can be nested ;; inside the literal, as long as they are balanced. ;; Create syntax table that ignores other characters. @@ -1210,10 +1217,10 @@ See `add-log-current-defun-function'." (save-restriction (narrow-to-region (point) limit) (forward-list)))) ; skip to the paired character - ;; single character delimiter + ;; Single character delimiter. (re-search-forward (concat "[^\\]\\(?:\\\\\\\\\\)*" (regexp-quote ops)) limit nil)) - ;; if we reached here, the closing delimiter was found + ;; If we reached here, the closing delimiter was found. (put-text-property (1- (point)) (point) 'syntax-table (string-to-syntax "|"))))) ) @@ -1260,7 +1267,7 @@ This should only be called after matching against `ruby-here-doc-end-re'." (4 (7 . ?/)) (6 (7 . ?/))) ("^=en\\(d\\)\\_>" 1 "!") - ;; general delimited string + ;; General delimited string. ("\\(^\\|[[ \t\n<+(,=]\\)\\(%[xrqQwW]?\\([^<[{(a-zA-Z0-9 \n]\\)[^\n\\\\]*\\(\\\\.[^\n\\\\]*\\)*\\(\\3\\)\\)" (3 "\"") (5 "\"")) diff --git a/test/ChangeLog b/test/ChangeLog index 66f8592c79c..ff38a8fa9e1 100644 --- a/test/ChangeLog +++ b/test/ChangeLog @@ -1,3 +1,7 @@ +2012-04-24 Stefan Monnier + + * indent/ruby.rb: New file, to test new syntax-propertize code. + 2012-04-11 Glenn Morris * automated/vc-bzr.el (vc-bzr-test-faulty-bzr-autoloads): New test. diff --git a/test/indent/ruby.rb b/test/indent/ruby.rb new file mode 100644 index 00000000000..c4a747a1c78 --- /dev/null +++ b/test/indent/ruby.rb @@ -0,0 +1,19 @@ +# Don't mis-match "sub" at the end of words. +a = asub / aslb + bsub / bslb; + +b = %Q{This is a "string"} +c = %w(foo + bar + baz) +d = %!hello! + +# A "do" after a slash means that slash is not a division, but it doesn't imply +# it's a regexp-ender, since it can be a regexp-starter instead! +x = toto / foo; if /do bar/ then + toto = 1 + end + +# Some Cucumber code: +Given /toto/ do + print "hello" +end