Make syntax.c call syntax-propertize on demand

* lisp/emacs-lisp/syntax.el (syntax--jit-propertize): New function.
(parse-sexp-propertize-function): Use it.
(syntax-propertize): Disable parse-sexp-propertize-function.

* src/syntax.c (parse_sexp_propertize, update_syntax_table_forward):
New functions.
(syms_of_syntax): New vars `parse-sexp-propertize-done' and
`parse-sexp-propertize-function'.

* src/syntax.h (struct gl_state_s): Add `e_property_truncated' field.
(UPDATE_SYNTAX_TABLE_FORWARD): Use update_syntax_table_forward.
(SETUP_BUFFER_SYNTAX_TABLE): Set e_property_truncated.

* lisp/progmodes/elisp-mode.el (elisp-byte-code-syntax-propertize):
Don't assume `point' is set.

5 files changed, 104 insertions, 25 deletions

diff --git a/etc/NEWS b/etc/NEWS
index 2f2bd157dd..49ce9b7eb0 100644
--- a/etc/NEWS
+++ b/etc/NEWS
@@ -977,6 +977,9 @@ be updated accordingly.
 
 * Lisp Changes in Emacs 25.1
 
+** syntax-propertize is now automatically called on-demand during forward
+parsing functions like `forward-sexp'.
+
 ** New hooks prefix-command-echo-keystrokes-functions and
 prefix-command-preserve-state-hook, to allow the definition of prefix
 commands other than the predefined C-u.
diff --git a/lisp/emacs-lisp/syntax.el b/lisp/emacs-lisp/syntax.el
index 05dd7d5750..6c9cba3e56 100644
--- a/lisp/emacs-lisp/syntax.el
+++ b/lisp/emacs-lisp/syntax.el
@@ -297,7 +297,10 @@ The return value is a function suitable for `syntax-propertize-function'."
     (set (make-local-variable 'parse-sexp-lookup-properties) t)
     (save-excursion
       (with-silent-modifications
+        (make-local-variable 'parse-sexp-propertize-done) ;Just in case!
         (let* ((start (max syntax-propertize--done (point-min)))
+               ;; Avoid recursion!
+               (parse-sexp-propertize-done most-positive-fixnum)
                (end (max pos
                          (min (point-max)
                               (+ start syntax-propertize-chunk-size))))
@@ -324,6 +327,18 @@ The return value is a function suitable for `syntax-propertize-function'."
                                   '(syntax-table nil syntax-multiline nil))
           (funcall syntax-propertize-function start end))))))
 
+;;; Link syntax-propertize with the new parse-sexp-propertize.
+
+(setq-default parse-sexp-propertize-function #'syntax--jit-propertize)
+(defun syntax--jit-propertize (charpos)
+  (if (not syntax-propertize-function)
+      (setq parse-sexp-propertize-done (1+ (point-max)))
+    (syntax-propertize charpos)
+    (setq parse-sexp-propertize-done
+          (if (= (point-max) syntax-propertize--done)
+              (1+ (point-max))
+            syntax-propertize--done))))
+
 ;;; Incrementally compute and memoize parser state.
 
 (defsubst syntax-ppss-depth (ppss)
diff --git a/lisp/progmodes/elisp-mode.el b/lisp/progmodes/elisp-mode.el
index 1ea9f17477..1cf52c0434 100644
--- a/lisp/progmodes/elisp-mode.el
+++ b/lisp/progmodes/elisp-mode.el
@@ -641,7 +641,7 @@ non-nil result supercedes the xrefs produced by
 ;; FIXME: name should be singular; match xref-find-definition
 (defun elisp--xref-find-definitions (symbol)
   ;; The file name is not known when `symbol' is defined via interactive eval.
-  (let (xrefs temp)
+  (let (xrefs)
 
     (let ((temp elisp-xref-find-def-functions))
       (while (and (null xrefs)
@@ -928,6 +928,7 @@ Semicolons start comments.
             (goto-char end)))))))
 
 (defun elisp-byte-code-syntax-propertize (start end)
+  (goto-char start)
   (elisp--byte-code-comment end (point))
   (funcall
    (syntax-propertize-rules
diff --git a/src/syntax.c b/src/syntax.c
index dc82210e79..776ff98436 100644
--- a/src/syntax.c
+++ b/src/syntax.c
@@ -247,7 +247,7 @@ SETUP_SYNTAX_TABLE (ptrdiff_t from, ptrdiff_t count)
   gl_state.offset = 0;
   if (parse_sexp_lookup_properties)
     if (count > 0 || from > BEGV)
-      update_syntax_table (count > 0 ? from : from - 1, count, 1, Qnil);
+      update_syntax_table (count > 0 ? from : from - 1, count, true, Qnil);
 }
 
 /* Same as above, but in OBJECT.  If OBJECT is nil, use current buffer.
@@ -313,7 +313,7 @@ update_syntax_table (ptrdiff_t charpos, EMACS_INT count, bool init,
 {
   Lisp_Object tmp_table;
   int cnt = 0;
-  bool invalidate = 1;
+  bool invalidate = true;
   INTERVAL i;
 
   if (init)
@@ -323,7 +323,7 @@ update_syntax_table (ptrdiff_t charpos, EMACS_INT count, bool init,
       gl_state.stop = gl_state.e_property;
       i = interval_of (charpos, object);
       gl_state.backward_i = gl_state.forward_i = i;
-      invalidate = 0;
+      invalidate = false;
       if (!i)
 	return;
       /* interval_of updates only ->position of the return value, so
@@ -359,7 +359,7 @@ update_syntax_table (ptrdiff_t charpos, EMACS_INT count, bool init,
       i = update_interval (i, charpos);
       if (INTERVAL_LAST_POS (i) != gl_state.b_property)
 	{
-	  invalidate = 0;
+	  invalidate = false;
 	  gl_state.forward_i = i;
 	  gl_state.e_property = INTERVAL_LAST_POS (i) - gl_state.offset;
 	}
@@ -372,7 +372,7 @@ update_syntax_table (ptrdiff_t charpos, EMACS_INT count, bool init,
       i = update_interval (i, charpos);
       if (i->position != gl_state.e_property)
 	{
-	  invalidate = 0;
+	  invalidate = false;
 	  gl_state.backward_i = i;
 	  gl_state.b_property = i->position - gl_state.offset;
 	}
@@ -460,14 +460,56 @@ update_syntax_table (ptrdiff_t charpos, EMACS_INT count, bool init,
     }
   eassert (i == NULL); /* This property goes to the end.  */
   if (count > 0)
-    gl_state.e_property = gl_state.stop;
+    {
+      gl_state.e_property = gl_state.stop;
+      gl_state.forward_i = i;
+    }
   else
     gl_state.b_property = gl_state.start;
 }
+
+static void
+parse_sexp_propertize (ptrdiff_t charpos)
+{
+  EMACS_INT modiffs = CHARS_MODIFF;
+  safe_call1 (Vparse_sexp_propertize_function,
+	      make_number (1 + charpos));
+  if (modiffs != CHARS_MODIFF)
+    error ("parse-sexp-propertize-function modified the buffer!");
+  if (parse_sexp_propertize_done <= charpos)
+    error ("parse-sexp-propertize-function did not move"
+	   " parse-sexp-propertize-done");
+  SETUP_SYNTAX_TABLE (charpos, 1);
+  if (gl_state.e_property > parse_sexp_propertize_done)
+    {
+      gl_state.e_property = parse_sexp_propertize_done;
+      gl_state.e_property_truncated = true;
+    }
+}
+
+void
+update_syntax_table_forward (ptrdiff_t charpos, bool init,
+		     Lisp_Object object)
+{
+  if (!(gl_state.e_property_truncated))
+    update_syntax_table (charpos, 1, init, object);
+  if ((gl_state.e_property > parse_sexp_propertize_done
+       || gl_state.e_property_truncated)
+      && NILP (object))
+    {
+      if (parse_sexp_propertize_done > charpos)
+	{
+	  gl_state.e_property = parse_sexp_propertize_done;
+	  gl_state.e_property_truncated = true;
+	}
+      else
+	parse_sexp_propertize (charpos);
+    }
+}
 
 /* Returns true if char at CHARPOS is quoted.
    Global syntax-table data should be set up already to be good at CHARPOS
-   or after.  On return global syntax data is good for lookup at CHARPOS. */
+   or after.  On return global syntax data is good for lookup at CHARPOS.  */
 
 static bool
 char_quoted (ptrdiff_t charpos, ptrdiff_t bytepos)
@@ -628,7 +670,7 @@ back_comment (ptrdiff_t from, ptrdiff_t from_byte, ptrdiff_t stop,
      OFROM[I] is position of the earliest comment-starter seen
      which is I+2X quotes from the comment-end.
      PARITY is current parity of quotes from the comment end.  */
-  int string_style = -1;	/* Presumed outside of any string. */
+  int string_style = -1;	/* Presumed outside of any string.  */
   bool string_lossage = 0;
   /* Not a real lossage: indicates that we have passed a matching comment
      starter plus a non-matching comment-ender, meaning that any matching
@@ -645,7 +687,7 @@ back_comment (ptrdiff_t from, ptrdiff_t from_byte, ptrdiff_t stop,
   ptrdiff_t defun_start = 0;
   ptrdiff_t defun_start_byte = 0;
   enum syntaxcode code;
-  ptrdiff_t nesting = 1;		/* current comment nesting */
+  ptrdiff_t nesting = 1;		/* Current comment nesting.  */
   int c;
   int syntax = 0;
 
@@ -876,7 +918,7 @@ back_comment (ptrdiff_t from, ptrdiff_t from_byte, ptrdiff_t stop,
 		   is nested, so we need to try again from within the
 		   surrounding comment.  Example: { a (* " *)  */
 		{
-		  /* FIXME: We should advance by one or two chars. */
+		  /* FIXME: We should advance by one or two chars.  */
 		  defun_start = state.comstr_start + 2;
 		  defun_start_byte = CHAR_TO_BYTE (defun_start);
 		}
@@ -3588,7 +3630,7 @@ syms_of_syntax (void)
   staticpro (&gl_state.current_syntax_table);
   staticpro (&gl_state.old_prop);
 
-  /* Defined in regex.c */
+  /* Defined in regex.c.  */
   staticpro (&re_match_object);
 
   DEFSYM (Qscan_error, "scan-error");
@@ -3606,6 +3648,20 @@ Otherwise, that text property is simply ignored.
 See the info node `(elisp)Syntax Properties' for a description of the
 `syntax-table' property.  */);
 
+  DEFVAR_INT ("parse-sexp-propertize-done", parse_sexp_propertize_done,
+	      doc: /* Position up to which syntax-table properties have been set.  */);
+  parse_sexp_propertize_done = -1;
+
+  DEFVAR_LISP ("parse-sexp-propertize-function",
+	       Vparse_sexp_propertize_function,
+	  doc: /* Function to set the `syntax-table' text property.
+Called with one argument, the position at which the property is needed.
+After running it, `parse-sexp-propertize-done' should be strictly greater
+than the argument passed.  */);
+  /* Note: Qnil is a temporary (and invalid) value; it will be properly set in
+     syntax.el.  */
+  Vparse_sexp_propertize_function = Qnil;
+
   words_include_escapes = 0;
   DEFVAR_BOOL ("words-include-escapes", words_include_escapes,
 	       doc: /* Non-nil means `forward-word', etc., should treat escape chars part of words.  */);
diff --git a/src/syntax.h b/src/syntax.h
index bfcb87168b..9c44181155 100644
--- a/src/syntax.h
+++ b/src/syntax.h
@@ -21,6 +21,7 @@ along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.  */
 INLINE_HEADER_BEGIN
 
 extern void update_syntax_table (ptrdiff_t, EMACS_INT, bool, Lisp_Object);
+extern void update_syntax_table_forward (ptrdiff_t, bool, Lisp_Object);
 
 /* The standard syntax table is stored where it will automatically
    be used in all new buffers.  */
@@ -52,30 +53,32 @@ enum syntaxcode
 		       other side by any char with the same syntaxcode.  */
     Sstring_fence,  /* Starts/ends string which is delimited on the
 		       other side by any char with the same syntaxcode.  */
-    Smax	 /* Upper bound on codes that are meaningful */
+    Smax	 /* Upper bound on codes that are meaningful.  */
   };
 
 
 struct gl_state_s
 {
-  Lisp_Object object;			/* The object we are scanning. */
-  ptrdiff_t start;			/* Where to stop. */
-  ptrdiff_t stop;			/* Where to stop. */
+  Lisp_Object object;			/* The object we are scanning.  */
+  ptrdiff_t start;			/* Where to stop.  */
+  ptrdiff_t stop;			/* Where to stop.  */
   bool use_global;			/* Whether to use global_code
-					   or c_s_t. */
-  Lisp_Object global_code;		/* Syntax code of current char. */
-  Lisp_Object current_syntax_table;	/* Syntax table for current pos. */
-  Lisp_Object old_prop;			/* Syntax-table prop at prev pos. */
-  ptrdiff_t b_property;			/* First index where c_s_t is valid. */
+					   or c_s_t.  */
+  Lisp_Object global_code;		/* Syntax code of current char.  */
+  Lisp_Object current_syntax_table;	/* Syntax table for current pos.  */
+  Lisp_Object old_prop;			/* Syntax-table prop at prev pos.  */
+  ptrdiff_t b_property;			/* First index where c_s_t is valid.  */
   ptrdiff_t e_property;			/* First index where c_s_t is
-					   not valid. */
-  INTERVAL forward_i;			/* Where to start lookup on forward */
+					   not valid.  */
+  bool e_property_truncated;		/* true if e_property if was truncated
+					   by parse_sexp_propertize_done.  */
+  INTERVAL forward_i;			/* Where to start lookup on forward.  */
   INTERVAL backward_i;			/* or backward movement.  The
 					   data in c_s_t is valid
 					   between these intervals,
 					   and possibly at the
 					   intervals too, depending
-					   on: */
+					   on:  */
   /* Offset for positions specified to UPDATE_SYNTAX_TABLE.  */
   ptrdiff_t offset;
 };
@@ -173,7 +176,7 @@ INLINE void
 UPDATE_SYNTAX_TABLE_FORWARD (ptrdiff_t charpos)
 {
   if (parse_sexp_lookup_properties && charpos >= gl_state.e_property)
-    update_syntax_table (charpos + gl_state.offset, 1, false, gl_state.object);
+    update_syntax_table_forward (charpos + gl_state.offset, false, gl_state.object);
 }
 
 /* Make syntax table state (gl_state) good for CHARPOS, assuming it is
@@ -201,6 +204,7 @@ INLINE void
 SETUP_BUFFER_SYNTAX_TABLE (void)
 {
   gl_state.use_global = false;
+  gl_state.e_property_truncated = false;
   gl_state.current_syntax_table = BVAR (current_buffer, syntax_table);
 }