Check for buffer and string overflow more precisely.

* buffer.h (BUF_BYTES_MAX): New macro.
* lisp.h (STRING_BYTES_MAX): New macro.
* alloc.c (Fmake_string):
* character.c (string_escape_byte8):
* coding.c (coding_alloc_by_realloc):
* doprnt.c (doprnt):
* editfns.c (Fformat):
* eval.c (verror):
Use STRING_BYTES_MAX, not MOST_POSITIVE_FIXNUM,
since they may not be the same number.
* editfns.c (Finsert_char):
* fileio.c (Finsert_file_contents):
Likewise for BUF_BYTES_MAX.

10 files changed, 37 insertions, 11 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 7fb1479e54..bf81533460 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,5 +1,20 @@
 2011-06-06  Paul Eggert  <eggert@cs.ucla.edu>
 
+	Check for buffer and string overflow more precisely.
+	* buffer.h (BUF_BYTES_MAX): New macro.
+	* lisp.h (STRING_BYTES_MAX): New macro.
+	* alloc.c (Fmake_string):
+	* character.c (string_escape_byte8):
+	* coding.c (coding_alloc_by_realloc):
+	* doprnt.c (doprnt):
+	* editfns.c (Fformat):
+	* eval.c (verror):
+	Use STRING_BYTES_MAX, not MOST_POSITIVE_FIXNUM,
+	since they may not be the same number.
+	* editfns.c (Finsert_char):
+	* fileio.c (Finsert_file_contents):
+	Likewise for BUF_BYTES_MAX.
+
 	* image.c: Use ptrdiff_t, not int, for sizes.
 	(slurp_file): Switch from int to ptrdiff_t.
 	All uses changed.
diff --git a/src/alloc.c b/src/alloc.c
index 8d0fdd125d..d9e00c3aeb 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -2211,7 +2211,7 @@ INIT must be an integer that represents a character.  */)
       int len = CHAR_STRING (c, str);
       EMACS_INT string_len = XINT (length);
 
-      if (string_len > MOST_POSITIVE_FIXNUM / len)
+      if (string_len > STRING_BYTES_MAX / len)
 	string_overflow ();
       nbytes = len * string_len;
       val = make_uninit_multibyte_string (string_len, nbytes);
diff --git a/src/buffer.h b/src/buffer.h
index 8c64a24e80..3c91bdfe57 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -306,6 +306,11 @@ do								\
   }								\
 while (0)
 
+/* Maximum number of bytes in a buffer.
+   A buffer cannot contain more bytes than a 1-origin fixnum can represent,
+   nor can it be so large that C pointer arithmetic stops working.  */
+#define BUF_BYTES_MAX min (MOST_POSITIVE_FIXNUM - 1, min (SIZE_MAX, PTRDIFF_MAX))
+
 /* Return the address of byte position N in current buffer.  */
 
 #define BYTE_POS_ADDR(n) \
diff --git a/src/character.c b/src/character.c
index 34e69da9cc..170952619e 100644
--- a/src/character.c
+++ b/src/character.c
@@ -838,7 +838,7 @@ string_escape_byte8 (Lisp_Object string)
   if (multibyte)
     {
       if ((MOST_POSITIVE_FIXNUM - nchars) / 3 < byte8_count
-	  || (MOST_POSITIVE_FIXNUM - nbytes) / 2 < byte8_count)
+	  || (STRING_BYTES_MAX - nbytes) / 2 < byte8_count)
 	string_overflow ();
 
       /* Convert 2-byte sequence of byte8 chars to 4-byte octal.  */
@@ -847,7 +847,7 @@ string_escape_byte8 (Lisp_Object string)
     }
   else
     {
-      if ((MOST_POSITIVE_FIXNUM - nchars) / 3 < byte8_count)
+      if ((STRING_BYTES_MAX - nchars) / 3 < byte8_count)
 	string_overflow ();
 
       /* Convert 1-byte sequence of byte8 chars to 4-byte octal.  */
diff --git a/src/coding.c b/src/coding.c
index 6ccaf354c7..64e8e41a5a 100644
--- a/src/coding.c
+++ b/src/coding.c
@@ -1071,8 +1071,8 @@ coding_set_destination (struct coding_system *coding)
 static void
 coding_alloc_by_realloc (struct coding_system *coding, EMACS_INT bytes)
 {
-  if (coding->dst_bytes >= MOST_POSITIVE_FIXNUM - bytes)
-    error ("Maximum size of buffer or string exceeded");
+  if (STRING_BYTES_MAX - coding->dst_bytes < bytes)
+    string_overflow ();
   coding->destination = (unsigned char *) xrealloc (coding->destination,
 						    coding->dst_bytes + bytes);
   coding->dst_bytes += bytes;
diff --git a/src/doprnt.c b/src/doprnt.c
index d2abc11991..5ca3ea89be 100644
--- a/src/doprnt.c
+++ b/src/doprnt.c
@@ -329,7 +329,7 @@ doprnt (char *buffer, register size_t bufsize, const char *format,
 		minlen = atoi (&fmtcpy[1]);
 	      string = va_arg (ap, char *);
 	      tem = strlen (string);
-	      if (tem > MOST_POSITIVE_FIXNUM)
+	      if (tem > STRING_BYTES_MAX)
 		error ("String for %%s or %%S format is too long");
 	      width = strwidth (string, tem);
 	      goto doit1;
@@ -338,7 +338,7 @@ doprnt (char *buffer, register size_t bufsize, const char *format,
 	    doit:
 	      /* Coming here means STRING contains ASCII only.  */
 	      tem = strlen (string);
-	      if (tem > MOST_POSITIVE_FIXNUM)
+	      if (tem > STRING_BYTES_MAX)
 		error ("Format width or precision too large");
 	      width = tem;
 	    doit1:
diff --git a/src/editfns.c b/src/editfns.c
index b961e602e4..b4ce9a1c57 100644
--- a/src/editfns.c
+++ b/src/editfns.c
@@ -2342,7 +2342,7 @@ from adjoining text, if those properties are sticky.  */)
     len = CHAR_STRING (XFASTINT (character), str);
   else
     str[0] = XFASTINT (character), len = 1;
-  if (MOST_POSITIVE_FIXNUM / len < XINT (count))
+  if (BUF_BYTES_MAX / len < XINT (count))
     error ("Maximum buffer size would be exceeded");
   n = XINT (count) * len;
   if (n <= 0)
@@ -3589,7 +3589,7 @@ usage: (format STRING &rest OBJECTS)  */)
   char initial_buffer[4000];
   char *buf = initial_buffer;
   EMACS_INT bufsize = sizeof initial_buffer;
-  EMACS_INT max_bufsize = min (MOST_POSITIVE_FIXNUM + 1, SIZE_MAX);
+  EMACS_INT max_bufsize = STRING_BYTES_MAX + 1;
   char *p;
   Lisp_Object buf_save_value IF_LINT (= {0});
   register char *format, *end, *format_start;
diff --git a/src/eval.c b/src/eval.c
index f8bc0a9f6a..ef5abac17a 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -1994,7 +1994,7 @@ verror (const char *m, va_list ap)
 {
   char buf[4000];
   size_t size = sizeof buf;
-  size_t size_max = min (MOST_POSITIVE_FIXNUM + 1, SIZE_MAX);
+  size_t size_max = STRING_BYTES_MAX + 1;
   size_t mlen = strlen (m);
   char *buffer = buf;
   size_t used;
diff --git a/src/fileio.c b/src/fileio.c
index 48dac80a39..82b31036fb 100644
--- a/src/fileio.c
+++ b/src/fileio.c
@@ -3248,7 +3248,7 @@ variable `last-coding-system-used' to the coding system actually used.  */)
   /* Check whether the size is too large or negative, which can happen on a
      platform that allows file sizes greater than the maximum off_t value.  */
   if (! not_regular
-      && ! (0 <= st.st_size && st.st_size <= MOST_POSITIVE_FIXNUM))
+      && ! (0 <= st.st_size && st.st_size <= BUF_BYTES_MAX))
     error ("Maximum buffer size exceeded");
 
   /* Prevent redisplay optimizations.  */
diff --git a/src/lisp.h b/src/lisp.h
index a9e7354f9f..a46436e718 100644
--- a/src/lisp.h
+++ b/src/lisp.h
@@ -763,6 +763,12 @@ extern EMACS_INT string_bytes (struct Lisp_String *);
 
 #endif /* not GC_CHECK_STRING_BYTES */
 
+/* A string cannot contain more bytes than a fixnum can represent,
+   nor can it be so long that C pointer arithmetic stops working on
+   the string plus a terminating null.  */
+#define STRING_BYTES_MAX  \
+  min (MOST_POSITIVE_FIXNUM, min (SIZE_MAX, PTRDIFF_MAX) - 1)
+
 /* Mark STR as a unibyte string.  */
 #define STRING_SET_UNIBYTE(STR)  \
   do { if (EQ (STR, empty_multibyte_string))  \