2003-01-27 Havoc Pennington <hp@pobox.com>

	* dbus/dbus-mempool.c (time_for_size): replace printf with
	_dbus_verbose

	* dbus/dbus-message-builder.c (_dbus_message_data_load): allow
	empty lines; fix the SAVE_LENGTH stuff to be
	START_LENGTH/END_LENGTH so it actually works; couple other
	bugfixes

	* test/Makefile.am (dist-hook): add dist-hook for .message files

	* dbus/dbus-string.c (DBUS_STRING_COPY_PREAMBLE): source of a copy
	can be constant or locked.
	(_dbus_string_free): allow freeing a const string as
	documented/intended

	* dbus/dbus-sysdeps.c (_dbus_concat_dir_and_file): utility

	* dbus/dbus-test-main.c (main): take an argument which is the
	directory containing test data

	* dbus/dbus-message.c (_dbus_message_test): pass a test_data_dir
	argument to this and load all the messages in test/data/
	checking that they can be loaded or not loaded as appropriate.

18 files changed, 775 insertions, 54 deletions

diff --git a/ChangeLog b/ChangeLog
index 596eae8a..8530244d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,29 @@
+2003-01-27  Havoc Pennington  <hp@pobox.com>
+
+	* dbus/dbus-mempool.c (time_for_size): replace printf with 
+	_dbus_verbose
+
+	* dbus/dbus-message-builder.c (_dbus_message_data_load): allow
+	empty lines; fix the SAVE_LENGTH stuff to be
+	START_LENGTH/END_LENGTH so it actually works; couple other 
+	bugfixes
+	
+	* test/Makefile.am (dist-hook): add dist-hook for .message files
+
+	* dbus/dbus-string.c (DBUS_STRING_COPY_PREAMBLE): source of a copy
+	can be constant or locked.
+	(_dbus_string_free): allow freeing a const string as
+	documented/intended
+
+	* dbus/dbus-sysdeps.c (_dbus_concat_dir_and_file): utility
+
+	* dbus/dbus-test-main.c (main): take an argument which is the 
+	directory containing test data
+
+	* dbus/dbus-message.c (_dbus_message_test): pass a test_data_dir
+	argument to this and load all the messages in test/data/
+	checking that they can be loaded or not loaded as appropriate.
+
 2003-01-27  Anders Carlsson  <andersca@codefactory.se>
 
 	* bus/dispatch.c: (bus_dispatch_message_handler):
diff --git a/dbus/Makefile.am b/dbus/Makefile.am
index 96169a5a..3f6bb63f 100644
--- a/dbus/Makefile.am
+++ b/dbus/Makefile.am
@@ -83,6 +83,7 @@ libdbus_1_la_LDFLAGS= -export-symbols-regex "^[^_].*"
 ## so if adding tests not to be run in make check, don't add them to 
 ## TESTS
 if DBUS_BUILD_TESTS
+TESTS_ENVIRONMENT="DBUS_TEST_DATA=$(top_srcdir)/test/data"
 TESTS=dbus-test
 else
 TESTS=
diff --git a/dbus/dbus-marshal.c b/dbus/dbus-marshal.c
index aa6e2c6b..9c17aabc 100644
--- a/dbus/dbus-marshal.c
+++ b/dbus/dbus-marshal.c
@@ -1149,7 +1149,7 @@ _dbus_marshal_test (void)
   if (!_dbus_marshal_int32_array (&str, DBUS_BIG_ENDIAN, array1, 3))
     _dbus_assert_not_reached ("could not marshal integer array");
   array2 = _dbus_demarshal_int32_array (&str, DBUS_BIG_ENDIAN, pos, &pos, &len);
-  printf ("length is: %d\n", len);
+
   if (len != 3)
     _dbus_assert_not_reached ("Signed integer array lengths differ!\n");
   dbus_free (array2);
diff --git a/dbus/dbus-mempool.c b/dbus/dbus-mempool.c
index a3aa086d..05e3749b 100644
--- a/dbus/dbus-mempool.c
+++ b/dbus/dbus-mempool.c
@@ -285,9 +285,9 @@ time_for_size (int size)
   void *to_free[FREE_ARRAY_SIZE];
   DBusMemPool *pool;
 
-  printf ("Timings for size %d\n", size);
+  _dbus_verbose ("Timings for size %d\n", size);
   
-  printf (" malloc\n");
+  _dbus_verbose (" malloc\n");
   
   start = clock ();
   
@@ -317,12 +317,12 @@ time_for_size (int size)
 
   end = clock ();
 
-  printf ("  created/destroyed %d elements in %g seconds\n",
-          N_ITERATIONS, (end - start) / (double) CLOCKS_PER_SEC);
+  _dbus_verbose ("  created/destroyed %d elements in %g seconds\n",
+                 N_ITERATIONS, (end - start) / (double) CLOCKS_PER_SEC);
 
 
 
-  printf (" mempools\n");
+  _dbus_verbose (" mempools\n");
   
   start = clock ();
 
@@ -356,10 +356,10 @@ time_for_size (int size)
   
   end = clock ();
 
-  printf ("  created/destroyed %d elements in %g seconds\n",
-          N_ITERATIONS, (end - start) / (double) CLOCKS_PER_SEC);
+  _dbus_verbose ("  created/destroyed %d elements in %g seconds\n",
+                 N_ITERATIONS, (end - start) / (double) CLOCKS_PER_SEC);
 
-  printf (" zeroed malloc\n");
+  _dbus_verbose (" zeroed malloc\n");
     
   start = clock ();
   
@@ -389,10 +389,10 @@ time_for_size (int size)
 
   end = clock ();
 
-  printf ("  created/destroyed %d elements in %g seconds\n",
-          N_ITERATIONS, (end - start) / (double) CLOCKS_PER_SEC);
+  _dbus_verbose ("  created/destroyed %d elements in %g seconds\n",
+                 N_ITERATIONS, (end - start) / (double) CLOCKS_PER_SEC);
   
-  printf (" zeroed mempools\n");
+  _dbus_verbose (" zeroed mempools\n");
   
   start = clock ();
 
@@ -426,8 +426,8 @@ time_for_size (int size)
   
   end = clock ();
 
-  printf ("  created/destroyed %d elements in %g seconds\n",
-          N_ITERATIONS, (end - start) / (double) CLOCKS_PER_SEC);
+  _dbus_verbose ("  created/destroyed %d elements in %g seconds\n",
+                 N_ITERATIONS, (end - start) / (double) CLOCKS_PER_SEC);
 }
 
 /**
diff --git a/dbus/dbus-message-builder.c b/dbus/dbus-message-builder.c
index d290ecd4..8d012294 100644
--- a/dbus/dbus-message-builder.c
+++ b/dbus/dbus-message-builder.c
@@ -45,20 +45,24 @@ pop_line (DBusString *source,
           DBusString *dest)
 {
   int eol;
+  dbus_bool_t have_newline;
   
   _dbus_string_set_length (dest, 0);
   
   eol = 0;
   if (_dbus_string_find (source, 0, "\n", &eol))
-    eol += 1; /* include newline */
+    {
+      have_newline = TRUE;
+      eol += 1; /* include newline */
+    }
   else
-    eol = _dbus_string_get_length (source);
-
-  if (eol == 0)
     {
-      _dbus_verbose ("no more data in file\n");
-      return FALSE;
+      eol = _dbus_string_get_length (source);
+      have_newline = FALSE;
     }
+
+  if (eol == 0)
+    return FALSE; /* eof */
   
   if (!_dbus_string_move_len (source, 0, eol,
                               dest, 0))
@@ -68,8 +72,12 @@ pop_line (DBusString *source,
     }
 
   /* dump the newline */
-  _dbus_string_set_length (dest,
-                           _dbus_string_get_length (dest) - 1);
+  if (have_newline)
+    {
+      _dbus_assert (_dbus_string_get_length (dest) > 0);
+      _dbus_string_set_length (dest,
+                               _dbus_string_get_length (dest) - 1);
+    }
   
   return TRUE;
 }
@@ -101,6 +109,7 @@ strip_leading_space (DBusString *str)
 typedef struct
 {
   DBusString name;
+  int start;  /**< Calculate length since here */
   int length; /**< length to write */
   int offset; /**< where to write it into the data */
   int endian; /**< endianness to write with */
@@ -111,6 +120,9 @@ free_saved_length (void *data)
 {
   SavedLength *sl = data;
 
+  if (sl == NULL)
+    return; /* all hash free functions have to accept NULL */
+  
   _dbus_string_free (&sl->name);
   dbus_free (sl);
 }
@@ -144,6 +156,7 @@ ensure_saved_length (DBusHashTable    *hash,
   if (!_dbus_hash_table_insert_string (hash, (char*)s, sl))
     goto failed;
 
+  sl->start = -1;
   sl->length = -1;
   sl->offset = -1;
   sl->endian = -1;
@@ -156,6 +169,28 @@ ensure_saved_length (DBusHashTable    *hash,
 }
 
 static dbus_bool_t
+save_start (DBusHashTable    *hash,
+            const DBusString *name,
+            int               start)
+{
+  SavedLength *sl;
+
+  sl = ensure_saved_length (hash, name);
+
+  if (sl == NULL)
+    return FALSE;
+  else if (sl->start >= 0)
+    {
+      _dbus_warn ("Same START_LENGTH given twice\n");
+      return FALSE;
+    }
+  else
+    sl->start = start;
+
+  return TRUE;
+}
+
+static dbus_bool_t
 save_length (DBusHashTable    *hash,
              const DBusString *name,
              int               length)
@@ -168,7 +203,7 @@ save_length (DBusHashTable    *hash,
     return FALSE;
   else if (sl->length >= 0)
     {
-      _dbus_warn ("Same SAVE_LENGTH given twice\n");
+      _dbus_warn ("Same END_LENGTH given twice\n");
       return FALSE;
     }
   else
@@ -303,7 +338,9 @@ append_saved_length (DBusString       *dest,
  *   ALIGN <N> aligns to the given value
  *   UNALIGN skips alignment for the next marshal
  *   BYTE <N> inserts the given integer in [0,255] or char in 'a' format
- *   SAVE_LENGTH <name> records the current length under the given name
+ *   START_LENGTH <name> marks the start of a length to measure
+ *   END_LENGTH <name> records the length since START_LENGTH under the given name
+ *                     (or if no START_LENGTH, absolute length)
  *   LENGTH <name> inserts the saved length of the same name
  *   CHOP <N> chops last N bytes off the data
  *   FIELD_NAME <abcd> inserts 4-byte field name
@@ -380,9 +417,14 @@ _dbus_message_data_load (DBusString       *dest,
       line_no += 1;
 
       strip_leading_space (&line);
-      
-      if (_dbus_string_starts_with_c_str (&line,
-                                          "#"))
+
+      if (_dbus_string_get_length (&line) == 0)
+        {
+          /* empty line */
+          goto next_iteration;
+        }
+      else if (_dbus_string_starts_with_c_str (&line,
+                                               "#"))
         {
           /* Ignore this comment */
           goto next_iteration;
@@ -455,12 +497,15 @@ _dbus_message_data_load (DBusString       *dest,
           strip_command_name (&line);
 
           if (!_dbus_string_parse_int (&line, 0, &val, NULL))
-            goto parse_failed;
+            {
+              _dbus_warn ("Failed to parse integer\n");
+              goto parse_failed;
+            }
 
           if (val > 16)
             {
               _dbus_warn ("Aligning to %ld boundary is crack\n",
-                             val);
+                          val);
               goto parse_failed;
             }
           
@@ -483,7 +528,10 @@ _dbus_message_data_load (DBusString       *dest,
           strip_command_name (&line);
 
           if (!_dbus_string_parse_int (&line, 0, &val, NULL))
-            goto parse_failed;
+            {
+              _dbus_warn ("Failed to parse integer to chop\n");
+              goto parse_failed;
+            }
 
           if (val > _dbus_string_get_length (dest))
             {
@@ -511,7 +559,11 @@ _dbus_message_data_load (DBusString       *dest,
             {
               long val;
               if (!_dbus_string_parse_int (&line, 0, &val, NULL))
-                goto parse_failed;
+                {
+                  _dbus_warn ("Failed to parse integer for BYTE\n");
+                  goto parse_failed;
+                }
+
               if (val > 255)
                 {
                   _dbus_warn ("A byte must be in range 0-255 not %ld\n",
@@ -524,14 +576,26 @@ _dbus_message_data_load (DBusString       *dest,
           _dbus_string_append_byte (dest, the_byte);
         }
       else if (_dbus_string_starts_with_c_str (&line,
-                                               "SAVE_LENGTH"))
+                                               "START_LENGTH"))
+        {
+          strip_command_name (&line);
+
+          if (!save_start (length_hash, &line,
+                           _dbus_string_get_length (dest)))
+            {
+              _dbus_warn ("failed to save length start\n");
+              goto parse_failed;
+            }
+        }
+      else if (_dbus_string_starts_with_c_str (&line,
+                                               "END_LENGTH"))
         {
           strip_command_name (&line);
 
           if (!save_length (length_hash, &line,
                             _dbus_string_get_length (dest)))
             {
-              _dbus_warn ("failed to save length\n");
+              _dbus_warn ("failed to save length end\n");
               goto parse_failed;
             }
         }
@@ -628,7 +692,10 @@ _dbus_message_data_load (DBusString       *dest,
           strip_command_name (&line);
 
           if (!_dbus_string_parse_int (&line, 0, &val, NULL))
-            goto parse_failed;
+            {
+              _dbus_warn ("could not parse integer for INT32\n");
+              goto parse_failed;
+            }
           
           if (!_dbus_marshal_int32 (dest, endian,
                                     val))
@@ -741,24 +808,27 @@ _dbus_message_data_load (DBusString       *dest,
       
       if (sl->length < 0)
         {
-          _dbus_warn ("Used LENGTH %s but never did SAVE_LENGTH\n",
+          _dbus_warn ("Used LENGTH %s but never did END_LENGTH\n",
                       s);
           goto out;
         }
       else if (sl->offset < 0)
         {
-          _dbus_warn ("Did SAVE_LENGTH %s but never used LENGTH\n",
+          _dbus_warn ("Did END_LENGTH %s but never used LENGTH\n",
                       s);
           goto out;
         }
       else
         {
-          _dbus_verbose ("Filling in length %s endian = %d offset = %d length = %d\n",
-                         s, sl->endian, sl->offset, sl->length);
+          if (sl->start < 0)
+            sl->start = 0;
+          
+          _dbus_verbose ("Filling in length %s endian = %d offset = %d start = %d length = %d\n",
+                         s, sl->endian, sl->offset, sl->start, sl->length);
           _dbus_marshal_set_int32 (dest,
                                    sl->endian,
                                    sl->offset,
-                                   sl->length);
+                                   sl->length - sl->start);
         }
     }
   
diff --git a/dbus/dbus-message.c b/dbus/dbus-message.c
index dae219a8..e0b85663 100644
--- a/dbus/dbus-message.c
+++ b/dbus/dbus-message.c
@@ -28,6 +28,7 @@
 #include "dbus-message-internal.h"
 #include "dbus-memory.h"
 #include "dbus-list.h"
+#include "dbus-message-builder.h"
 #include <string.h>
 
 /**
@@ -2018,6 +2019,334 @@ message_iter_test (DBusMessage *message)
   dbus_message_iter_unref (iter);
 }
 
+static dbus_bool_t
+check_have_valid_message (DBusMessageLoader *loader)
+{
+  DBusMessage *message;
+  dbus_bool_t retval;
+
+  message = NULL;
+  retval = FALSE;
+  
+  if (_dbus_message_loader_get_is_corrupted (loader))
+    {
+      _dbus_warn ("loader corrupted on message that was expected to be valid\n");
+      goto failed;
+    }
+  
+  message = _dbus_message_loader_pop_message (loader);
+  if (message == NULL)
+    {
+      _dbus_warn ("didn't load message that was expected to be valid (message not popped)\n");
+      goto failed;
+    }
+  
+  if (_dbus_string_get_length (&loader->data) > 0)
+    {
+      _dbus_warn ("had leftover bytes from expected-to-be-valid single message\n");
+      goto failed;
+    }
+
+  retval = TRUE;
+
+ failed:
+  if (message)
+    dbus_message_unref (message);
+  return retval;
+}
+
+static dbus_bool_t
+check_invalid_message (DBusMessageLoader *loader)
+{
+  dbus_bool_t retval;
+
+  retval = FALSE;
+  
+  if (!_dbus_message_loader_get_is_corrupted (loader))
+    {
+      _dbus_warn ("loader not corrupted on message that was expected to be invalid\n");
+      goto failed;
+    }
+
+  retval = TRUE;
+
+ failed:
+  return retval;
+}
+
+static dbus_bool_t
+check_incomplete_message (DBusMessageLoader *loader)
+{
+  DBusMessage *message;
+  dbus_bool_t retval;
+
+  message = NULL;
+  retval = FALSE;
+  
+  if (_dbus_message_loader_get_is_corrupted (loader))
+    {
+      _dbus_warn ("loader corrupted on message that was expected to be valid (but incomplete)\n");
+      goto failed;
+    }
+  
+  message = _dbus_message_loader_pop_message (loader);
+  if (message != NULL)
+    {
+      _dbus_warn ("loaded message that was expected to be incomplete\n");
+      goto failed;
+    }
+
+  retval = TRUE;
+
+ failed:
+  if (message)
+    dbus_message_unref (message);
+  return retval;
+}
+
+typedef enum
+{
+  MESSAGE_VALID,
+  MESSAGE_INVALID,
+  MESSAGE_INCOMPLETE
+} ExpectedMessageValidity;
+
+static dbus_bool_t
+check_loader_results (DBusMessageLoader      *loader,
+                      ExpectedMessageValidity validity)
+{
+  switch (validity)
+    {
+    case MESSAGE_VALID:
+      return check_have_valid_message (loader);
+    case MESSAGE_INVALID:
+      return check_invalid_message (loader);
+    case MESSAGE_INCOMPLETE:
+      return check_incomplete_message (loader);
+    }
+
+  _dbus_assert_not_reached ("bad ExpectedMessageValidity");
+  return FALSE;
+}
+
+static dbus_bool_t
+try_message (const DBusString       *filename,
+             ExpectedMessageValidity validity)
+{
+  DBusString data;
+  DBusMessageLoader *loader;
+  dbus_bool_t retval;
+  int len;
+  int i;
+  const char *filename_c;
+
+  _dbus_string_get_const_data (filename, &filename_c);
+  
+  if (!_dbus_string_ends_with_c_str (filename, ".message"))
+    {
+      _dbus_verbose ("Skipping non-.message file %s\n",
+                     filename_c);
+      return TRUE;
+    }
+
+  {
+    const char *s;
+    _dbus_string_get_const_data (filename, &s);
+    printf ("    %s\n", s);
+  }
+  
+  _dbus_verbose (" expecting %s\n",
+                 validity == MESSAGE_VALID ? "valid" :
+                 (validity == MESSAGE_INVALID ? "invalid" : "incomplete"));
+
+  loader = NULL;
+  retval = FALSE;
+  
+  if (!_dbus_string_init (&data, _DBUS_INT_MAX))
+    _dbus_assert_not_reached ("could not allocate string\n");
+  
+  if (!_dbus_message_data_load (&data, filename))
+    {
+      const char *s;      
+      _dbus_string_get_const_data (filename, &s);
+      _dbus_warn ("Could not load message file %s\n", s);
+      goto failed;
+    }
+
+  /* Write the data one byte at a time */
+  
+  loader = _dbus_message_loader_new ();
+
+  len = _dbus_string_get_length (&data);
+  for (i = 0; i < len; i++)
+    {
+      DBusString *buffer;
+
+      _dbus_message_loader_get_buffer (loader, &buffer);
+      _dbus_string_append_byte (buffer,
+                                _dbus_string_get_byte (&data, i));
+      _dbus_message_loader_return_buffer (loader, buffer, 1);
+    }
+  
+  if (!check_loader_results (loader, validity))
+    goto failed;
+
+  _dbus_message_loader_unref (loader);
+  loader = NULL;
+
+  /* Write the data all at once */
+  
+  loader = _dbus_message_loader_new ();
+
+  {
+    DBusString *buffer;
+    
+    _dbus_message_loader_get_buffer (loader, &buffer);
+    _dbus_string_copy (&data, 0, buffer,
+                       _dbus_string_get_length (buffer));
+    _dbus_message_loader_return_buffer (loader, buffer, 1);
+  }
+  
+  if (!check_loader_results (loader, validity))
+    goto failed;
+
+  _dbus_message_loader_unref (loader);
+  loader = NULL;  
+
+  /* Write the data 2 bytes at a time */
+  
+  loader = _dbus_message_loader_new ();
+
+  len = _dbus_string_get_length (&data);
+  for (i = 0; i < len; i += 2)
+    {
+      DBusString *buffer;
+
+      _dbus_message_loader_get_buffer (loader, &buffer);
+      _dbus_string_append_byte (buffer,
+                                _dbus_string_get_byte (&data, i));
+      if ((i+1) < len)
+        _dbus_string_append_byte (buffer,
+                                  _dbus_string_get_byte (&data, i+1));
+      _dbus_message_loader_return_buffer (loader, buffer, 1);
+    }
+  
+  if (!check_loader_results (loader, validity))
+    goto failed;
+
+  _dbus_message_loader_unref (loader);
+  loader = NULL;
+  
+  retval = TRUE;
+  
+ failed:
+  if (!retval)
+    {
+      const char *s;
+
+      if (_dbus_string_get_length (&data) > 0)
+        _dbus_verbose_bytes_of_string (&data, 0,
+                                       _dbus_string_get_length (&data));
+      
+      _dbus_string_get_const_data (filename, &s);
+      _dbus_warn ("Failed message loader test on %s\n",
+                  s);
+    }
+  
+  if (loader)
+    _dbus_message_loader_unref (loader);
+  _dbus_string_free (&data);
+  
+  return retval;
+}
+
+static dbus_bool_t
+process_test_subdir (const DBusString       *test_base_dir,
+                     const char             *subdir,
+                     ExpectedMessageValidity validity)
+{
+  DBusString test_directory;
+  DBusString filename;
+  DBusDirIter *dir;
+  dbus_bool_t retval;
+  DBusResultCode result;
+
+  retval = FALSE;
+  dir = NULL;
+  
+  if (!_dbus_string_init (&test_directory, _DBUS_INT_MAX))
+    _dbus_assert_not_reached ("didn't allocate test_directory\n");
+
+  _dbus_string_init_const (&filename, subdir);
+  
+  if (!_dbus_string_copy (test_base_dir, 0,
+                          &test_directory, 0))
+    _dbus_assert_not_reached ("couldn't copy test_base_dir to test_directory");
+  
+  if (!_dbus_concat_dir_and_file (&test_directory, &filename))    
+    _dbus_assert_not_reached ("could't allocate full path");
+
+  _dbus_string_free (&filename);
+  if (!_dbus_string_init (&filename, _DBUS_INT_MAX))
+    _dbus_assert_not_reached ("didn't allocate filename string\n");
+  
+  dir = _dbus_directory_open (&test_directory, &result);
+  if (dir == NULL)
+    {
+      const char *s;
+      _dbus_string_get_const_data (&test_directory, &s);
+      _dbus_warn ("Could not open %s: %s\n", s,
+                  dbus_result_to_string (result));
+      goto failed;
+    }
+
+  printf ("Testing:\n");
+  
+  result = DBUS_RESULT_SUCCESS;
+  while (_dbus_directory_get_next_file (dir, &filename, &result))
+    {
+      DBusString full_path;
+
+      if (!_dbus_string_init (&full_path, _DBUS_INT_MAX))
+        _dbus_assert_not_reached ("couldn't init string");
+
+      if (!_dbus_string_copy (&test_directory, 0, &full_path, 0))
+        _dbus_assert_not_reached ("couldn't copy dir to full_path");
+
+      if (!_dbus_concat_dir_and_file (&full_path, &filename))
+        _dbus_assert_not_reached ("couldn't concat file to dir");
+      
+      if (!try_message (&full_path, validity))
+        {
+          _dbus_string_free (&full_path);
+          goto failed;
+        }
+      else
+        _dbus_string_free (&full_path);
+    }
+
+  if (result != DBUS_RESULT_SUCCESS)
+    {
+      const char *s;
+      _dbus_string_get_const_data (&test_directory, &s);
+      _dbus_warn ("Could not get next file in %s: %s\n",
+                  s, dbus_result_to_string (result));
+      goto failed;
+    }
+    
+  retval = TRUE;
+  
+ failed:
+
+  if (dir)
+    _dbus_directory_close (dir);
+  _dbus_string_free (&test_directory);
+  _dbus_string_free (&filename);
+
+  return retval;
+}
+                     
+
 /**
  * @ingroup DBusMessageInternals
  * Unit test for DBusMessage.
@@ -2025,7 +2354,7 @@ message_iter_test (DBusMessage *message)
  * @returns #TRUE on success.
  */
 dbus_bool_t
-_dbus_message_test (void)
+_dbus_message_test (const char *test_data_dir)
 {
   DBusMessage *message;
   DBusMessageLoader *loader;
@@ -2034,6 +2363,8 @@ _dbus_message_test (void)
   dbus_int32_t our_int;
   char *our_str;
   double our_double;
+  DBusString test_directory;
+  dbus_bool_t retval;
   
   /* Test the vararg functions */
   message = dbus_message_new ("org.freedesktop.DBus.Test", "testMessage");
@@ -2122,7 +2453,32 @@ _dbus_message_test (void)
   dbus_message_unref (message);
   _dbus_message_loader_unref (loader);
 
-  return TRUE;
+  /* Now load every message in test_data_dir if we have one */
+  if (test_data_dir == NULL)
+    return TRUE;
+
+  retval = FALSE;
+  
+  _dbus_string_init_const (&test_directory, test_data_dir);
+
+  if (!process_test_subdir (&test_directory, "valid-messages",
+                            MESSAGE_VALID))
+    goto failed;
+  if (!process_test_subdir (&test_directory, "invalid-messages",
+                            MESSAGE_INVALID))
+    goto failed;
+  
+  if (!process_test_subdir (&test_directory, "incomplete-messages",
+                            MESSAGE_INCOMPLETE))
+    goto failed;
+
+  retval = TRUE;
+  
+ failed:
+
+  _dbus_string_free (&test_directory);
+  
+  return retval;
 }
 
 #endif /* DBUS_BUILD_TESTS */
diff --git a/dbus/dbus-string.c b/dbus/dbus-string.c
index ac84cda8..246c9a1d 100644
--- a/dbus/dbus-string.c
+++ b/dbus/dbus-string.c
@@ -252,11 +252,11 @@ _dbus_string_init_const_len (DBusString *str,
 void
 _dbus_string_free (DBusString *str)
 {
-  DBUS_LOCKED_STRING_PREAMBLE (str);
+  DBusRealString *real = (DBusRealString*) str;
+  DBUS_GENERIC_STRING_PREAMBLE (real);
   
   if (real->constant)
     return;
-  
   dbus_free (real->str);
 
   real->invalid = TRUE;
@@ -864,8 +864,6 @@ copy (DBusRealString *source,
   _dbus_assert ((source) != (dest));                                    \
   DBUS_GENERIC_STRING_PREAMBLE (real_source);                           \
   DBUS_GENERIC_STRING_PREAMBLE (real_dest);                             \
-  _dbus_assert (!real_source->constant);                                \
-  _dbus_assert (!real_source->locked);                                  \
   _dbus_assert (!real_dest->constant);                                  \
   _dbus_assert (!real_dest->locked);                                    \
   _dbus_assert ((start) >= 0);                                          \
@@ -1404,6 +1402,46 @@ _dbus_string_starts_with_c_str (const DBusString *a,
     return FALSE;
 }
 
+/**
+ * Returns whether a string ends with the given suffix
+ *
+ * @param a the string
+ * @param c_str the C-style string
+ * @returns #TRUE if the string ends with the suffix
+ */
+dbus_bool_t
+_dbus_string_ends_with_c_str (const DBusString *a,
+                              const char       *c_str)
+{
+  const unsigned char *ap;
+  const unsigned char *bp;
+  const unsigned char *a_end;
+  int c_str_len;
+  const DBusRealString *real_a = (const DBusRealString*) a;
+  DBUS_GENERIC_STRING_PREAMBLE (real_a);
+
+  c_str_len = strlen (c_str);
+  if (real_a->len < c_str_len)
+    return FALSE;
+  
+  ap = real_a->str + (real_a->len - c_str_len);
+  bp = (const unsigned char*) c_str;
+  a_end = real_a->str + real_a->len;
+  while (ap != a_end)
+    {
+      if (*ap != *bp)
+        return FALSE;
+      
+      ++ap;
+      ++bp;
+    }
+
+  _dbus_assert (*ap == '\0');
+  _dbus_assert (*bp == '\0');
+  
+  return TRUE;
+}
+
 static const signed char base64_table[] = {
   /* 0 */ 'A',
   /* 1 */ 'B',
diff --git a/dbus/dbus-string.h b/dbus/dbus-string.h
index a0cc1ee0..c762a68f 100644
--- a/dbus/dbus-string.h
+++ b/dbus/dbus-string.h
@@ -165,6 +165,8 @@ dbus_bool_t _dbus_string_equal_c_str  (const DBusString *a,
 
 dbus_bool_t _dbus_string_starts_with_c_str  (const DBusString *a,
                                              const char       *c_str);
+dbus_bool_t _dbus_string_ends_with_c_str    (const DBusString *a,
+                                             const char       *c_str);
 
 dbus_bool_t _dbus_string_base64_encode (const DBusString *source,
                                         int               start,
diff --git a/dbus/dbus-sysdeps.c b/dbus/dbus-sysdeps.c
index d8b202c7..bcb81f7d 100644
--- a/dbus/dbus-sysdeps.c
+++ b/dbus/dbus-sysdeps.c
@@ -32,11 +32,13 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <sys/socket.h>
+#include <dirent.h>
 #include <sys/un.h>
 #include <pwd.h>
 #include <time.h>
 #include <sys/time.h>
 #include <sys/stat.h>
+
 #ifdef HAVE_WRITEV
 #include <sys/uio.h>
 #endif
@@ -1100,4 +1102,148 @@ _dbus_file_get_contents (DBusString       *str,
     }
 }
 
+/**
+ * Appends the given filename to the given directory.
+ *
+ * @param dir the directory name
+ * @param next_component the filename
+ * @returns #TRUE on success
+ */
+dbus_bool_t
+_dbus_concat_dir_and_file (DBusString       *dir,
+                           const DBusString *next_component)
+{
+  dbus_bool_t dir_ends_in_slash;
+  dbus_bool_t file_starts_with_slash;
+
+  if (_dbus_string_get_length (dir) == 0 ||
+      _dbus_string_get_length (next_component) == 0)
+    return TRUE;
+  
+  dir_ends_in_slash = '/' == _dbus_string_get_byte (dir,
+                                                    _dbus_string_get_length (dir) - 1);
+
+  file_starts_with_slash = '/' == _dbus_string_get_byte (next_component, 0);
+
+  if (dir_ends_in_slash && file_starts_with_slash)
+    {
+      _dbus_string_shorten (dir, 1);
+    }
+  else if (!(dir_ends_in_slash || file_starts_with_slash))
+    {
+      if (!_dbus_string_append_byte (dir, '/'))
+        return FALSE;
+    }
+
+  return _dbus_string_copy (next_component, 0, dir,
+                            _dbus_string_get_length (dir));
+}
+
+struct DBusDirIter
+{
+  DIR *d;
+  
+};
+
+/**
+ * Open a directory to iterate over.
+ *
+ * @param filename the directory name
+ * @param result return location for error code if #NULL returned
+ * @returns new iterator, or #NULL on error
+ */
+DBusDirIter*
+_dbus_directory_open (const DBusString *filename,
+                      DBusResultCode   *result)
+{
+  DIR *d;
+  DBusDirIter *iter;
+  const char *filename_c;
+
+  _dbus_string_get_const_data (filename, &filename_c);
+
+  d = opendir (filename_c);
+  if (d == NULL)
+    {
+      dbus_set_result (result, _dbus_result_from_errno (errno));
+      return NULL;
+    }
+  
+  iter = dbus_new0 (DBusDirIter, 1);
+  if (iter == NULL)
+    {
+      closedir (d);
+      dbus_set_result (result, DBUS_RESULT_NO_MEMORY);
+      return NULL;
+    }
+
+  iter->d = d;
+
+  return iter;
+}
+
+/**
+ * Get next file in the directory. Will not return "." or ".."
+ * on UNIX. If an error occurs, the contents of "filename"
+ * are undefined. #DBUS_RESULT_SUCCESS is always returned
+ * in result if no error occurs.
+ *
+ * @todo for thread safety, I think we have to use
+ * readdir_r(). (GLib has the same issue, should file a bug.)
+ *
+ * @param iter the iterator
+ * @param filename string to be set to the next file in the dir
+ * @param result return location for error, or #DBUS_RESULT_SUCCESS
+ * @returns #TRUE if filename was filled in with a new filename
+ */
+dbus_bool_t
+_dbus_directory_get_next_file (DBusDirIter      *iter,
+                               DBusString       *filename,
+                               DBusResultCode   *result)
+{
+  /* we always have to put something in result, since return
+   * value means whether there's a filename and doesn't
+   * reliably indicate whether an error was set.
+   */
+  struct dirent *ent;
+  
+  dbus_set_result (result, DBUS_RESULT_SUCCESS);
+
+ again:
+  errno = 0;
+  ent = readdir (iter->d);
+  if (ent == NULL)
+    {
+      dbus_set_result (result,
+                       _dbus_result_from_errno (errno));
+      return FALSE;
+    }
+  else if (ent->d_name[0] == '.' &&
+           (ent->d_name[1] == '\0' ||
+            (ent->d_name[1] == '.' && ent->d_name[2] == '\0')))
+    goto again;
+  else
+    {
+      _dbus_string_set_length (filename, 0);
+      if (!_dbus_string_append (filename, ent->d_name))
+        {
+          dbus_set_result (result, DBUS_RESULT_NO_MEMORY);
+          return FALSE;
+        }
+      else
+        return TRUE;
+    }
+}
+
+/**
+ * Closes a directory iteration.
+ */
+void
+_dbus_directory_close (DBusDirIter *iter)
+{
+  closedir (iter->d);
+  dbus_free (iter);
+}
+
+
 /** @} end of sysdeps */
diff --git a/dbus/dbus-sysdeps.h b/dbus/dbus-sysdeps.h
index 8ee7c8b7..fc552e7f 100644
--- a/dbus/dbus-sysdeps.h
+++ b/dbus/dbus-sysdeps.h
@@ -25,6 +25,7 @@
 #define DBUS_SYSDEPS_H
 
 #include <dbus/dbus-string.h>
+#include <dbus/dbus-errors.h>
 
 /* this is perhaps bogus, but strcmp() etc. are faster if we use the
  * stuff straight out of string.h, so have this here for now.
@@ -124,6 +125,19 @@ void _dbus_get_current_time (long *tv_sec,
 DBusResultCode _dbus_file_get_contents (DBusString       *str,
                                         const DBusString *filename);
 
+dbus_bool_t _dbus_concat_dir_and_file (DBusString       *dir,
+                                       const DBusString *next_component);
+
+typedef struct DBusDirIter DBusDirIter;
+
+DBusDirIter* _dbus_directory_open          (const DBusString *filename,
+                                            DBusResultCode   *result);
+dbus_bool_t  _dbus_directory_get_next_file (DBusDirIter      *iter,
+                                            DBusString       *filename,
+                                            DBusResultCode   *result);
+void         _dbus_directory_close         (DBusDirIter      *iter);
+
+
 DBUS_END_DECLS;
 
 #endif /* DBUS_SYSDEPS_H */
diff --git a/dbus/dbus-test-main.c b/dbus/dbus-test-main.c
index 8ed77cce..301021ee 100644
--- a/dbus/dbus-test-main.c
+++ b/dbus/dbus-test-main.c
@@ -31,6 +31,13 @@ int
 main (int    argc,
       char **argv)
 {
-  dbus_internal_symbol_do_not_use_run_tests ();
+  const char *test_data_dir;
+
+  if (argc > 1)
+    test_data_dir = argv[1];
+  else
+    test_data_dir = NULL;
+  
+  dbus_internal_symbol_do_not_use_run_tests (test_data_dir);
   return 0;
 }
diff --git a/dbus/dbus-test.c b/dbus/dbus-test.c
index 31ed51ab..337ef10b 100644
--- a/dbus/dbus-test.c
+++ b/dbus/dbus-test.c
@@ -23,13 +23,14 @@
 
 #include <config.h>
 #include "dbus-test.h"
+#include "dbus-sysdeps.h"
 #include <stdio.h>
 #include <stdlib.h>
 
 static void
 die (const char *failure)
 {
-  fprintf (stderr, "Failed: %s\n", failure);
+  fprintf (stderr, "Unit test failed: %s\n", failure);
   exit (1);
 }
 
@@ -39,11 +40,21 @@ die (const char *failure)
  * any app other than our test app, this symbol
  * won't exist in some builds of the library.
  * (with --enable-tests=no)
+ *
+ * @param test_data_dir the directory with test data (test/data normally)
  */
 void
-dbus_internal_symbol_do_not_use_run_tests (void)
+dbus_internal_symbol_do_not_use_run_tests (const char *test_data_dir)
 {
 #ifdef DBUS_BUILD_TESTS
+  if (test_data_dir == NULL)
+    test_data_dir = _dbus_getenv ("DBUS_TEST_DATA");
+
+  if (test_data_dir != NULL)
+    printf ("Test data in %s\n", test_data_dir);
+  else
+    printf ("No test data!\n");
+  
   printf ("%s: running string tests\n", "dbus-test");
   if (!_dbus_string_test ())
     die ("strings");
@@ -53,7 +64,7 @@ dbus_internal_symbol_do_not_use_run_tests (void)
     die ("marshalling");
 
   printf ("%s: running message tests\n", "dbus-test");
-  if (!_dbus_message_test ())
+  if (!_dbus_message_test (test_data_dir))
     die ("messages");
   
   printf ("%s: running memory pool tests\n", "dbus-test");
diff --git a/dbus/dbus-test.h b/dbus/dbus-test.h
index 64faf12b..ebc17bff 100644
--- a/dbus/dbus-test.h
+++ b/dbus/dbus-test.h
@@ -31,8 +31,8 @@ dbus_bool_t _dbus_list_test     (void);
 dbus_bool_t _dbus_marshal_test  (void);
 dbus_bool_t _dbus_mem_pool_test (void);
 dbus_bool_t _dbus_string_test   (void);
-dbus_bool_t _dbus_message_test  (void);
+dbus_bool_t _dbus_message_test  (const char *test_data_dir);
 
-void dbus_internal_symbol_do_not_use_run_tests (void);
+void dbus_internal_symbol_do_not_use_run_tests (const char *test_data_dir);
 
 #endif /* DBUS_TEST_H */
diff --git a/test/Makefile.am b/test/Makefile.am
index 4ddf2603..f0118ee6 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -24,3 +24,13 @@ TEST_LIBS=$(DBUS_TEST_LIBS) $(top_builddir)/dbus/libdbus-convenience.la $(top_bu
 echo_client_LDADD=$(TEST_LIBS)
 echo_server_LDADD=$(TEST_LIBS)
 
+dist-hook:
+	DIRS="data data/valid-messages" ;			\
+	for D in $DIRS; do					\
+		test -d $(distdir)/$D || mkdir $(distdir)/$D ;	\
+	done ;							\
+	FILES=`find -name "*.message"` ;			\
+        for F in $FILES; do					\
+                echo '-- Disting file '$$F ;			\
+		cp $F $(distdir)/$$F ;				\
+	done
diff --git a/test/data/incomplete-messages/missing-body.message b/test/data/incomplete-messages/missing-body.message
new file mode 100644
index 00000000..c97ef7a3
--- /dev/null
+++ b/test/data/incomplete-messages/missing-body.message
@@ -0,0 +1,12 @@
+## message that's missing an expected body
+
+VALID_HEADER
+END_LENGTH Header
+
+## create the body, then chop it off
+START_LENGTH Body
+TYPE INT32
+INT32 37
+END_LENGTH Body
+
+CHOP 8
diff --git a/test/data/invalid-messages/bad-endian.message b/test/data/invalid-messages/bad-endian.message
new file mode 100644
index 00000000..7a7b75d1
--- /dev/null
+++ b/test/data/invalid-messages/bad-endian.message
@@ -0,0 +1,13 @@
+## message with invalid endianness tag
+
+BYTE 'i'
+BYTE 0
+BYTE 0
+BYTE 0
+LENGTH Header
+LENGTH Body
+## client serial
+INT32 7
+END_LENGTH Header
+START_LENGTH Body
+END_LENGTH Body
diff --git a/test/data/valid-messages/simplest-manual.message b/test/data/valid-messages/simplest-manual.message
new file mode 100644
index 00000000..bf5ddc5b
--- /dev/null
+++ b/test/data/valid-messages/simplest-manual.message
@@ -0,0 +1,14 @@
+## like simplest.message, but doesn't use VALID_HEADER
+## convenience command. mostly to test the test framework.
+
+BYTE 'l'
+BYTE 0
+BYTE 0
+BYTE 0
+LENGTH Header
+LENGTH Body
+## client serial
+INT32 7
+END_LENGTH Header
+START_LENGTH Body
+END_LENGTH Body
diff --git a/test/data/valid-messages/simplest.message b/test/data/valid-messages/simplest.message
index 949aa852..872a58a6 100644
--- a/test/data/valid-messages/simplest.message
+++ b/test/data/valid-messages/simplest.message
@@ -1,6 +1,7 @@
 ## simplest possible valid message
 
-## this does a LENGTH Header and LENGTH Body
+## VALID_HEADER includes a LENGTH Header and LENGTH Body
 VALID_HEADER
-SET_LENGTH Header
-SET_LENGTH Body
+END_LENGTH Header
+START_LENGTH Body
+END_LENGTH Body