fix tokenize

base/str.cc:
    Fix tokenize so that it doesn't behave incorrectly when there
    are empty strings.
test/tokentest.cc:
    Clean up the test function so it's easier to see what's going on

--HG--
extra : convert_revision : c7a3db7bc516d3575b1cc4ab7afbd0f1fbe1ec6f

1 files changed, 12 insertions, 11 deletions

diff --git a/base/str.cc b/base/str.cc
index 3a11bb17d..15f44dad2 100644
--- a/base/str.cc
+++ b/base/str.cc
@@ -26,11 +26,10 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <iostream>
-
-#include <string.h>
 #include <ctype.h>
 
+#include <cstring>
+#include <iostream>
 #include <string>
 #include <vector>
 
@@ -75,15 +74,17 @@ tokenize(vector<string>& v, const string &s, char token, bool ignore)
     string::size_type first = 0;
     string::size_type last = s.find_first_of(token);
 
-    if (ignore) {
-        if (last == first) {
-            while (last == first)
-                last = s.find_first_of(token, ++first);
+    if (s.empty())
+        return;
 
-            if (last == string::npos) {
-                v.push_back(s);
-                return;
-            }
+    if (ignore && last == first) {
+        while (last == first)
+            last = s.find_first_of(token, ++first);
+
+        if (last == string::npos) {
+            if (first != s.size())
+                v.push_back(s.substr(first));
+            return;
         }
     }