Add utility for tokenizing objects.

Gruntfile.js

@@ -6,6 +6,7 @@ var semver = require('semver'),
       ],
       bloodhound: [
       'src/bloodhound/version.js',
+      'src/bloodhound/tokenizers.js',
       'src/bloodhound/lru_cache.js',
       'src/bloodhound/persistent_storage.js',
       'src/bloodhound/transport.js',

karma.conf.js

@@ -14,6 +14,7 @@ files = [
   'test/vendor/**/*',
   'src/common/utils.js',
   'src/bloodhound/version.js',
+  'src/bloodhound/tokenizers.js',
   'src/bloodhound/lru_cache.js',
   'src/bloodhound/persistent_storage.js',
   'src/bloodhound/transport.js',

src/bloodhound/bloodhound.js

@@ -51,15 +51,7 @@
     return Bloodhound;
   };
 
-  Bloodhound.tokenizers = {
-    whitespace: function whitespaceTokenizer(s) {
-      return s.split(/\s+/);
-    },
-
-    nonword: function nonwordTokenizer(s) {
-      return s.split(/\W+/);
-    }
-  };
+  Bloodhound.tokenizers = tokenizers;
 
   // instance methods
   // ----------------

src/bloodhound/tokenizers.js

@@ -0,0 +1,27 @@
+/*
+ * typeahead.js
+ * https://github.com/twitter/typeahead.js
+ * Copyright 2013-2014 Twitter, Inc. and other contributors; Licensed MIT
+ */
+
+var tokenizers = (function(root) {
+
+  return {
+    nonword: nonword,
+    whitespace: whitespace,
+    obj: {
+      nonword: getObjTokenizer(nonword),
+      whitespace: getObjTokenizer(whitespace)
+    }
+  };
+
+  function whitespace(s) { return s.split(/\s+/); }
+
+  function nonword(s) { return s.split(/\W+/); }
+
+  function getObjTokenizer(tokenizer) {
+    return function setKey(key) {
+      return function tokenize(o) { return tokenizer(o[key]); };
+    };
+  }
+})();

test/tokenizers_spec.js

@@ -0,0 +1,26 @@
+describe('tokenizers', function() {
+
+  it('.whitespace should tokenize on whitespace', function() {
+    var tokens = tokenizers.whitespace('big-deal ok');
+    expect(tokens).toEqual(['big-deal', 'ok']);
+  });
+
+  it('.nonword should tokenize on non-word characters', function() {
+    var tokens = tokenizers.nonword('big-deal ok');
+    expect(tokens).toEqual(['big', 'deal', 'ok']);
+  });
+
+  it('.obj.whitespace should tokenize on whitespace', function() {
+    var t = tokenizers.obj.whitespace('val');
+    var tokens = t({ val: 'big-deal ok' });
+
+    expect(tokens).toEqual(['big-deal', 'ok']);
+  });
+
+  it('.obj.nonword should tokenize on non-word characters', function() {
+    var t = tokenizers.obj.nonword('val');
+    var tokens = t({ val: 'big-deal ok' });
+
+    expect(tokens).toEqual(['big', 'deal', 'ok']);
+  });
+});