Skip to content

Commit

Permalink
Merge pull request #14 from fluggo/feature/2.0
Browse files Browse the repository at this point in the history 
2.0.0: Asynchronous loading and null-on-failed-parse
  • Loading branch information
@Beh01der
Beh01der committed May 6, 2016
2 parents 0ed906d + 9c47859 commit 33a25a9
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 47 deletions.
8 changes: 5 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,17 @@ console.log(pattern.parseSync(str));
```

## API
* **loadDefault(callback, [loadModules])** - creates default pattern collection including all built-in patterns from `./patterns` folder. By providing *loadModules* parameter you can limit number of loaded patterns: `loadDefault(..., ['grok-patterns']);`. Callback receives *patterns* collection filled in with default templates: `function(patterns)`
* **loadDefault([loadModules,] callback)** - creates new pattern collection including all built-in patterns from `./patterns` folder. By providing *loadModules* parameter you can limit number of loaded patterns: `loadDefault(['grok-patterns'] ,...);`. Callback receives *patterns* collection filled in with default templates: `function(err, patterns)`.

* **loadDefaultSync([loadModules])** - creates default pattern collection and returns it `GrokCollection`.
* **loadDefaultSync([loadModules])** - creates new default pattern collection and returns it `GrokCollection`.

* **new GrokCollection()** - creates a new empty pattern collection.

* **GrokCollection.createPattern(expression, [id])** - creates new pattern and adds it to the collection. Find out more about pattern syntax [here](http://logstash.net/docs/1.4.2/filters/grok) and about regular expression syntax [here](http://www.geocities.jp/kosako3/oniguruma/doc/RE.txt)

* **GrokCollection.getPattern(id)** - returns existing pattern `GrokPattern`

* **GrokCollection.load(filePath, next)** - loads patterns from file
* **GrokCollection.load(filePath, callback)** - asynchronously loads patterns from file. Callback is `function(err)`.

* **GrokCollection.loadSync(filePath)** - loads patterns from file and returns number of newly loaded patterns `number`

Expand Down
121 changes: 79 additions & 42 deletions lib/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
var fs = require('fs');
var async = require('async');
var OnigRegExp = require('oniguruma').OnigRegExp;
var Map = require('collections/fast-map');

Expand All @@ -16,18 +17,19 @@ function GrokPattern(expression, id) {
}

t.regexp.search(str, function(err, result) {
if(err || !result)
return next(err, result);

var r = {};

if (!err && result) {
result.forEach(function(item, index) {
var field = t.fields[index];
if(field && item.match) {
r[field] = item.match;
}
});
}
result.forEach(function(item, index) {
var field = t.fields[index];
if(field && item.match) {
r[field] = item.match;
}
});

next(err, r, result);
return next(err, r, result);
});
};

Expand All @@ -37,23 +39,25 @@ function GrokPattern(expression, id) {
}

var result = t.regexp.searchSync(str);

if(!result)
return null;

var r = {};

if (result) {
result.forEach(function(item, index) {
var field = t.fields[index];
if(field && item.match) {
r[field] = item.match;
}
});
}
result.forEach(function(item, index) {
var field = t.fields[index];
if(field && item.match) {
r[field] = item.match;
}
});

return r;
};
}

var subPatternsRegex = /%\{[A-Z0-9_]+(?::[a-z0-9_]+)?\}/g; // %{subPattern} or %{subPattern:fieldName}
var nestedFieldNamesRegex = /(\(\?<([a-z0-9_]+)>)|\(\?:|\(\?>|\(\?!|\(\?<!|\(|\\\(|\\\)|\)|\[|\\\[|\\\]|\]/g
var subPatternsRegex = /%\{[A-Z0-9_]+(?::[A-Za-z0-9_]+)?\}/g; // %{subPattern} or %{subPattern:fieldName}
var nestedFieldNamesRegex = /(\(\?<([A-Za-z0-9_]+)>)|\(\?:|\(\?>|\(\?!|\(\?<!|\(|\\\(|\\\)|\)|\[|\\\[|\\\]|\]/g

function GrokCollection() {
var t = this;
Expand Down Expand Up @@ -134,9 +138,9 @@ function GrokCollection() {
var patternLineRegex = /^([A-Z0-9_]+)\s+(.+)/;
var splitLineRegex = /\r?\n/;

function doLoad(filePath) {
function doLoad(file) {
var i = 0;
var file = fs.readFileSync(filePath);

if (file) {
var lines = file.toString().split(splitLineRegex);
if (lines && lines.length) {
Expand Down Expand Up @@ -170,44 +174,77 @@ function GrokCollection() {
return patterns.get(id);
};

t.load = function (filePath, next) {
doLoad(filePath);
next();
t.load = function (filePath, callback) {
fs.readFile(filePath, function(err, file) {
if(err)
return callback(err);

doLoad(file);
return callback();
});
};

t.loadSync = doLoad;
t.loadSync = function(filePath) {
return doLoad(fs.readFileSync(filePath));
};

t.count = function () {
return patterns.length;
};
}

var defaultPatterns;
var patternsDir = __dirname + '/patterns/';

function doLoadDefault(loadModules) {
if (!defaultPatterns) {
defaultPatterns = new GrokCollection();
var files = fs.readdirSync(patternsDir);
if (files && files.length) {
files.filter(function(file) {
return !loadModules || !loadModules.length || loadModules.indexOf(file) !== -1;
}).forEach(function (file) {
defaultPatterns.loadSync(patternsDir + file);
})
}
function doLoadDefaultSync(loadModules) {
var result = new GrokCollection();

var files = fs.readdirSync(patternsDir);
if (files && files.length) {
files.filter(function(file) {
return !loadModules || !loadModules.length || loadModules.indexOf(file) !== -1;
}).forEach(function (file) {
result.loadSync(patternsDir + file);
})
}

return defaultPatterns;
return result;
}

function doLoadDefault(loadModules, callback) {
return fs.readdir(patternsDir, function(err, files) {
if(err)
return callback(err);

var result = new GrokCollection();

return async.parallel(
files.filter(function(file) {
return !loadModules || !loadModules.length || loadModules.indexOf(file) !== -1;
}).map(function (file) {
return function(callback) {
return result.load(patternsDir + file, callback);
};
}),
function(err) {
if(err)
return callback(err);

return callback(null, result);
});
});
}

module.exports = {
loadDefault: function (callback, loadModules) {
doLoadDefault(loadModules);
callback(defaultPatterns);
loadDefault: function (loadModules, callback) {
if(arguments.length < 2) {
callback = loadModules;
loadModules = null;
}

doLoadDefault(loadModules, callback);
},

loadDefaultSync: doLoadDefault,
loadDefaultSync: doLoadDefaultSync,

GrokCollection: GrokCollection
};
94 changes: 93 additions & 1 deletion lib/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ var grok = require('./index.js');
var expect = require("chai").expect;

function testParse(p, str, expected, done) {
grok.loadDefault(function (patterns) {
grok.loadDefault(function (err, patterns) {
expect(err).to.be.null;

var pattern = patterns.createPattern(p);
pattern.parse(str, function (err, result) {
expect(err).to.be.null;
Expand All @@ -13,8 +15,74 @@ function testParse(p, str, expected, done) {
}

describe('grok', function() {
describe('loadDefault', function() {
it('is asynchronous', function() {
var isDone = false;

grok.loadDefault(function(patterns) {
isDone = true;
});

expect(isDone, 'was done immediately after return').to.be.false;
});
});

describe('#parseSync()', function () {
it('returns null when a parse fails', function() {
var patterns = grok.loadDefaultSync();
var pattern = patterns.createPattern('%{WORD:verb} %{WORD:adjective}');

var result = pattern.parseSync('test');
expect(result).to.be.null;
});
});

describe('#parse()', function () {

it('is asynchronous', function(done) {
grok.loadDefault(function (err, patterns) {
expect(err).to.be.null;

var pattern = patterns.createPattern('%{WORD:verb}');
var isDone = false;

pattern.parse('test', function(err, result) {
isDone = true;
});

expect(isDone).to.be.false;
done();
});
});

it('returns null when a parse fails', function(done) {
grok.loadDefault(function (err, patterns) {
expect(err).to.be.null;

var pattern = patterns.createPattern('%{WORD:verb} %{WORD:adjective}');

pattern.parse('test', function(err, result) {
expect(err).to.not.exist;
expect(result).to.be.null;
done();
});
});
});

it('parses to attributes with uppercase in their names', function(done) {
grok.loadDefault(function (err, patterns) {
expect(err).to.be.null;

var pattern = patterns.createPattern('%{WORD:verb} %{WORD:testVariable}');

pattern.parse('test worp', function(err, result) {
expect(err).to.not.exist;
expect(result).to.deep.equal({verb: 'test', testVariable: 'worp'});
done();
});
});
});

it('should parse a simple custom pattern', function (done) {
var p = '(?<verb>\\w+)\\s+(?<url>/\\w+)';
var str = 'DELETE /ping HTTP/1.1';
Expand Down Expand Up @@ -144,3 +212,27 @@ describe('grok', function() {

});
});

describe('GrokCollection', function() {
describe('load', function() {
it('is asynchronous', function() {
var coll = new grok.GrokCollection();
var isDone = false;

coll.load(require.resolve('./patterns/grok-patterns'), function() {
isDone = true;
});

expect(isDone, 'was done immediately after return').to.be.false;
});
});

describe('loadSync', function() {
it('returns number of patterns', function() {
var coll = new grok.GrokCollection();
var result = coll.loadSync(require.resolve('./patterns/grok-patterns'));

expect(result, 'should match number of loaded patterns').to.equal(coll.count());
});
});
});
3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "node-grok",
"version": "1.0.9",
"version": "2.0.0",
"description": "Regular expression template library inspired by logstash grok filter module",
"repository": "https://github.com/Beh01der/node-grok.git",
"main": "./lib/index.js",
Expand All @@ -15,6 +15,7 @@
"author": "Andrey Chausenko",
"license": "ISC",
"dependencies": {
"async": "^2.0.0-rc.2",
"collections": "3.0.0",
"oniguruma": "6.0.1"
},
Expand Down

0 comments on commit 33a25a9

Please sign in to comment.