From e3c2e7a3bd461831150fbbfc36366f56329c8fcb Mon Sep 17 00:00:00 2001 From: Oliver Brooks Date: Tue, 5 Jul 2016 14:42:07 +0100 Subject: [PATCH 1/4] add sync API --- README.md | 23 +++++- index.js | 36 +++++++++ lib/operators/param.js | 7 ++ lib/operators/require.js | 13 ++++ lib/parser/sync.js | 153 +++++++++++++++++++++++++++++++++++++++ sync.js | 34 +++++++++ test/functional/async.js | 39 ++++++++++ test/functional/index.js | 38 +--------- test/functional/sync.js | 36 +++++++++ 9 files changed, 340 insertions(+), 39 deletions(-) create mode 100644 lib/parser/sync.js create mode 100644 sync.js create mode 100644 test/functional/async.js create mode 100644 test/functional/sync.js diff --git a/README.md b/README.md index 1140b91..ceb1284 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # sql-stamp The tiny SQL templating library, with the aim to be as simple as possible so to not get in the way of you writing SQL. -[![Build Status](https://travis-ci.org/orangemug/sql-stamp.svg?branch=master)](https://travis-ci.org/orangemug/sql-stamp) -[![Test Coverage](https://codeclimate.com/github/orangemug/sql-stamp/badges/coverage.svg)](https://codeclimate.com/github/orangemug/sql-stamp/coverage) -[![Code Climate](https://codeclimate.com/github/orangemug/sql-stamp/badges/gpa.svg)](https://codeclimate.com/github/orangemug/sql-stamp) +[![Build Status](https://travis-ci.org/orangemug/sql-stamp.svg?branch=master)](https://travis-ci.org/orangemug/sql-stamp) +[![Test Coverage](https://codeclimate.com/github/orangemug/sql-stamp/badges/coverage.svg)](https://codeclimate.com/github/orangemug/sql-stamp/coverage) +[![Code Climate](https://codeclimate.com/github/orangemug/sql-stamp/badges/gpa.svg)](https://codeclimate.com/github/orangemug/sql-stamp) [![Dependency Status](https://david-dm.org/orangemug/sql-stamp.svg)](https://david-dm.org/orangemug/sql-stamp) [![Dev Dependency Status](https://david-dm.org/orangemug/sql-stamp/dev-status.svg)](https://david-dm.org/orangemug/sql-stamp#info=devDependencies) @@ -112,6 +112,23 @@ You'll get more descriptive errors about where the error happened in your source You can see some more examples in the tests here [here](test/errors/index.js) +## Sync API + +sql-stamp has a synchronous API. This is useful for processing SQL templates and exporting from node modules. + +```js + var sqlStamp = require("sql-stamp/sync"); + var templater = sqlStamp([ + /* Pass a list of SQL templates */ + __dirname+"/friends.sql", + __dirname+"/example.sql" + ]); + templater1(__dirname+"/example.sql", {foo: "bar"}); // => {sql: "select...", args: ["bar"]} + + var files = glob.sync("./sql/**/*.sql") + var templater2 = sqlStamp(files); + templater2(__dirname+"../lib/sql/foo.sql", {foo: "bar"}); // => {sql: "select...", args: ["bar"]} +``` ## Test Run the unit tests diff --git a/index.js b/index.js index e7baccf..d648ce9 100644 --- a/index.js +++ b/index.js @@ -38,3 +38,39 @@ module.exports = function(files, opts, callback) { }) .nodeify(callback); } + + +/** + * Initialize a SQL templater synchronously + * @param {Array} files + * @param {Object} [opts] + * @param {Function} [callback] + * @return {Promise} + */ +exports.sync = function(files, opts) { + opts = assign({ + prettyErrors: false + }, opts); + + // Generate the templates + var templates = {}; + files.forEach(function(filepath) { + filepath = path.resolve(filepath); + templates[filepath] = parser(filepath, templates, opts); + }); + + return Bluebird + .props(templates) + .then(function(_templates) { + return function(key, data) { + key = path.normalize(key); + + if(_templates.hasOwnProperty(key)) { + return _templates[key](data); + } else { + throw new errors.NoSuchTemplate(key); + } + } + }) + .nodeify(callback); +} diff --git a/lib/operators/param.js b/lib/operators/param.js index 4d9df93..0309455 100644 --- a/lib/operators/param.js +++ b/lib/operators/param.js @@ -8,6 +8,13 @@ module.exports = { throw new errors.TooManyArgs(); } }, + parseSync: function(key, dflt) { + if(arguments.length < 1) { + throw new errors.TooFewArgs(); + } else if (arguments.length > 2) { + throw new errors.TooManyArgs(); + } + }, run: function(key, dflt) { var arg; if(!this.data.hasOwnProperty(key) && dflt === undefined) { diff --git a/lib/operators/require.js b/lib/operators/require.js index 1e6e278..d2c4a1b 100644 --- a/lib/operators/require.js +++ b/lib/operators/require.js @@ -3,6 +3,19 @@ var path = require("path"); var errors = require("../errors"); module.exports = { + parseSync: function(relFilepath, dataKey) { + var filepath = path.resolve(path.dirname(this.path), relFilepath); + + if(!this.templates.hasOwnProperty(filepath)) { + return { + template: this.recurse(filepath) + }; + } else { + return { + template: this.templates[filepath] + }; + } + }, parse: function(relFilepath, dataKey) { var filepath = path.resolve(path.dirname(this.path), relFilepath); diff --git a/lib/parser/sync.js b/lib/parser/sync.js new file mode 100644 index 0000000..3f87736 --- /dev/null +++ b/lib/parser/sync.js @@ -0,0 +1,153 @@ +var util = require("../util"); +var operators = require("../operators"); +var runner = require("./runner"); +var stack = require("./stack"); +var errors = require("../errors"); + +var fs = require("fs"); + +function parseRuleSync(templates, filepath, raw, type, args, indent, ln, opts) { + var operator = operators[type]; + var operatorArgs = args.split(",") + .map(util.chomp) + .map(util.removeQuotes) + .filter(util.removeEmpty); + + var _args; + // Operator + if(operator.parseSync) { + _args = operator.parseSync.apply({ + templates: templates, + recurse: function(filepath) { + var data = parseSync(filepath, templates, opts); + // As a cache so we don't need to parse twice + templates[filepath] = data; + return data; + }, + path: filepath + }, operatorArgs); + } + + return { + sql: raw, + ln: ln, + fn: function(ctx) { + var out = operator.run.apply({templates: templates, data: ctx, parseArgs: _args}, operatorArgs); + + // TODO: Nasty + out.sql = out.sql.split("\n").map(function(line, idx) { + if(idx > 0) { + return indent + line; + } else { + return line; + } + }).join("\n"); + return out; + } + }; +} + + + + + +function parseSync(filepath, templates, opts) { + try { + var sqlRaw = util.chomp( + fs.readFileSync(filepath) + ); + + var tokens = []; + var errStack = []; + var indent = ""; + var foundChar = false; + + var re = /(?:([^}{\n]+)|(\{(.)([^}]*)\})|([\n]))/gm; + var ln = 0; + + var posMap = { + sql: 1, + cmd: 2, + cmdType: 3, + cmdArgs: 4, + nl: 5 + }; + + var idx = -1; + var m; + + while(m = re.exec(sqlRaw)) { + idx++; + if(!foundChar && m[0].match(/^\s*$/m)) { + indent += m[0]; + } else { + foundChar = true; + } + + if(m[posMap.sql]) { + tokens.push({ + sql: m[posMap.sql], + ln: ln, + idx: idx + }); + } else if(m[posMap.cmd]) { + var _m = m; + var _ln = ln; + var _idx = idx; + + if(m[posMap.cmdType].match(/^[=!>?]/)) { + + try { + var token = parseRuleSync(templates, filepath, m[posMap.cmd], m[posMap.cmdType], m[posMap.cmdArgs], indent, ln, opts) + tokens.push(token); + } catch (err) { + errStack.push({ + indent: indent, + idx: _idx, + err: err + }); + + tokens.push({ + ln: _ln, + sql: _m[0], + idx: _idx + }); + } + } else { + tokens.push({ + sql: m[posMap.cmd], + ln: ln, + idx: idx + }); + } + + } else if(m[posMap.nl]) { + tokens.push({ + sql: m[posMap.nl], + ln: ln, + idx: idx + }); + indent = ""; + foundChar = false; + ln++; + } + } + + if(errStack.length > 0) { + // TODO: Should be tracing all errStack here... + throw errStack[0].err + stack.trace(tokens, errStack[0], 100); + } + + // Return a template runner + var ret = runner.bind(null, tokens, opts); + return ret; + + } catch (err) { + if(err.code === "ENOENT") { + throw new errors.SQLError("No such template '"+filepath+"'"); + } + throw err; + } +} + +module.exports = parseSync; diff --git a/sync.js b/sync.js new file mode 100644 index 0000000..6b22275 --- /dev/null +++ b/sync.js @@ -0,0 +1,34 @@ +var assign = require("lodash.assign"); +var path = require("path"); +var parser = require("./lib/parser/sync"); +var errors = require("./lib/errors"); + +/** + * Initialize a SQL templater synchronously + * @param {Array} files + * @param {Object} [opts] + * @param {Function} [callback] + * @return {Promise} + */ +module.exports = function(files, opts) { + opts = assign({ + prettyErrors: false + }, opts); + + // Generate the templates + var templates = {}; + files.forEach(function(filepath) { + filepath = path.resolve(filepath); + templates[filepath] = parser(filepath, templates, opts); + }); + + return function(key, data) { + key = path.normalize(key); + + if(templates.hasOwnProperty(key)) { + return templates[key](data); + } else { + throw new errors.NoSuchTemplate(key); + } + } +} diff --git a/test/functional/async.js b/test/functional/async.js new file mode 100644 index 0000000..bd7955c --- /dev/null +++ b/test/functional/async.js @@ -0,0 +1,39 @@ +var assert = require("assert"); +var sqlStamp = require("../../"); +var util = require("../util"); + +var results = util.readSync([ + "./out.sql" +], __dirname); + +describe("end-to-end", function() { + + describe("async", function () { + var tmpl; + + before(function(done) { + sqlStamp([ + __dirname+"/example.sql", + __dirname+"/friends.sql", + ], {}, function(err, _tmpl) { + tmpl = _tmpl; + done(); + }); + }); + + + it("should work", function() { + var out = tmpl(__dirname+"/example.sql", { + accountId: 1, + filterDisabled: false, + filterKey: "role", + filterVal: "dev" + }); + + assert.equal(out.args.length, 2); + assert.equal(out.args[0], 1); + assert.equal(out.args[1], "dev"); + assert.equal(out.sql, results["./out.sql"]); + }); + }); +}); diff --git a/test/functional/index.js b/test/functional/index.js index 09b3275..eb6a616 100644 --- a/test/functional/index.js +++ b/test/functional/index.js @@ -1,36 +1,2 @@ -var assert = require("assert"); -var sqlStamp = require("../../"); -var util = require("../util"); - -var results = util.readSync([ - "./out.sql" -], __dirname); - -describe("end-to-end", function() { - var tmpl; - - before(function(done) { - sqlStamp([ - __dirname+"/example.sql", - __dirname+"/friends.sql", - ], {}, function(err, _tmpl) { - tmpl = _tmpl; - done(); - }); - }); - - - it("should work", function() { - var out = tmpl(__dirname+"/example.sql", { - accountId: 1, - filterDisabled: false, - filterKey: "role", - filterVal: "dev" - }); - - assert.equal(out.args.length, 2); - assert.equal(out.args[0], 1); - assert.equal(out.args[1], "dev"); - assert.equal(out.sql, results["./out.sql"]); - }); -}); +require("./async"); +require("./sync"); diff --git a/test/functional/sync.js b/test/functional/sync.js new file mode 100644 index 0000000..e412d38 --- /dev/null +++ b/test/functional/sync.js @@ -0,0 +1,36 @@ +var assert = require("assert"); +var sqlStampSync = require("../../sync"); +var util = require("../util"); + +var results = util.readSync([ + "./out.sql" +], __dirname); + +describe("end-to-end", function() { + + describe("sync", function () { + var tmpl; + + before(function() { + tmpl = sqlStampSync([ + __dirname+"/example.sql", + __dirname+"/friends.sql", + ]); + }); + + + it("should work", function() { + var out = tmpl(__dirname+"/example.sql", { + accountId: 1, + filterDisabled: false, + filterKey: "role", + filterVal: "dev" + }); + + assert.equal(out.args.length, 2); + assert.equal(out.args[0], 1); + assert.equal(out.args[1], "dev"); + assert.equal(out.sql, results["./out.sql"]); + }); + }); +}); From 503ebd3b76f081e5dacbf119d60678b67f58422b Mon Sep 17 00:00:00 2001 From: Oliver Brooks Date: Tue, 5 Jul 2016 14:44:00 +0100 Subject: [PATCH 2/4] remove unused code from index --- index.js | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/index.js b/index.js index d648ce9..ab906fd 100644 --- a/index.js +++ b/index.js @@ -37,40 +37,4 @@ module.exports = function(files, opts, callback) { } }) .nodeify(callback); -} - - -/** - * Initialize a SQL templater synchronously - * @param {Array} files - * @param {Object} [opts] - * @param {Function} [callback] - * @return {Promise} - */ -exports.sync = function(files, opts) { - opts = assign({ - prettyErrors: false - }, opts); - - // Generate the templates - var templates = {}; - files.forEach(function(filepath) { - filepath = path.resolve(filepath); - templates[filepath] = parser(filepath, templates, opts); - }); - - return Bluebird - .props(templates) - .then(function(_templates) { - return function(key, data) { - key = path.normalize(key); - - if(_templates.hasOwnProperty(key)) { - return _templates[key](data); - } else { - throw new errors.NoSuchTemplate(key); - } - } - }) - .nodeify(callback); -} +}; From 7440e2f2caa04f47dcd9e5c617e952aa0a490158 Mon Sep 17 00:00:00 2001 From: Oliver Brooks Date: Tue, 5 Jul 2016 19:20:34 +0100 Subject: [PATCH 3/4] remove some duplicate logic and test parser --- lib/operators/param.js | 7 ------- lib/operators/require.js | 16 +--------------- lib/parser/index.js | 29 ++++++++++++++--------------- lib/parser/regex.js | 12 ++++++++++++ lib/parser/sync.js | 32 +++++++++++--------------------- test/index.js | 1 + test/parser/regex.js | 38 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 77 insertions(+), 58 deletions(-) create mode 100644 lib/parser/regex.js create mode 100644 test/parser/regex.js diff --git a/lib/operators/param.js b/lib/operators/param.js index 0309455..4d9df93 100644 --- a/lib/operators/param.js +++ b/lib/operators/param.js @@ -8,13 +8,6 @@ module.exports = { throw new errors.TooManyArgs(); } }, - parseSync: function(key, dflt) { - if(arguments.length < 1) { - throw new errors.TooFewArgs(); - } else if (arguments.length > 2) { - throw new errors.TooManyArgs(); - } - }, run: function(key, dflt) { var arg; if(!this.data.hasOwnProperty(key) && dflt === undefined) { diff --git a/lib/operators/require.js b/lib/operators/require.js index d2c4a1b..51f5c01 100644 --- a/lib/operators/require.js +++ b/lib/operators/require.js @@ -1,9 +1,8 @@ -var Bluebird = require("bluebird"); var path = require("path"); var errors = require("../errors"); module.exports = { - parseSync: function(relFilepath, dataKey) { + parse: function(relFilepath, dataKey) { var filepath = path.resolve(path.dirname(this.path), relFilepath); if(!this.templates.hasOwnProperty(filepath)) { @@ -16,19 +15,6 @@ module.exports = { }; } }, - parse: function(relFilepath, dataKey) { - var filepath = path.resolve(path.dirname(this.path), relFilepath); - - if(!this.templates.hasOwnProperty(filepath)) { - return Bluebird.props({ - template: this.recurse(filepath) - }); - } else { - return Bluebird.props({ - template: this.templates[filepath] - }); - } - }, run: function(relFilepath, dataKey) { return this.parseArgs.template( dataKey ? this.data[dataKey] : this.data diff --git a/lib/parser/index.js b/lib/parser/index.js index 09978e2..8c75131 100644 --- a/lib/parser/index.js +++ b/lib/parser/index.js @@ -1,9 +1,10 @@ -var Bluebird = require("bluebird"); -var util = require("../util"); -var operators = require("../operators"); -var runner = require("./runner"); -var stack = require("./stack"); -var errors = require("../errors"); +var Bluebird = require("bluebird"); +var errors = require("../errors"); +var operators = require("../operators"); +var parserRegex = require("./regex"); +var runner = require("./runner"); +var stack = require("./stack"); +var util = require("../util"); var fs = Bluebird.promisifyAll( require("fs") @@ -21,7 +22,7 @@ function parseRule(templates, filepath, raw, type, args, indent, ln, opts) { return Bluebird.resolve() .then(function() { if(operator.parse) { - return operator.parse.apply({ + var obj = operator.parse.apply({ templates: templates, recurse: function(filepath) { return parse(filepath, templates, opts) @@ -33,6 +34,10 @@ function parseRule(templates, filepath, raw, type, args, indent, ln, opts) { }, path: filepath }, operatorArgs); + + if (obj) { + return Bluebird.props(obj); + } } }) .then(function(_args) { @@ -70,16 +75,10 @@ function parse(filepath, templates, opts) { var indent = ""; var foundChar = false; - var re = /(?:(\{([#=?!>])([^}{]*)\})|([\n])|(.[^}{\n]*))/gm; + var re = parserRegex.re(); var ln = 0; - var posMap = { - cmd: 1, - cmdType: 2, - cmdArgs: 3, - nl: 4, - sql: 5, - }; + var posMap = parserRegex.posMap; var idx = -1; var m; diff --git a/lib/parser/regex.js b/lib/parser/regex.js new file mode 100644 index 0000000..009e1e7 --- /dev/null +++ b/lib/parser/regex.js @@ -0,0 +1,12 @@ +module.exports = { + re: function () { + return /(?:(\{([#=?!>])([^}{]*)\})|([\n])|(.[^}{\n]*))/gm; + }, + posMap: { + cmd: 1, + cmdType: 2, + cmdArgs: 3, + nl: 4, + sql: 5, + } +}; diff --git a/lib/parser/sync.js b/lib/parser/sync.js index 3f87736..60da176 100644 --- a/lib/parser/sync.js +++ b/lib/parser/sync.js @@ -1,10 +1,10 @@ -var util = require("../util"); -var operators = require("../operators"); -var runner = require("./runner"); -var stack = require("./stack"); -var errors = require("../errors"); - -var fs = require("fs"); +var errors = require("../errors"); +var fs = require("fs"); +var operators = require("../operators"); +var parserRegex = require("./regex"); +var runner = require("./runner"); +var stack = require("./stack"); +var util = require("../util"); function parseRuleSync(templates, filepath, raw, type, args, indent, ln, opts) { var operator = operators[type]; @@ -15,8 +15,8 @@ function parseRuleSync(templates, filepath, raw, type, args, indent, ln, opts) { var _args; // Operator - if(operator.parseSync) { - _args = operator.parseSync.apply({ + if(operator.parse) { + _args = operator.parse.apply({ templates: templates, recurse: function(filepath) { var data = parseSync(filepath, templates, opts); @@ -47,10 +47,6 @@ function parseRuleSync(templates, filepath, raw, type, args, indent, ln, opts) { }; } - - - - function parseSync(filepath, templates, opts) { try { var sqlRaw = util.chomp( @@ -62,16 +58,10 @@ function parseSync(filepath, templates, opts) { var indent = ""; var foundChar = false; - var re = /(?:([^}{\n]+)|(\{(.)([^}]*)\})|([\n]))/gm; var ln = 0; - var posMap = { - sql: 1, - cmd: 2, - cmdType: 3, - cmdArgs: 4, - nl: 5 - }; + var re = parserRegex.re(); + var posMap = parserRegex.posMap; var idx = -1; var m; diff --git a/test/index.js b/test/index.js index 6224072..e4c7fdc 100644 --- a/test/index.js +++ b/test/index.js @@ -5,6 +5,7 @@ require("./operators/param-default"); require("./operators/raw"); require("./operators/require"); require("./operators/no-operator"); +require("./parser/regex"); // Full end-to-end test require("./functional"); diff --git a/test/parser/regex.js b/test/parser/regex.js new file mode 100644 index 0000000..f4dcd8e --- /dev/null +++ b/test/parser/regex.js @@ -0,0 +1,38 @@ +var regex = require("../../lib/parser/regex"); +var assert = require("assert"); + +describe("parser/regex", function () { + + it("should parse {>foo}", function () { + var test1 = "{>foo}"; + var results = regex.re().exec(test1); + assert.equal(results[1], test1); + assert.equal(results[2], ">"); + assert.equal(results[3], "foo"); + }); + + it("should parse {=foo}", function () { + var test2 = "{=foo}"; + var results = regex.re().exec(test2); + assert.equal(results[1], test2); + assert.equal(results[2], "="); + assert.equal(results[3], "foo"); + }); + + it("should parse {!foo}", function () { + var test4 = "{!foo}" + var results = regex.re().exec(test4); + assert.equal(results[1], test4); + assert.equal(results[2], "!"); + assert.equal(results[3], "foo"); + }); + + it("should parse {{=foo}}", function () { + var test3 = "{=foo}" + var results = regex.re().exec(test3); + assert.equal(results[1], test3); + assert.equal(results[2], "="); + assert.equal(results[3], "foo"); + }); + +}); From 25801f99613a588bbd46f4a2751328f1769f246c Mon Sep 17 00:00:00 2001 From: Oliver Brooks Date: Wed, 6 Jul 2016 17:32:16 +0100 Subject: [PATCH 4/4] handle number substitutions --- lib/parser/index.js | 2 +- lib/parser/sync.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/parser/index.js b/lib/parser/index.js index 4144cdc..f3e1f78 100644 --- a/lib/parser/index.js +++ b/lib/parser/index.js @@ -48,7 +48,7 @@ function parseRule(templates, filepath, raw, type, args, indent, ln, opts) { var out = operator.run.apply({templates: templates, data: ctx, parseArgs: _args}, operatorArgs); // TODO: Nasty - out.sql = out.sql.split("\n").map(function(line, idx) { + out.sql = String(out.sql).split("\n").map(function(line, idx) { if(idx > 0) { return indent + line; } else { diff --git a/lib/parser/sync.js b/lib/parser/sync.js index 458225a..5627ec6 100644 --- a/lib/parser/sync.js +++ b/lib/parser/sync.js @@ -35,7 +35,7 @@ function parseRuleSync(templates, filepath, raw, type, args, indent, ln, opts) { var out = operator.run.apply({templates: templates, data: ctx, parseArgs: _args}, operatorArgs); // TODO: Nasty - out.sql = out.sql.split("\n").map(function(line, idx) { + out.sql = String(out.sql).split("\n").map(function(line, idx) { if(idx > 0) { return indent + line; } else {