From a492d8e23d511cd5b5a1ffb2e559d2e0bc63ac93 Mon Sep 17 00:00:00 2001 From: harttle Date: Mon, 7 Dec 2020 23:10:49 +0800 Subject: [PATCH] fix: raw block not ignoring {% characters, fixes #263 --- bin/character-gen.js | 6 +- src/builtin/tags/assign.ts | 2 +- src/builtin/tags/capture.ts | 2 +- src/builtin/tags/decrement.ts | 2 +- src/builtin/tags/for.ts | 4 +- src/builtin/tags/include.ts | 2 +- src/builtin/tags/increment.ts | 2 +- src/builtin/tags/render.ts | 6 +- src/builtin/tags/tablerow.ts | 4 +- src/parser/match-operator.ts | 4 +- src/parser/tokenizer.ts | 94 +++++++++++++------ src/parser/whitespace-ctrl.ts | 1 - src/template/tag/tag.ts | 3 +- src/tokens/hash-token.ts | 4 +- .../{word-token.ts => identifier-token.ts} | 3 +- src/tokens/number-token.ts | 6 +- src/tokens/property-access-token.ts | 10 +- src/tokens/tag-token.ts | 12 +-- src/util/character.ts | 2 +- src/util/type-guards.ts | 4 +- test/e2e/issues.ts | 6 ++ test/integration/builtin/tags/raw.ts | 10 +- test/unit/parser/tokenizer.ts | 33 ++++++- test/unit/template/filter/filter.ts | 12 +-- 24 files changed, 153 insertions(+), 81 deletions(-) rename src/tokens/{word-token.ts => identifier-token.ts} (85%) diff --git a/bin/character-gen.js b/bin/character-gen.js index be4975077c..1615791ccd 100644 --- a/bin/character-gen.js +++ b/bin/character-gen.js @@ -4,7 +4,7 @@ const isQuote = c => c === '"' || c === "'" const isOperator = c => '!=<>'.includes(c) const isNumber = c => c >= '0' && c <= '9' const isCharacter = c => (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') -const isVariable = c => '_-?'.includes(c) || isCharacter(c) || isNumber(c) +const isIdentifier = c => '_-?'.includes(c) || isCharacter(c) || isNumber(c) const isBlank = c => c === '\n' || c === '\t' || c === ' ' || c === '\r' || c === '\v' || c === '\f' const isInlineBlank = c => c === '\t' || c === ' ' || c === '\r' const isSign = c => c === '-' || c === '+' @@ -15,7 +15,7 @@ const types = [] for (let i = 0; i < 128; i++) { const c = String.fromCharCode(i) let n = 0 - if (isVariable(c)) n |= 1 + if (isIdentifier(c)) n |= 1 if (isOperator(c)) n |= 2 if (isBlank(c)) n |= 4 if (isQuote(c)) n |= 8 @@ -31,7 +31,7 @@ console.log(` // This file is generated by bin/character-gen.js // bitmask character types to boost performance export const TYPES = [${types.join(', ')}] -export const VARIABLE = 1 +export const IDENTIFIER = 1 export const OPERATOR = 2 export const BLANK = 4 export const QUOTE = 8 diff --git a/src/builtin/tags/assign.ts b/src/builtin/tags/assign.ts index 482d9bd0cb..d952215c62 100644 --- a/src/builtin/tags/assign.ts +++ b/src/builtin/tags/assign.ts @@ -3,7 +3,7 @@ import { Tokenizer, assert, TagImplOptions, TagToken, Context } from '../../type export default { parse: function (token: TagToken) { const tokenizer = new Tokenizer(token.args) - this.key = tokenizer.readWord().content + this.key = tokenizer.readIdentifier().content tokenizer.skipBlank() assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`) tokenizer.advance() diff --git a/src/builtin/tags/capture.ts b/src/builtin/tags/capture.ts index 139d9b613e..3b0aa1fd39 100644 --- a/src/builtin/tags/capture.ts +++ b/src/builtin/tags/capture.ts @@ -25,7 +25,7 @@ export default { } as TagImplOptions function readVariableName (tokenizer: Tokenizer) { - const word = tokenizer.readWord().content + const word = tokenizer.readIdentifier().content if (word) return word const quoted = tokenizer.readQuoted() if (quoted) return evalQuotedToken(quoted) diff --git a/src/builtin/tags/decrement.ts b/src/builtin/tags/decrement.ts index e7e94be175..0db11fe7f2 100644 --- a/src/builtin/tags/decrement.ts +++ b/src/builtin/tags/decrement.ts @@ -4,7 +4,7 @@ import { isNumber, stringify } from '../../util/underscore' export default { parse: function (token: TagToken) { const tokenizer = new Tokenizer(token.args) - this.variable = tokenizer.readWord().content + this.variable = tokenizer.readIdentifier().content }, render: function (context: Context, emitter: Emitter) { const scope = context.environments diff --git a/src/builtin/tags/for.ts b/src/builtin/tags/for.ts index f68dd03ca2..94f23fd830 100644 --- a/src/builtin/tags/for.ts +++ b/src/builtin/tags/for.ts @@ -8,8 +8,8 @@ export default { parse: function (token: TagToken, remainTokens: TopLevelToken[]) { const toknenizer = new Tokenizer(token.args) - const variable = toknenizer.readWord() - const inStr = toknenizer.readWord() + const variable = toknenizer.readIdentifier() + const inStr = toknenizer.readIdentifier() const collection = toknenizer.readValue() assert( variable.size() && inStr.content === 'in' && collection, diff --git a/src/builtin/tags/include.ts b/src/builtin/tags/include.ts index aa11511a5e..504f4eb83c 100644 --- a/src/builtin/tags/include.ts +++ b/src/builtin/tags/include.ts @@ -11,7 +11,7 @@ export default { assert(this.file, () => `illegal argument "${token.args}"`) const begin = tokenizer.p - const withStr = tokenizer.readWord() + const withStr = tokenizer.readIdentifier() if (withStr.content === 'with') { tokenizer.skipBlank() if (tokenizer.peek() !== ':') { diff --git a/src/builtin/tags/increment.ts b/src/builtin/tags/increment.ts index 39d2655eb6..8107acae62 100644 --- a/src/builtin/tags/increment.ts +++ b/src/builtin/tags/increment.ts @@ -4,7 +4,7 @@ import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../typ export default { parse: function (token: TagToken) { const tokenizer = new Tokenizer(token.args) - this.variable = tokenizer.readWord().content + this.variable = tokenizer.readIdentifier().content }, render: function (context: Context, emitter: Emitter) { const scope = context.environments diff --git a/src/builtin/tags/render.ts b/src/builtin/tags/render.ts index f6fe37224b..48d0c7f50e 100644 --- a/src/builtin/tags/render.ts +++ b/src/builtin/tags/render.ts @@ -15,16 +15,16 @@ export default { while (!tokenizer.end()) { tokenizer.skipBlank() const begin = tokenizer.p - const keyword = tokenizer.readWord() + const keyword = tokenizer.readIdentifier() if (keyword.content === 'with' || keyword.content === 'for') { tokenizer.skipBlank() if (tokenizer.peek() !== ':') { const value = tokenizer.readValue() if (value) { const beforeAs = tokenizer.p - const asStr = tokenizer.readWord() + const asStr = tokenizer.readIdentifier() let alias - if (asStr.content === 'as') alias = tokenizer.readWord() + if (asStr.content === 'as') alias = tokenizer.readIdentifier() else tokenizer.p = beforeAs this[keyword.content] = { value, alias: alias && alias.content } diff --git a/src/builtin/tags/tablerow.ts b/src/builtin/tags/tablerow.ts index 41ebd6125b..8dc0dbd42b 100644 --- a/src/builtin/tags/tablerow.ts +++ b/src/builtin/tags/tablerow.ts @@ -7,10 +7,10 @@ export default { parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { const tokenizer = new Tokenizer(tagToken.args) - this.variable = tokenizer.readWord() + this.variable = tokenizer.readIdentifier() tokenizer.skipBlank() - const tmp = tokenizer.readWord() + const tmp = tokenizer.readIdentifier() assert(tmp && tmp.content === 'in', () => `illegal tag: ${tagToken.getText()}`) this.collection = tokenizer.readValue() diff --git a/src/parser/match-operator.ts b/src/parser/match-operator.ts index e1883b66df..206c7a5f12 100644 --- a/src/parser/match-operator.ts +++ b/src/parser/match-operator.ts @@ -1,4 +1,4 @@ -import { VARIABLE } from '../util/character' +import { IDENTIFIER } from '../util/character' const trie = { a: { n: { d: { end: true, needBoundary: true } } }, @@ -19,6 +19,6 @@ export function matchOperator (str: string, begin: number, end = str.length) { if (node['end']) info = node } if (!info) return -1 - if (info['needBoundary'] && str.charCodeAt(i) & VARIABLE) return -1 + if (info['needBoundary'] && str.charCodeAt(i) & IDENTIFIER) return -1 return i } diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index 16e253788f..7949f7cd71 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -1,6 +1,6 @@ import { whiteSpaceCtrl } from './whitespace-ctrl' import { NumberToken } from '../tokens/number-token' -import { WordToken } from '../tokens/word-token' +import { IdentifierToken } from '../tokens/identifier-token' import { literalValues } from '../util/literal' import { LiteralToken } from '../tokens/literal-token' import { OperatorToken } from '../tokens/operator-token' @@ -20,12 +20,14 @@ import { ValueToken } from '../tokens/value-token' import { OutputToken } from '../tokens/output-token' import { TokenizationError } from '../util/error' import { NormalizedFullOptions, defaultOptions } from '../liquid-options' -import { TYPES, QUOTE, BLANK, VARIABLE } from '../util/character' +import { TYPES, QUOTE, BLANK, IDENTIFIER } from '../util/character' import { matchOperator } from './match-operator' export class Tokenizer { p = 0 N: number + private rawBeginAt = -1 + constructor ( private input: string, private file: string = '' @@ -70,7 +72,7 @@ export class Tokenizer { assert(this.peek() === '|', () => `unexpected token at ${this.snapshot()}`) this.p++ const begin = this.p - const name = this.readWord() + const name = this.readIdentifier() if (!name.size()) return null const args = [] this.skipBlank() @@ -107,8 +109,9 @@ export class Tokenizer { readTopLevelToken (options: NormalizedFullOptions): TopLevelToken { const { tagDelimiterLeft, outputDelimiterLeft } = options - if (this.matchWord(tagDelimiterLeft)) return this.readTagToken(options) - if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken(options) + if (this.rawBeginAt > -1) return this.readEndrawOrRawContent(options) + if (this.match(tagDelimiterLeft)) return this.readTagToken(options) + if (this.match(outputDelimiterLeft)) return this.readOutputToken(options) return this.readHTMLToken(options) } @@ -116,8 +119,8 @@ export class Tokenizer { const begin = this.p while (this.p < this.N) { const { tagDelimiterLeft, outputDelimiterLeft } = options - if (this.matchWord(tagDelimiterLeft)) break - if (this.matchWord(outputDelimiterLeft)) break + if (this.match(tagDelimiterLeft)) break + if (this.match(outputDelimiterLeft)) break ++this.p } return new HTMLToken(this.input, begin, this.p, this.file) @@ -128,9 +131,11 @@ export class Tokenizer { const { tagDelimiterRight } = options const begin = this.p if (this.readTo(tagDelimiterRight) === -1) { - this.mkError(`tag ${this.snapshot(begin)} not closed`, begin) + throw this.mkError(`tag ${this.snapshot(begin)} not closed`, begin) } - return new TagToken(input, begin, this.p, options, file) + const token = new TagToken(input, begin, this.p, options, file) + if (token.name === 'raw') this.rawBeginAt = begin + return token } readOutputToken (options: NormalizedFullOptions): OutputToken { @@ -138,24 +143,59 @@ export class Tokenizer { const { outputDelimiterRight } = options const begin = this.p if (this.readTo(outputDelimiterRight) === -1) { - this.mkError(`output ${this.snapshot(begin)} not closed`, begin) + throw this.mkError(`output ${this.snapshot(begin)} not closed`, begin) } return new OutputToken(input, begin, this.p, options, file) } + readEndrawOrRawContent (options: NormalizedFullOptions): HTMLToken | TagToken { + const { tagDelimiterLeft, tagDelimiterRight } = options + const begin = this.p + let leftPos = this.readTo(tagDelimiterLeft) - tagDelimiterLeft.length + while (this.p < this.N) { + if (this.readIdentifier().getText() !== 'endraw') { + leftPos = this.readTo(tagDelimiterLeft) - tagDelimiterLeft.length + continue + } + while (this.p <= this.N) { + if (this.rmatch(tagDelimiterRight)) { + const end = this.p + if (begin === leftPos) { + this.rawBeginAt = -1 + return new TagToken(this.input, begin, end, options, this.file) + } else { + this.p = leftPos + return new HTMLToken(this.input, begin, leftPos, this.file) + } + } + if (this.rmatch(tagDelimiterLeft)) break + this.p++ + } + } + throw this.mkError(`raw ${this.snapshot(this.rawBeginAt)} not closed`, begin) + } + mkError (msg: string, begin: number) { - throw new TokenizationError(msg, new WordToken(this.input, begin, this.N, this.file)) + return new TokenizationError(msg, new IdentifierToken(this.input, begin, this.N, this.file)) } snapshot (begin: number = this.p) { return JSON.stringify(ellipsis(this.input.slice(begin), 16)) } - readWord (): WordToken { // rename to identifier + /** + * @deprecated + */ + readWord () { + console.warn('Tokenizer#readWord() will be removed, use #readIdentifier instead') + return this.readIdentifier() + } + + readIdentifier (): IdentifierToken { this.skipBlank() const begin = this.p - while (this.peekType() & VARIABLE) ++this.p - return new WordToken(this.input, begin, this.p, this.file) + while (this.peekType() & IDENTIFIER) ++this.p + return new IdentifierToken(this.input, begin, this.p, this.file) } readHashes () { @@ -171,7 +211,7 @@ export class Tokenizer { this.skipBlank() if (this.peek() === ',') ++this.p const begin = this.p - const name = this.readWord() + const name = this.readIdentifier() if (!name.size()) return let value @@ -198,7 +238,7 @@ export class Tokenizer { readTo (end: string): number { while (this.p < this.N) { ++this.p - if (this.reverseMatchWord(end)) return this.p + if (this.rmatch(end)) return this.p } return -1 } @@ -216,21 +256,21 @@ export class Tokenizer { return new PropertyAccessToken(prop, [], this.p) } - const variable = this.readWord() + const variable = this.readIdentifier() if (!variable.size()) return let isNumber = variable.isNumber(true) - const props: (QuotedToken | WordToken)[] = [] + const props: (QuotedToken | IdentifierToken)[] = [] while (true) { if (this.peek() === '[') { isNumber = false this.p++ - const prop = this.readValue() || new WordToken(this.input, this.p, this.p, this.file) + const prop = this.readValue() || new IdentifierToken(this.input, this.p, this.p, this.file) this.readTo(']') props.push(prop) } else if (this.peek() === '.' && this.peek(1) !== '.') { // skip range syntax this.p++ - const prop = this.readWord() + const prop = this.readIdentifier() if (!prop.size()) break if (!prop.isNumber()) isNumber = false props.push(prop) @@ -239,7 +279,7 @@ export class Tokenizer { if (!props.length && literalValues.hasOwnProperty(variable.content)) { return new LiteralToken(this.input, variable.begin, variable.end, this.file) } - if (isNumber) return new NumberToken(variable, props[0] as WordToken) + if (isNumber) return new NumberToken(variable, props[0] as IdentifierToken) return new PropertyAccessToken(variable, props, this.p) } @@ -276,22 +316,22 @@ export class Tokenizer { return new QuotedToken(this.input, begin, this.p, this.file) } - readFileName (): WordToken { + readFileName (): IdentifierToken { const begin = this.p while (!(this.peekType() & BLANK) && this.peek() !== ',' && this.p < this.N) this.p++ - return new WordToken(this.input, begin, this.p, this.file) + return new IdentifierToken(this.input, begin, this.p, this.file) } - matchWord (word: string) { + match (word: string) { for (let i = 0; i < word.length; i++) { if (word[i] !== this.input[this.p + i]) return false } return true } - reverseMatchWord (word: string) { - for (let i = 0; i < word.length; i++) { - if (word[word.length - 1 - i] !== this.input[this.p - 1 - i]) return false + rmatch (pattern: string) { + for (let i = 0; i < pattern.length; i++) { + if (pattern[pattern.length - 1 - i] !== this.input[this.p - 1 - i]) return false } return true } diff --git a/src/parser/whitespace-ctrl.ts b/src/parser/whitespace-ctrl.ts index f54fbcf043..49a621febd 100644 --- a/src/parser/whitespace-ctrl.ts +++ b/src/parser/whitespace-ctrl.ts @@ -4,7 +4,6 @@ import { NormalizedFullOptions } from '../liquid-options' import { TYPES, INLINE_BLANK, BLANK } from '../util/character' export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions) { - options = { greedy: true, ...options } let inRaw = false for (let i = 0; i < tokens.length; i++) { diff --git a/src/template/tag/tag.ts b/src/template/tag/tag.ts index 1a38923ea1..bbe0884bd8 100644 --- a/src/template/tag/tag.ts +++ b/src/template/tag/tag.ts @@ -1,13 +1,12 @@ import { isFunction } from '../../util/underscore' import { Liquid } from '../../liquid' import { TemplateImpl } from '../../template/template-impl' -import { Emitter, Hash, Context, TagImplOptions, TagToken, Template, TopLevelToken } from '../../types' +import { Emitter, Hash, Context, TagToken, Template, TopLevelToken } from '../../types' import { TagImpl } from './tag-impl' export class Tag extends TemplateImpl implements Template { public name: string private impl: TagImpl - private static impls: { [key: string]: TagImplOptions } = {} public constructor (token: TagToken, tokens: TopLevelToken[], liquid: Liquid) { super(token) diff --git a/src/tokens/hash-token.ts b/src/tokens/hash-token.ts index 288807c97a..6f691939cc 100644 --- a/src/tokens/hash-token.ts +++ b/src/tokens/hash-token.ts @@ -1,6 +1,6 @@ import { Token } from './token' import { ValueToken } from './value-token' -import { WordToken } from './word-token' +import { IdentifierToken } from './identifier-token' import { TokenKind } from '../parser/token-kind' export class HashToken extends Token { @@ -8,7 +8,7 @@ export class HashToken extends Token { public input: string, public begin: number, public end: number, - public name: WordToken, + public name: IdentifierToken, public value?: ValueToken, public file?: string ) { diff --git a/src/tokens/word-token.ts b/src/tokens/identifier-token.ts similarity index 85% rename from src/tokens/word-token.ts rename to src/tokens/identifier-token.ts index f19de6fbdd..056a85de47 100644 --- a/src/tokens/word-token.ts +++ b/src/tokens/identifier-token.ts @@ -2,8 +2,7 @@ import { Token } from './token' import { NUMBER, TYPES, SIGN } from '../util/character' import { TokenKind } from '../parser/token-kind' -// a word can be an identifier, a number, a keyword or a single-word-literal -export class WordToken extends Token { +export class IdentifierToken extends Token { public content: string constructor ( public input: string, diff --git a/src/tokens/number-token.ts b/src/tokens/number-token.ts index b029d0a0dd..e8ea5da238 100644 --- a/src/tokens/number-token.ts +++ b/src/tokens/number-token.ts @@ -1,11 +1,11 @@ import { Token } from './token' -import { WordToken } from './word-token' +import { IdentifierToken } from './identifier-token' import { TokenKind } from '../parser/token-kind' export class NumberToken extends Token { constructor ( - public whole: WordToken, - public decimal?: WordToken + public whole: IdentifierToken, + public decimal?: IdentifierToken ) { super(TokenKind.Number, whole.input, whole.begin, decimal ? decimal.end : whole.end, whole.file) } diff --git a/src/tokens/property-access-token.ts b/src/tokens/property-access-token.ts index 8da3c9236f..e0ed70b851 100644 --- a/src/tokens/property-access-token.ts +++ b/src/tokens/property-access-token.ts @@ -1,20 +1,20 @@ import { Token } from './token' -import { WordToken } from './word-token' +import { IdentifierToken } from './identifier-token' import { QuotedToken } from './quoted-token' import { TokenKind } from '../parser/token-kind' import { parseStringLiteral } from '../parser/parse-string-literal' export class PropertyAccessToken extends Token { constructor ( - public variable: WordToken | QuotedToken, - public props: (WordToken | QuotedToken | PropertyAccessToken)[], + public variable: IdentifierToken | QuotedToken, + public props: (IdentifierToken | QuotedToken | PropertyAccessToken)[], end: number ) { super(TokenKind.PropertyAccess, variable.input, variable.begin, end, variable.file) } - getVariableAsText() { - if (this.variable instanceof WordToken) { + getVariableAsText () { + if (this.variable instanceof IdentifierToken) { return this.variable.getText() } else { return parseStringLiteral(this.variable.getText()) diff --git a/src/tokens/tag-token.ts b/src/tokens/tag-token.ts index b7b3d68e70..25503c8fae 100644 --- a/src/tokens/tag-token.ts +++ b/src/tokens/tag-token.ts @@ -1,8 +1,8 @@ import { DelimitedToken } from './delimited-token' -import { BLANK, TYPES, VARIABLE } from '../util/character' import { TokenizationError } from '../util/error' import { NormalizedFullOptions } from '../liquid-options' import { TokenKind } from '../parser/token-kind' +import { Tokenizer } from '../parser/tokenizer' export class TagToken extends DelimitedToken { public name: string @@ -18,13 +18,11 @@ export class TagToken extends DelimitedToken { const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length) super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file) - let nameEnd = 0 - while (TYPES[this.content.charCodeAt(nameEnd)] & VARIABLE) nameEnd++ - this.name = this.content.slice(0, nameEnd) + const tokenizer = new Tokenizer(this.content) + this.name = tokenizer.readIdentifier().getText() if (!this.name) throw new TokenizationError(`illegal tag syntax`, this) - let argsBegin = nameEnd - while (TYPES[this.content.charCodeAt(argsBegin)] & BLANK) argsBegin++ - this.args = this.content.slice(argsBegin) + tokenizer.skipBlank() + this.args = tokenizer.remaining() } } diff --git a/src/util/character.ts b/src/util/character.ts index be1fa85d80..cbec80b360 100644 --- a/src/util/character.ts +++ b/src/util/character.ts @@ -3,7 +3,7 @@ // This file is generated by bin/character-gen.js // bitmask character types to boost performance export const TYPES = [0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 4, 4, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 8, 0, 0, 0, 0, 8, 0, 0, 0, 64, 0, 65, 0, 0, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 0, 0, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0] -export const VARIABLE = 1 +export const IDENTIFIER = 1 export const OPERATOR = 2 export const BLANK = 4 export const QUOTE = 8 diff --git a/src/util/type-guards.ts b/src/util/type-guards.ts index fb86561139..f2e02bf171 100644 --- a/src/util/type-guards.ts +++ b/src/util/type-guards.ts @@ -1,6 +1,6 @@ import { OperatorToken } from '../tokens/operator-token' import { DelimitedToken } from '../tokens/delimited-token' -import { WordToken } from '../tokens/word-token' +import { IdentifierToken } from '../tokens/identifier-token' import { TagToken } from '../tokens/tag-token' import { HTMLToken } from '../tokens/html-token' import { OutputToken } from '../tokens/output-token' @@ -47,7 +47,7 @@ export function isPropertyAccessToken (val: any): val is PropertyAccessToken { return getKind(val) === TokenKind.PropertyAccess } -export function isWordToken (val: any): val is WordToken { +export function isWordToken (val: any): val is IdentifierToken { return getKind(val) === TokenKind.Word } diff --git a/test/e2e/issues.ts b/test/e2e/issues.ts index a0aac01697..11be1b8e9b 100644 --- a/test/e2e/issues.ts +++ b/test/e2e/issues.ts @@ -36,4 +36,10 @@ describe('Issues', function () { // should stringify the regexp rather than execute it expect(html).to.equal(INPUT) }) + it('#263 raw/endraw block not ignoring {% characters', () => { + const template = `{% raw %}This is a code snippet showing how {% breaks the raw block.{% endraw %}` + const engine = new Liquid() + const html = engine.parseAndRenderSync(template) + expect(html).to.equal('This is a code snippet showing how {% breaks the raw block.') + }) }) diff --git a/test/integration/builtin/tags/raw.ts b/test/integration/builtin/tags/raw.ts index 3d62b837c0..a9f0d64a37 100644 --- a/test/integration/builtin/tags/raw.ts +++ b/test/integration/builtin/tags/raw.ts @@ -1,11 +1,15 @@ import { Liquid } from '../../../../src/liquid' -import { expect } from 'chai' +import * as chai from 'chai' +import * as chaiAsPromised from 'chai-as-promised' + +const expect = chai.expect +chai.use(chaiAsPromised) describe('tags/raw', function () { const liquid = new Liquid() it('should throw when not closed', async function () { - const p = liquid.parseAndRender('{% raw%}') - return expect(p).be.rejectedWith(/{% raw%} not closed/) + const p = liquid.parseAndRender('{% raw %}') + return expect(p).be.rejectedWith(/{% raw %} not closed/) }) it('should output filters as it is', async function () { const src = '{% raw %}{{ 5 | plus: 6 }}{% endraw %} is equal to 11.' diff --git a/test/unit/parser/tokenizer.ts b/test/unit/parser/tokenizer.ts index 743edf7c97..9453fafd35 100644 --- a/test/unit/parser/tokenizer.ts +++ b/test/unit/parser/tokenizer.ts @@ -1,5 +1,5 @@ import { expect } from 'chai' -import { WordToken } from '../../../src/tokens/word-token' +import { IdentifierToken } from '../../../src/tokens/identifier-token' import { NumberToken } from '../../../src/tokens/number-token' import { PropertyAccessToken } from '../../../src/tokens/property-access-token' import { RangeToken } from '../../../src/tokens/range-token' @@ -19,6 +19,10 @@ describe('Tokenize', function () { expect(new Tokenizer('a[ b][ "c d" ]').readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]') expect(new Tokenizer('a.b[c[d.e]]').readValueOrThrow().getText()).to.equal('a.b[c[d.e]]') }) + it('should read identifier', () => { + expect(new Tokenizer('foo bar').readIdentifier()).to.haveOwnProperty('content', 'foo') + expect(new Tokenizer('foo bar').readWord()).to.haveOwnProperty('content', 'foo') + }) it('should read number value', () => { const token: NumberToken = new Tokenizer('2.33.2').readValueOrThrow() as any expect(token).to.be.instanceOf(NumberToken) @@ -100,6 +104,29 @@ describe('Tokenize', function () { expect(tag.name).to.equal('for') expect(tag.args).to.equal('p in a[1]') }) + it('should allow unclosed tag inside {% raw %}', function () { + const html = '{%raw%} {%if%} {%else {%endraw%}' + const tokenizer = new Tokenizer(html) + const tokens = tokenizer.readTopLevelTokens() + + expect(tokens.length).to.equal(3) + expect(tokens[0]).to.haveOwnProperty('name', 'raw') + expect((tokens[1] as any).getContent()).to.equal(' {%if%} {%else ') + }) + it('should allow unclosed endraw tag inside {% raw %}', function () { + const html = '{%raw%} {%endraw {%raw%} {%endraw%}' + const tokenizer = new Tokenizer(html) + const tokens = tokenizer.readTopLevelTokens() + + expect(tokens.length).to.equal(3) + expect(tokens[0]).to.haveOwnProperty('name', 'raw') + expect((tokens[1] as any).getContent()).to.equal(' {%endraw {%raw%} ') + }) + it('should throw when {% raw %} not closed', function () { + const html = '{%raw%} {%endraw {%raw%}' + const tokenizer = new Tokenizer(html) + expect(() => tokenizer.readTopLevelTokens()).to.throw('raw "{%raw%} {%end..." not closed, line:1, col:8') + }) it('should read output token', function () { const html = '

{{foo | date: "%Y-%m-%d"}}

' const tokenizer = new Tokenizer(html) @@ -259,7 +286,7 @@ describe('Tokenize', function () { expect(token!.args[0]).to.be.instanceOf(PropertyAccessToken) expect(pa.variable.content).to.equal('obj') expect(pa.props).to.have.lengthOf(1) - expect(pa.props[0]).to.be.instanceOf(WordToken) + expect(pa.props[0]).to.be.instanceOf(IdentifierToken) expect(pa.props[0].getText()).to.equal('foo') }) it('should read a filter with obj["foo"] argument', function () { @@ -329,7 +356,7 @@ describe('Tokenize', function () { expect(pa.props).to.have.lengthOf(2) const [p1, p2] = pa.props - expect(p1).to.be.instanceOf(WordToken) + expect(p1).to.be.instanceOf(IdentifierToken) expect(p1.getText()).to.equal('') expect(p2).to.be.instanceOf(PropertyAccessToken) expect(p2.getText()).to.equal('b') diff --git a/test/unit/template/filter/filter.ts b/test/unit/template/filter/filter.ts index 93061a3109..a0f774f06b 100644 --- a/test/unit/template/filter/filter.ts +++ b/test/unit/template/filter/filter.ts @@ -5,7 +5,7 @@ import { Context } from '../../../../src/context/context' import { toThenable } from '../../../../src/util/async' import { NumberToken } from '../../../../src/tokens/number-token' import { QuotedToken } from '../../../../src/tokens/quoted-token' -import { WordToken } from '../../../../src/tokens/word-token' +import { IdentifierToken } from '../../../../src/tokens/identifier-token' import { FilterMap } from '../../../../src/template/filter/filter-map' chai.use(sinonChai) @@ -30,14 +30,14 @@ describe('filter', function () { it('should call filter impl with correct arguments', async function () { const spy = sinon.spy() filters.set('foo', spy) - const thirty = new NumberToken(new WordToken('30', 0, 2), undefined) + const thirty = new NumberToken(new IdentifierToken('30', 0, 2), undefined) await toThenable(filters.create('foo', [thirty]).render('foo', ctx)) expect(spy).to.have.been.calledWith('foo', 30) }) it('should call filter impl with correct this arg', async function () { const spy = sinon.spy() filters.set('foo', spy) - const thirty = new NumberToken(new WordToken('33', 0, 2), undefined) + const thirty = new NumberToken(new IdentifierToken('33', 0, 2), undefined) await toThenable(filters.create('foo', [thirty]).render('foo', ctx)) expect(spy).to.have.been.calledOn(sinon.match.has('context', ctx)) }) @@ -48,13 +48,13 @@ describe('filter', function () { it('should render filters with argument', async function () { filters.set('add', (a, b) => a + b) - const two = new NumberToken(new WordToken('2', 0, 1), undefined) + const two = new NumberToken(new IdentifierToken('2', 0, 1), undefined) expect(await toThenable(filters.create('add', [two]).render(3, ctx))).to.equal(5) }) it('should render filters with multiple arguments', async function () { filters.set('add', (a, b, c) => a + b + c) - const two = new NumberToken(new WordToken('2', 0, 1), undefined) + const two = new NumberToken(new IdentifierToken('2', 0, 1), undefined) const c = new QuotedToken('"c"', 0, 3) expect(await toThenable(filters.create('add', [two, c]).render(3, ctx))).to.equal('5c') }) @@ -73,7 +73,7 @@ describe('filter', function () { it('should support key value pairs', async function () { filters.set('add', (a, b) => b[0] + ':' + (a + b[1])) - const two = new NumberToken(new WordToken('2', 0, 1), undefined) + const two = new NumberToken(new IdentifierToken('2', 0, 1), undefined) expect(await toThenable((filters.create('add', [['num', two]]).render(3, ctx)))).to.equal('num:5') }) })