From 04a47871a3033c38dcca49017fe262232a8d5b61 Mon Sep 17 00:00:00 2001 From: James Bell Date: Wed, 18 Oct 2017 18:20:07 +0200 Subject: [PATCH 01/27] Initial Commit --- src/melody/melody-code-frame/index.js | 49 ++ src/melody/melody-code-frame/lineNumbers.js | 45 ++ src/melody/melody-extension-core/index.js | 166 ++++ src/melody/melody-extension-core/operators.js | 402 ++++++++++ .../parser/autoescape.js | 71 ++ .../melody-extension-core/parser/block.js | 72 ++ src/melody/melody-extension-core/parser/do.js | 28 + .../melody-extension-core/parser/embed.js | 58 ++ .../melody-extension-core/parser/extends.js | 31 + .../melody-extension-core/parser/filter.js | 47 ++ .../melody-extension-core/parser/flush.js | 29 + .../melody-extension-core/parser/for.js | 89 +++ .../melody-extension-core/parser/from.js | 63 ++ src/melody/melody-extension-core/parser/if.js | 69 ++ .../melody-extension-core/parser/import.js | 44 ++ .../melody-extension-core/parser/include.js | 44 ++ .../melody-extension-core/parser/macro.js | 81 ++ .../melody-extension-core/parser/mount.js | 60 ++ .../melody-extension-core/parser/set.js | 86 +++ .../melody-extension-core/parser/spaceless.js | 39 + .../melody-extension-core/parser/use.js | 58 ++ src/melody/melody-extension-core/types.js | 311 ++++++++ .../melody-extension-core/visitors/filters.js | 154 ++++ .../melody-extension-core/visitors/for.js | 392 ++++++++++ .../visitors/functions.js | 100 +++ .../melody-extension-core/visitors/tests.js | 127 ++++ src/melody/melody-parser/Associativity.js | 17 + src/melody/melody-parser/CharStream.js | 82 ++ src/melody/melody-parser/Lexer.js | 602 +++++++++++++++ src/melody/melody-parser/Parser.js | 707 ++++++++++++++++++ src/melody/melody-parser/TokenStream.js | 177 +++++ src/melody/melody-parser/TokenTypes.js | 73 ++ src/melody/melody-parser/elementInfo.js | 43 ++ src/melody/melody-parser/index.js | 52 ++ src/melody/melody-parser/util.js | 56 ++ src/melody/melody-traverse/Binding.js | 61 ++ src/melody/melody-traverse/Path.js | 403 ++++++++++ src/melody/melody-traverse/Scope.js | 183 +++++ .../melody-traverse/TraversalContext.js | 151 ++++ src/melody/melody-traverse/index.js | 21 + src/melody/melody-traverse/traverse.js | 47 ++ src/melody/melody-traverse/visitors.js | 109 +++ src/melody/melody-types/index.js | 358 +++++++++ 43 files changed, 5857 insertions(+) create mode 100644 src/melody/melody-code-frame/index.js create mode 100644 src/melody/melody-code-frame/lineNumbers.js create mode 100644 src/melody/melody-extension-core/index.js create mode 100644 src/melody/melody-extension-core/operators.js create mode 100644 src/melody/melody-extension-core/parser/autoescape.js create mode 100644 src/melody/melody-extension-core/parser/block.js create mode 100644 src/melody/melody-extension-core/parser/do.js create mode 100644 src/melody/melody-extension-core/parser/embed.js create mode 100644 src/melody/melody-extension-core/parser/extends.js create mode 100644 src/melody/melody-extension-core/parser/filter.js create mode 100644 src/melody/melody-extension-core/parser/flush.js create mode 100644 src/melody/melody-extension-core/parser/for.js create mode 100644 src/melody/melody-extension-core/parser/from.js create mode 100644 src/melody/melody-extension-core/parser/if.js create mode 100644 src/melody/melody-extension-core/parser/import.js create mode 100644 src/melody/melody-extension-core/parser/include.js create mode 100644 src/melody/melody-extension-core/parser/macro.js create mode 100644 src/melody/melody-extension-core/parser/mount.js create mode 100644 src/melody/melody-extension-core/parser/set.js create mode 100644 src/melody/melody-extension-core/parser/spaceless.js create mode 100644 src/melody/melody-extension-core/parser/use.js create mode 100644 src/melody/melody-extension-core/types.js create mode 100644 src/melody/melody-extension-core/visitors/filters.js create mode 100644 src/melody/melody-extension-core/visitors/for.js create mode 100644 src/melody/melody-extension-core/visitors/functions.js create mode 100644 src/melody/melody-extension-core/visitors/tests.js create mode 100644 src/melody/melody-parser/Associativity.js create mode 100644 src/melody/melody-parser/CharStream.js create mode 100644 src/melody/melody-parser/Lexer.js create mode 100644 src/melody/melody-parser/Parser.js create mode 100644 src/melody/melody-parser/TokenStream.js create mode 100644 src/melody/melody-parser/TokenTypes.js create mode 100644 src/melody/melody-parser/elementInfo.js create mode 100644 src/melody/melody-parser/index.js create mode 100644 src/melody/melody-parser/util.js create mode 100644 src/melody/melody-traverse/Binding.js create mode 100644 src/melody/melody-traverse/Path.js create mode 100644 src/melody/melody-traverse/Scope.js create mode 100644 src/melody/melody-traverse/TraversalContext.js create mode 100644 src/melody/melody-traverse/index.js create mode 100644 src/melody/melody-traverse/traverse.js create mode 100644 src/melody/melody-traverse/visitors.js create mode 100644 src/melody/melody-types/index.js diff --git a/src/melody/melody-code-frame/index.js b/src/melody/melody-code-frame/index.js new file mode 100644 index 00000000..3e219859 --- /dev/null +++ b/src/melody/melody-code-frame/index.js @@ -0,0 +1,49 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import lineNumbers from './lineNumbers'; +import { repeat } from 'lodash'; + +const NEWLINE = /\r\n|[\n\r\u2028\u2029]/; + +export default function({ rawLines, lineNumber, colNumber, length }) { + const lines = rawLines.split(NEWLINE), + start = Math.max(lineNumber - 3, 0), + end = Math.min(lineNumber + 3, lines.length); + + return lineNumbers(lines.slice(start, end), { + start: start + 1, + before: ' ', + after: ' | ', + transform(params) { + if (params.number !== lineNumber) { + return; + } + + if (colNumber) { + //params.line = params.line.substring(0, colNumber) + chalk.underline(params.line.substring(colNumber, colNumber + length)) + params.line.substring(colNumber + length, params.line.length - 1); + params.line += `\n${params.before}${repeat( + ' ', + params.width, + )}${params.after}${repeat(' ', colNumber)}${repeat( + '^', + length, + )}`; + } + + params.before = params.before.replace(/^./, '>'); + }, + }).join('\n'); +} diff --git a/src/melody/melody-code-frame/lineNumbers.js b/src/melody/melody-code-frame/lineNumbers.js new file mode 100644 index 00000000..8ebc1571 --- /dev/null +++ b/src/melody/melody-code-frame/lineNumbers.js @@ -0,0 +1,45 @@ +// Copyright 2014, 2015 Simon Lydell +// X11 (“MIT”) Licensed. (See LICENSE.) +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { padStart } from 'lodash'; + +const get = options => (key, defaultValue) => + key in options ? options[key] : defaultValue; + +function lineNumbers(lines, options) { + const getOption = get(options); + const transform = getOption('transform', Function.prototype); + const padding = getOption('padding', ' '); + const before = getOption('before', ' '); + const after = getOption('after', ' | '); + const start = getOption('start', 1); + const end = start + lines.length - 1; + const width = String(end).length; + return lines.map(function(line, index) { + const number = start + index; + const params = { before, number, width, after, line }; + transform(params); + return ( + params.before + + padStart(params.number, width, padding) + + params.after + + params.line + ); + }); +} + +export default lineNumbers; diff --git a/src/melody/melody-extension-core/index.js b/src/melody/melody-extension-core/index.js new file mode 100644 index 00000000..51ce6229 --- /dev/null +++ b/src/melody/melody-extension-core/index.js @@ -0,0 +1,166 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { unaryOperators, binaryOperators, tests } from './operators'; +import { AutoescapeParser } from './parser/autoescape'; +import { BlockParser } from './parser/block'; +import { DoParser } from './parser/do'; +import { EmbedParser } from './parser/embed'; +import { ExtendsParser } from './parser/extends'; +import { FilterParser } from './parser/filter'; +import { FlushParser } from './parser/flush'; +import { ForParser } from './parser/for'; +import { FromParser } from './parser/from'; +import { IfParser } from './parser/if'; +import { ImportParser } from './parser/import'; +import { IncludeParser } from './parser/include'; +import { MacroParser } from './parser/macro'; +import { SetParser } from './parser/set'; +import { SpacelessParser } from './parser/spaceless'; +import { UseParser } from './parser/use'; +import { MountParser } from './parser/mount'; + +import forVisitor from './visitors/for'; +import testVisitor from './visitors/tests'; +import filters from './visitors/filters'; +import functions from './visitors/functions'; + +const filterMap = [ + 'attrs', + 'classes', + 'styles', + 'batch', + 'escape', + 'format', + 'merge', + 'nl2br', + 'number_format', + 'raw', + 'replace', + 'reverse', + 'round', + 'striptags', + 'title', + 'url_encode', +].reduce((map, filterName) => { + map[filterName] = 'melody-runtime'; + return map; +}, Object.create(null)); + +Object.assign(filterMap, filters); + +const functionMap = [ + 'attribute', + 'constant', + 'cycle', + 'date', + 'max', + 'min', + 'random', + 'range', + 'source', + 'template_from_string', +].reduce((map, functionName) => { + map[functionName] = 'melody-runtime'; + return map; +}, Object.create(null)); +Object.assign(functionMap, functions); + +export const extension = { + tags: [ + AutoescapeParser, + BlockParser, + DoParser, + EmbedParser, + ExtendsParser, + FilterParser, + FlushParser, + ForParser, + FromParser, + IfParser, + ImportParser, + IncludeParser, + MacroParser, + SetParser, + SpacelessParser, + UseParser, + MountParser, + ], + unaryOperators, + binaryOperators, + tests, + visitors: [forVisitor, testVisitor], + filterMap, + functionMap, +}; + +export { + AutoescapeBlock, + BlockStatement, + BlockCallExpression, + MountStatement, + DoStatement, + EmbedStatement, + ExtendsStatement, + FilterBlockStatement, + FlushStatement, + ForStatement, + ImportDeclaration, + FromStatement, + IfStatement, + IncludeStatement, + MacroDeclarationStatement, + VariableDeclarationStatement, + SetStatement, + SpacelessBlock, + AliasExpression, + UseStatement, + UnaryNotExpression, + UnaryNeqExpression, + UnaryPosExpression, + BinaryOrExpression, + BinaryAndExpression, + BitwiseOrExpression, + BitwiseXorExpression, + BitwiseAndExpression, + BinaryEqualsExpression, + BinaryNotEqualsExpression, + BinaryLessThanExpression, + BinaryGreaterThanExpression, + BinaryLessThanOrEqualExpression, + BinaryGreaterThanOrEqualExpression, + BinaryNotInExpression, + BinaryInExpression, + BinaryMatchesExpression, + BinaryStartsWithExpression, + BinaryEndsWithExpression, + BinaryRangeExpression, + BinaryAddExpression, + BinaryMulExpression, + BinaryDivExpression, + BinaryFloorDivExpression, + BinaryModExpression, + BinaryPowerExpression, + BinaryNullCoalesceExpression, + TestEvenExpression, + TestOddExpression, + TestDefinedExpression, + TestSameAsExpression, + TestNullExpression, + TestDivisibleByExpression, + TestConstantExpression, + TestEmptyExpression, + TestIterableExpression, +} from './types'; diff --git a/src/melody/melody-extension-core/operators.js b/src/melody/melody-extension-core/operators.js new file mode 100644 index 00000000..b4a65ae2 --- /dev/null +++ b/src/melody/melody-extension-core/operators.js @@ -0,0 +1,402 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Node, + BinaryExpression, + BinaryConcatExpression, + UnaryExpression, + type, + alias, + visitor, +} from 'melody-types'; +import { + Types, + setStartFromToken, + setEndFromToken, + copyStart, + copyEnd, + copyLoc, + LEFT, +} from 'melody-parser'; + +export const unaryOperators = []; +export const binaryOperators = []; +export const tests = []; + +//region Unary Expressions +export const UnaryNotExpression = createUnaryOperator( + 'not', + 'UnaryNotExpression', + 50, +); +export const UnaryNeqExpression = createUnaryOperator( + '-', + 'UnaryNeqExpression', + 500, +); +export const UnaryPosExpression = createUnaryOperator( + '+', + 'UnaryPosExpression', + 500, +); +//endregion + +//region Binary Expressions +export const BinaryOrExpression = createBinaryOperatorNode({ + text: 'or', + type: 'BinaryOrExpression', + precedence: 10, + associativity: LEFT, +}); +export const BinaryAndExpression = createBinaryOperatorNode({ + text: 'and', + type: 'BinaryAndExpression', + precedence: 15, + associativity: LEFT, +}); + +export const BitwiseOrExpression = createBinaryOperatorNode({ + text: 'b-or', + type: 'BitwiseOrExpression', + precedence: 16, + associativity: LEFT, +}); +export const BitwiseXorExpression = createBinaryOperatorNode({ + text: 'b-xor', + type: 'BitwiseXOrExpression', + precedence: 17, + associativity: LEFT, +}); +export const BitwiseAndExpression = createBinaryOperatorNode({ + text: 'b-and', + type: 'BitwiseAndExpression', + precedence: 18, + associativity: LEFT, +}); + +export const BinaryEqualsExpression = createBinaryOperatorNode({ + text: '==', + type: 'BinaryEqualsExpression', + precedence: 20, + associativity: LEFT, +}); +export const BinaryNotEqualsExpression = createBinaryOperatorNode({ + text: '!=', + type: 'BinaryNotEqualsExpression', + precedence: 20, + associativity: LEFT, +}); +export const BinaryLessThanExpression = createBinaryOperatorNode({ + text: '<', + type: 'BinaryLessThanExpression', + precedence: 20, + associativity: LEFT, +}); +export const BinaryGreaterThanExpression = createBinaryOperatorNode({ + text: '>', + type: 'BinaryGreaterThanExpression', + precedence: 20, + associativity: LEFT, +}); +export const BinaryLessThanOrEqualExpression = createBinaryOperatorNode({ + text: '<=', + type: 'BinaryLessThanOrEqualExpression', + precedence: 20, + associativity: LEFT, +}); +export const BinaryGreaterThanOrEqualExpression = createBinaryOperatorNode({ + text: '>=', + type: 'BinaryGreaterThanOrEqualExpression', + precedence: 20, + associativity: LEFT, +}); + +export const BinaryNotInExpression = createBinaryOperatorNode({ + text: 'not in', + type: 'BinaryNotInExpression', + precedence: 20, + associativity: LEFT, +}); +export const BinaryInExpression = createBinaryOperatorNode({ + text: 'in', + type: 'BinaryInExpression', + precedence: 20, + associativity: LEFT, +}); +export const BinaryMatchesExpression = createBinaryOperatorNode({ + text: 'matches', + type: 'BinaryMatchesExpression', + precedence: 20, + associativity: LEFT, +}); +export const BinaryStartsWithExpression = createBinaryOperatorNode({ + text: 'starts with', + type: 'BinaryStartsWithExpression', + precedence: 20, + associativity: LEFT, +}); +export const BinaryEndsWithExpression = createBinaryOperatorNode({ + text: 'ends with', + type: 'BinaryEndsWithExpression', + precedence: 20, + associativity: LEFT, +}); + +export const BinaryRangeExpression = createBinaryOperatorNode({ + text: '..', + type: 'BinaryRangeExpression', + precedence: 25, + associativity: LEFT, +}); + +export const BinaryAddExpression = createBinaryOperatorNode({ + text: '+', + type: 'BinaryAddExpression', + precedence: 30, + associativity: LEFT, +}); +export const BinarySubExpression = createBinaryOperatorNode({ + text: '-', + type: 'BinarySubExpression', + precedence: 30, + associativity: LEFT, +}); +binaryOperators.push({ + text: '~', + precedence: 40, + associativity: LEFT, + createNode(token, lhs, rhs) { + const op = new BinaryConcatExpression(lhs, rhs); + copyStart(op, lhs); + copyEnd(op, rhs); + return op; + }, +}); +export const BinaryMulExpression = createBinaryOperatorNode({ + text: '*', + type: 'BinaryMulExpression', + precedence: 60, + associativity: LEFT, +}); +export const BinaryDivExpression = createBinaryOperatorNode({ + text: '/', + type: 'BinaryDivExpression', + precedence: 60, + associativity: LEFT, +}); +export const BinaryFloorDivExpression = createBinaryOperatorNode({ + text: '//', + type: 'BinaryFloorDivExpression', + precedence: 60, + associativity: LEFT, +}); +export const BinaryModExpression = createBinaryOperatorNode({ + text: '%', + type: 'BinaryModExpression', + precedence: 60, + associativity: LEFT, +}); + +binaryOperators.push({ + text: 'is', + precedence: 100, + associativity: LEFT, + parse(parser, token, expr) { + const tokens = parser.tokens; + + let not = false; + if (tokens.nextIf(Types.OPERATOR, 'not')) { + not = true; + } + + const test = getTest(parser); + let args = null; + if (tokens.test(Types.LPAREN)) { + args = parser.matchArguments(); + } + const testExpression = test.createNode(expr, args); + setStartFromToken(testExpression, token); + setEndFromToken(testExpression, tokens.la(-1)); + if (not) { + return copyLoc( + new UnaryNotExpression(testExpression), + testExpression, + ); + } + return testExpression; + }, +}); + +function getTest(parser) { + const tokens = parser.tokens; + const nameToken = tokens.la(0); + if (nameToken.type !== Types.NULL) { + tokens.expect(Types.SYMBOL); + } else { + tokens.next(); + } + let testName = nameToken.text; + if (!parser.hasTest(testName)) { + // try 2-words tests + const continuedNameToken = tokens.expect(Types.SYMBOL); + testName += ' ' + continuedNameToken.text; + if (!parser.hasTest(testName)) { + parser.error({ + title: `Unknown test "${testName}"`, + pos: nameToken.pos, + }); + } + } + + return parser.getTest(testName); +} + +export const BinaryPowerExpression = createBinaryOperatorNode({ + text: '**', + type: 'BinaryPowerExpression', + precedence: 200, + associativity: LEFT, +}); +export const BinaryNullCoalesceExpression = createBinaryOperatorNode({ + text: '??', + type: 'BinaryNullCoalesceExpression', + precedence: 300, + associativity: LEFT, +}); +//endregion + +//region Test Expressions +export const TestEvenExpression = createTest('even', 'TestEvenExpression'); +export const TestOddExpression = createTest('odd', 'TestOddExpression'); +export const TestDefinedExpression = createTest( + 'defined', + 'TestDefinedExpression', +); +export const TestSameAsExpression = createTest( + 'same as', + 'TestSameAsExpression', +); +tests.push({ + text: 'sameas', + createNode(expr, args) { + // todo: add deprecation warning + return new TestSameAsExpression(expr, args); + }, +}); +export const TestNullExpression = createTest('null', 'TestNullExpression'); +tests.push({ + text: 'none', + createNode(expr, args) { + return new TestNullExpression(expr, args); + }, +}); +export const TestDivisibleByExpression = createTest( + 'divisible by', + 'TestDivisibleByExpression', +); +tests.push({ + text: 'divisibleby', + createNode(expr, args) { + // todo: add deprecation warning + return new TestDivisibleByExpression(expr, args); + }, +}); +export const TestConstantExpression = createTest( + 'constant', + 'TestConstantExpression', +); +export const TestEmptyExpression = createTest('empty', 'TestEmptyExpression'); +export const TestIterableExpression = createTest( + 'iterable', + 'TestIterableExpression', +); +//endregion + +//region Utilities +function createTest(text, typeName) { + const TestExpression = class extends Node { + constructor(expr: Node, args?: Array) { + super(); + this.expression = expr; + this.arguments = args; + } + }; + type(TestExpression, typeName); + alias(TestExpression, 'Expression', 'TestExpression'); + visitor(TestExpression, 'expression', 'arguments'); + + tests.push({ + text, + createNode(expr, args) { + return new TestExpression(expr, args); + }, + }); + + return TestExpression; +} + +function createBinaryOperatorNode(options) { + const { text, precedence, associativity } = options; + const BinarySubclass = class extends BinaryExpression { + constructor(left: Node, right: Node) { + super(text, left, right); + } + }; + type(BinarySubclass, options.type); + alias(BinarySubclass, 'BinaryExpression', 'Binary', 'Expression'); + visitor(BinarySubclass, 'left', 'right'); + + const operator = { + text, + precedence, + associativity, + }; + if (options.parse) { + operator.parse = options.parse; + } else if (options.createNode) { + operator.createNode = options.createNode; + } else { + operator.createNode = (token, lhs, rhs) => new BinarySubclass(lhs, rhs); + } + binaryOperators.push(operator); + + return BinarySubclass; +} + +function createUnaryOperator(operator, typeName, precedence) { + const UnarySubclass = class extends UnaryExpression { + constructor(argument: Node) { + super(operator, argument); + } + }; + type(UnarySubclass, typeName); + alias(UnarySubclass, 'Expression', 'UnaryLike'); + visitor(UnarySubclass, 'argument'); + + unaryOperators.push({ + text: operator, + precedence, + createNode(token, expr) { + const op = new UnarySubclass(expr); + setStartFromToken(op, token); + copyEnd(op, expr); + return op; + }, + }); + + return UnarySubclass; +} +//endregion diff --git a/src/melody/melody-extension-core/parser/autoescape.js b/src/melody/melody-extension-core/parser/autoescape.js new file mode 100644 index 00000000..99d41222 --- /dev/null +++ b/src/melody/melody-extension-core/parser/autoescape.js @@ -0,0 +1,71 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { AutoescapeBlock } from './../types'; + +export const AutoescapeParser = { + name: 'autoescape', + parse(parser, token) { + const tokens = parser.tokens; + + let escapeType = null, + stringStartToken; + if (tokens.nextIf(Types.TAG_END)) { + escapeType = null; + } else if ((stringStartToken = tokens.nextIf(Types.STRING_START))) { + escapeType = tokens.expect(Types.STRING).text; + if (!tokens.nextIf(Types.STRING_END)) { + parser.error({ + title: + 'autoescape type declaration must be a simple string', + pos: tokens.la(0).pos, + advice: `The type declaration for autoescape must be a simple string such as 'html' or 'js'. +I expected the current string to end with a ${stringStartToken.text} but instead found ${Types + .ERROR_TABLE[tokens.lat(0)] || tokens.lat(0)}.`, + }); + } + } else if (tokens.nextIf(Types.FALSE)) { + escapeType = false; + } else if (tokens.nextIf(Types.TRUE)) { + escapeType = true; + } else { + parser.error({ + title: 'Invalid autoescape type declaration', + pos: tokens.la(0).pos, + advice: `Expected type of autoescape to be a string, boolean or not specified. Found ${tokens.la( + 0, + ).type} instead.`, + }); + } + + const autoescape = new AutoescapeBlock(escapeType); + setStartFromToken(autoescape, token); + let tagEndToken; + autoescape.expressions = parser.parse((_, token, tokens) => { + if ( + token.type === Types.TAG_START && + tokens.nextIf(Types.SYMBOL, 'endautoescape') + ) { + tagEndToken = tokens.expect(Types.TAG_END); + return true; + } + return false; + }).expressions; + setEndFromToken(autoescape, tagEndToken); + + return autoescape; + }, +}; diff --git a/src/melody/melody-extension-core/parser/block.js b/src/melody/melody-extension-core/parser/block.js new file mode 100644 index 00000000..1f626c5d --- /dev/null +++ b/src/melody/melody-extension-core/parser/block.js @@ -0,0 +1,72 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Identifier, PrintExpressionStatement } from 'melody-types'; +import { + Types, + setStartFromToken, + setEndFromToken, + createNode, +} from 'melody-parser'; +import { BlockStatement } from './../types'; + +export const BlockParser = { + name: 'block', + parse(parser, token) { + const tokens = parser.tokens, + nameToken = tokens.expect(Types.SYMBOL); + + let blockStatement; + if (tokens.nextIf(Types.TAG_END)) { + blockStatement = new BlockStatement( + createNode(Identifier, nameToken, nameToken.text), + parser.parse((tokenText, token, tokens) => { + return !!( + token.type === Types.TAG_START && + tokens.nextIf(Types.SYMBOL, 'endblock') + ); + }).expressions, + ); + + if (tokens.nextIf(Types.SYMBOL, nameToken.text)) { + if (tokens.lat(0) !== Types.TAG_END) { + const unexpectedToken = tokens.next(); + parser.error({ + title: 'Block name mismatch', + pos: unexpectedToken.pos, + advice: + unexpectedToken.type == Types.SYMBOL + ? `Expected end of block ${nameToken.text} but instead found end of block ${tokens.la( + 0, + ).text}.` + : `endblock must be followed by either '%}' or the name of the open block. Found a token of type ${Types + .ERROR_TABLE[unexpectedToken.type] || + unexpectedToken.type} instead.`, + }); + } + } + } else { + blockStatement = new BlockStatement( + createNode(Identifier, nameToken, nameToken.text), + new PrintExpressionStatement(parser.matchExpression()), + ); + } + + setStartFromToken(blockStatement, token); + setEndFromToken(blockStatement, tokens.expect(Types.TAG_END)); + + return blockStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/do.js b/src/melody/melody-extension-core/parser/do.js new file mode 100644 index 00000000..9853a1e7 --- /dev/null +++ b/src/melody/melody-extension-core/parser/do.js @@ -0,0 +1,28 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { DoStatement } from './../types'; + +export const DoParser = { + name: 'do', + parse(parser, token) { + const tokens = parser.tokens, + doStatement = new DoStatement(parser.matchExpression()); + setStartFromToken(doStatement, token); + setEndFromToken(doStatement, tokens.expect(Types.TAG_END)); + return doStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/embed.js b/src/melody/melody-extension-core/parser/embed.js new file mode 100644 index 00000000..d7e89287 --- /dev/null +++ b/src/melody/melody-extension-core/parser/embed.js @@ -0,0 +1,58 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Node } from 'melody-types'; +import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { filter } from 'lodash'; +import { EmbedStatement } from './../types'; + +export const EmbedParser = { + name: 'embed', + parse(parser, token) { + const tokens = parser.tokens; + + const embedStatement = new EmbedStatement(parser.matchExpression()); + + if (tokens.nextIf(Types.SYMBOL, 'ignore')) { + tokens.expect(Types.SYMBOL, 'missing'); + embedStatement.ignoreMissing = true; + } + + if (tokens.nextIf(Types.SYMBOL, 'with')) { + embedStatement.argument = parser.matchExpression(); + } + + if (tokens.nextIf(Types.SYMBOL, 'only')) { + embedStatement.contextFree = true; + } + + tokens.expect(Types.TAG_END); + + embedStatement.blocks = filter( + parser.parse((tokenText, token, tokens) => { + return !!( + token.type === Types.TAG_START && + tokens.nextIf(Types.SYMBOL, 'endembed') + ); + }).expressions, + Node.isBlockStatement, + ); + + setStartFromToken(embedStatement, token); + setEndFromToken(embedStatement, tokens.expect(Types.TAG_END)); + + return embedStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/extends.js b/src/melody/melody-extension-core/parser/extends.js new file mode 100644 index 00000000..0a0f7378 --- /dev/null +++ b/src/melody/melody-extension-core/parser/extends.js @@ -0,0 +1,31 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { ExtendsStatement } from './../types'; + +export const ExtendsParser = { + name: 'extends', + parse(parser, token) { + const tokens = parser.tokens; + + const extendsStatement = new ExtendsStatement(parser.matchExpression()); + + setStartFromToken(extendsStatement, token); + setEndFromToken(extendsStatement, tokens.expect(Types.TAG_END)); + + return extendsStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/filter.js b/src/melody/melody-extension-core/parser/filter.js new file mode 100644 index 00000000..0c08836d --- /dev/null +++ b/src/melody/melody-extension-core/parser/filter.js @@ -0,0 +1,47 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Identifier } from 'melody-types'; +import { + Types, + setStartFromToken, + setEndFromToken, + createNode, +} from 'melody-parser'; +import { FilterBlockStatement } from './../types'; + +export const FilterParser = { + name: 'filter', + parse(parser, token) { + const tokens = parser.tokens, + ref = createNode(Identifier, token, 'filter'), + filterExpression = parser.matchFilterExpression(ref); + tokens.expect(Types.TAG_END); + const body = parser.parse((text, token, tokens) => { + return ( + token.type === Types.TAG_START && + tokens.nextIf(Types.SYMBOL, 'endfilter') + ); + }).expressions; + + const filterBlockStatement = new FilterBlockStatement( + filterExpression, + body, + ); + setStartFromToken(filterBlockStatement, token); + setEndFromToken(filterBlockStatement, tokens.expect(Types.TAG_END)); + return filterBlockStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/flush.js b/src/melody/melody-extension-core/parser/flush.js new file mode 100644 index 00000000..6381a4d1 --- /dev/null +++ b/src/melody/melody-extension-core/parser/flush.js @@ -0,0 +1,29 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { FlushStatement } from './../types'; + +export const FlushParser = { + name: 'flush', + parse(parser, token) { + const tokens = parser.tokens, + flushStatement = new FlushStatement(); + + setStartFromToken(flushStatement, token); + setEndFromToken(flushStatement, tokens.expect(Types.TAG_END)); + return flushStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/for.js b/src/melody/melody-extension-core/parser/for.js new file mode 100644 index 00000000..f934f89e --- /dev/null +++ b/src/melody/melody-extension-core/parser/for.js @@ -0,0 +1,89 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Identifier } from 'melody-types'; +import { + Types, + setStartFromToken, + setEndFromToken, + createNode, +} from 'melody-parser'; +import { ForStatement } from './../types'; + +export const ForParser = { + name: 'for', + parse(parser, token) { + const tokens = parser.tokens, + forStatement = new ForStatement(); + + const keyTarget = tokens.expect(Types.SYMBOL); + if (tokens.nextIf(Types.COMMA)) { + forStatement.keyTarget = createNode( + Identifier, + keyTarget, + keyTarget.text, + ); + const valueTarget = tokens.expect(Types.SYMBOL); + forStatement.valueTarget = createNode( + Identifier, + valueTarget, + valueTarget.text, + ); + } else { + forStatement.keyTarget = null; + forStatement.valueTarget = createNode( + Identifier, + keyTarget, + keyTarget.text, + ); + } + + tokens.expect(Types.OPERATOR, 'in'); + + forStatement.sequence = parser.matchExpression(); + + if (tokens.nextIf(Types.SYMBOL, 'if')) { + forStatement.condition = parser.matchExpression(); + } + + tokens.expect(Types.TAG_END); + + forStatement.body = parser.parse((tokenText, token, tokens) => { + return ( + token.type === Types.TAG_START && + (tokens.test(Types.SYMBOL, 'else') || + tokens.test(Types.SYMBOL, 'endfor')) + ); + }); + + if (tokens.nextIf(Types.SYMBOL, 'else')) { + tokens.expect(Types.TAG_END); + forStatement.otherwise = parser.parse( + (tokenText, token, tokens) => { + return ( + token.type === Types.TAG_START && + tokens.test(Types.SYMBOL, 'endfor') + ); + }, + ); + } + tokens.expect(Types.SYMBOL, 'endfor'); + + setStartFromToken(forStatement, token); + setEndFromToken(forStatement, tokens.expect(Types.TAG_END)); + + return forStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/from.js b/src/melody/melody-extension-core/parser/from.js new file mode 100644 index 00000000..cd51781b --- /dev/null +++ b/src/melody/melody-extension-core/parser/from.js @@ -0,0 +1,63 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Identifier } from 'melody-types'; +import { + Types, + setStartFromToken, + setEndFromToken, + createNode, +} from 'melody-parser'; +import { ImportDeclaration, FromStatement } from './../types'; + +export const FromParser = { + name: 'from', + parse(parser, token) { + const tokens = parser.tokens, + source = parser.matchExpression(), + imports = []; + + tokens.expect(Types.SYMBOL, 'import'); + + do { + const name = tokens.expect(Types.SYMBOL); + + let alias = name; + if (tokens.nextIf(Types.SYMBOL, 'as')) { + alias = tokens.expect(Types.SYMBOL); + } + + const importDeclaration = new ImportDeclaration( + createNode(Identifier, name, name.text), + createNode(Identifier, alias, alias.text), + ); + setStartFromToken(importDeclaration, name); + setEndFromToken(importDeclaration, alias); + + imports.push(importDeclaration); + + if (!tokens.nextIf(Types.COMMA)) { + break; + } + } while (!tokens.test(Types.EOF)); + + const fromStatement = new FromStatement(source, imports); + + setStartFromToken(fromStatement, token); + setEndFromToken(fromStatement, tokens.expect(Types.TAG_END)); + + return fromStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/if.js b/src/melody/melody-extension-core/parser/if.js new file mode 100644 index 00000000..03ac44db --- /dev/null +++ b/src/melody/melody-extension-core/parser/if.js @@ -0,0 +1,69 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { IfStatement } from './../types'; + +export const IfParser = { + name: 'if', + parse(parser, token) { + const tokens = parser.tokens; + let test = parser.matchExpression(), + alternate = null; + + tokens.expect(Types.TAG_END); + + const ifStatement = new IfStatement( + test, + parser.parse(matchConsequent).expressions, + ); + + do { + if (tokens.nextIf(Types.SYMBOL, 'else')) { + tokens.expect(Types.TAG_END); + (alternate || ifStatement).alternate = parser.parse( + matchAlternate, + ).expressions; + } else if (tokens.nextIf(Types.SYMBOL, 'elseif')) { + test = parser.matchExpression(); + tokens.expect(Types.TAG_END); + const consequent = parser.parse(matchConsequent).expressions; + alternate = (alternate || ifStatement + ).alternate = new IfStatement(test, consequent); + } + + if (tokens.nextIf(Types.SYMBOL, 'endif')) { + break; + } + } while (!tokens.test(Types.EOF)); + + setStartFromToken(ifStatement, token); + setEndFromToken(ifStatement, tokens.expect(Types.TAG_END)); + + return ifStatement; + }, +}; + +function matchConsequent(tokenText, token, tokens) { + if (token.type === Types.TAG_START) { + const next = tokens.la(0).text; + return next === 'else' || next === 'endif' || next === 'elseif'; + } + return false; +} + +function matchAlternate(tokenText, token, tokens) { + return token.type === Types.TAG_START && tokens.test(Types.SYMBOL, 'endif'); +} diff --git a/src/melody/melody-extension-core/parser/import.js b/src/melody/melody-extension-core/parser/import.js new file mode 100644 index 00000000..73c57213 --- /dev/null +++ b/src/melody/melody-extension-core/parser/import.js @@ -0,0 +1,44 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Identifier } from 'melody-types'; +import { + Types, + setStartFromToken, + setEndFromToken, + createNode, +} from 'melody-parser'; +import { ImportDeclaration } from './../types'; + +export const ImportParser = { + name: 'import', + parse(parser, token) { + const tokens = parser.tokens, + source = parser.matchExpression(); + + tokens.expect(Types.SYMBOL, 'as'); + const alias = tokens.expect(Types.SYMBOL); + + const importStatement = new ImportDeclaration( + source, + createNode(Identifier, alias, alias.text), + ); + + setStartFromToken(importStatement, token); + setEndFromToken(importStatement, tokens.expect(Types.TAG_END)); + + return importStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/include.js b/src/melody/melody-extension-core/parser/include.js new file mode 100644 index 00000000..6f465f89 --- /dev/null +++ b/src/melody/melody-extension-core/parser/include.js @@ -0,0 +1,44 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { IncludeStatement } from './../types'; + +export const IncludeParser = { + name: 'include', + parse(parser, token) { + const tokens = parser.tokens; + + const includeStatement = new IncludeStatement(parser.matchExpression()); + + if (tokens.nextIf(Types.SYMBOL, 'ignore')) { + tokens.expect(Types.SYMBOL, 'missing'); + includeStatement.ignoreMissing = true; + } + + if (tokens.nextIf(Types.SYMBOL, 'with')) { + includeStatement.argument = parser.matchExpression(); + } + + if (tokens.nextIf(Types.SYMBOL, 'only')) { + includeStatement.contextFree = true; + } + + setStartFromToken(includeStatement, token); + setEndFromToken(includeStatement, tokens.expect(Types.TAG_END)); + + return includeStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/macro.js b/src/melody/melody-extension-core/parser/macro.js new file mode 100644 index 00000000..f8415760 --- /dev/null +++ b/src/melody/melody-extension-core/parser/macro.js @@ -0,0 +1,81 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Identifier } from 'melody-types'; +import { + Types, + setStartFromToken, + setEndFromToken, + createNode, +} from 'melody-parser'; +import { MacroDeclarationStatement } from './../types'; + +export const MacroParser = { + name: 'macro', + parse(parser, token) { + const tokens = parser.tokens; + + const nameToken = tokens.expect(Types.SYMBOL); + const args = []; + + tokens.expect(Types.LPAREN); + while (!tokens.test(Types.RPAREN) && !tokens.test(Types.EOF)) { + const arg = tokens.expect(Types.SYMBOL); + args.push(createNode(Identifier, arg, arg.text)); + + if (!tokens.nextIf(Types.COMMA) && !tokens.test(Types.RPAREN)) { + // not followed by comma or rparen + parser.error({ + title: 'Expected comma or ")"', + pos: tokens.la(0).pos, + advice: + 'The argument list of a macro can only consist of parameter names separated by commas.', + }); + } + } + tokens.expect(Types.RPAREN); + + const body = parser.parse((tokenText, token, tokens) => { + return !!( + token.type === Types.TAG_START && + tokens.nextIf(Types.SYMBOL, 'endmacro') + ); + }); + + if (tokens.test(Types.SYMBOL)) { + var nameEndToken = tokens.next(); + if (nameToken.text !== nameEndToken.text) { + parser.error({ + title: `Macro name mismatch, expected "${nameToken.text}" but found "${nameEndToken.text}"`, + pos: nameEndToken.pos, + }); + } + } + + const macroDeclarationStatement = new MacroDeclarationStatement( + createNode(Identifier, nameToken, nameToken.text), + args, + body, + ); + + setStartFromToken(macroDeclarationStatement, token); + setEndFromToken( + macroDeclarationStatement, + tokens.expect(Types.TAG_END), + ); + + return macroDeclarationStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/mount.js b/src/melody/melody-extension-core/parser/mount.js new file mode 100644 index 00000000..a6d4b3d1 --- /dev/null +++ b/src/melody/melody-extension-core/parser/mount.js @@ -0,0 +1,60 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Identifier } from 'melody-types'; +import { MountStatement } from '../types'; +import { + Types, + setStartFromToken, + setEndFromToken, + createNode, +} from 'melody-parser'; + +export const MountParser = { + name: 'mount', + parse(parser, token) { + const tokens = parser.tokens; + + let name = null, + source = null, + key = null, + argument = null; + + if (tokens.test(Types.STRING_START)) { + source = parser.matchStringExpression(); + } else { + const nameToken = tokens.expect(Types.SYMBOL); + name = createNode(Identifier, nameToken, nameToken.text); + if (tokens.nextIf(Types.SYMBOL, 'from')) { + source = parser.matchStringExpression(); + } + } + + if (tokens.nextIf(Types.SYMBOL, 'as')) { + key = parser.matchExpression(); + } + + if (tokens.nextIf(Types.SYMBOL, 'with')) { + argument = parser.matchExpression(); + } + + const mountStatement = new MountStatement(name, source, key, argument); + + setStartFromToken(mountStatement, token); + setEndFromToken(mountStatement, tokens.expect(Types.TAG_END)); + + return mountStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/set.js b/src/melody/melody-extension-core/parser/set.js new file mode 100644 index 00000000..66550703 --- /dev/null +++ b/src/melody/melody-extension-core/parser/set.js @@ -0,0 +1,86 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Identifier } from 'melody-types'; +import { + Types, + setStartFromToken, + setEndFromToken, + createNode, +} from 'melody-parser'; +import { VariableDeclarationStatement, SetStatement } from './../types'; + +export const SetParser = { + name: 'set', + parse(parser, token) { + const tokens = parser.tokens, + names = [], + values = []; + + do { + const name = tokens.expect(Types.SYMBOL); + names.push(createNode(Identifier, name, name.text)); + } while (tokens.nextIf(Types.COMMA)); + + if (tokens.nextIf(Types.ASSIGNMENT)) { + do { + values.push(parser.matchExpression()); + } while (tokens.nextIf(Types.COMMA)); + } else { + if (names.length !== 1) { + parser.error({ + title: 'Illegal multi-set', + pos: tokens.la(0).pos, + advice: + 'When using set with a block, you cannot have multiple targets.', + }); + } + tokens.expect(Types.TAG_END); + + values[0] = parser.parse((tokenText, token, tokens) => { + return !!( + token.type === Types.TAG_START && + tokens.nextIf(Types.SYMBOL, 'endset') + ); + }).expressions; + } + + if (names.length !== values.length) { + parser.error({ + title: 'Mismatch of set names and values', + pos: token.pos, + advice: `When using set, you must ensure that the number of +assigned variable names is identical to the supplied values. However, here I've found +${names.length} variable names and ${values.length} values.`, + }); + } + + // now join names and values + const assignments = []; + for (let i = 0, len = names.length; i < len; i++) { + assignments[i] = new VariableDeclarationStatement( + names[i], + values[i], + ); + } + + const setStatement = new SetStatement(assignments); + + setStartFromToken(setStatement, token); + setEndFromToken(setStatement, tokens.expect(Types.TAG_END)); + + return setStatement; + }, +}; diff --git a/src/melody/melody-extension-core/parser/spaceless.js b/src/melody/melody-extension-core/parser/spaceless.js new file mode 100644 index 00000000..1cb154b3 --- /dev/null +++ b/src/melody/melody-extension-core/parser/spaceless.js @@ -0,0 +1,39 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { SpacelessBlock } from './../types'; + +export const SpacelessParser = { + name: 'spaceless', + parse(parser, token) { + const tokens = parser.tokens; + + tokens.expect(Types.TAG_END); + + const body = parser.parse((tokenText, token, tokens) => { + return !!( + token.type === Types.TAG_START && + tokens.nextIf(Types.SYMBOL, 'endspaceless') + ); + }).expressions; + + const spacelessBlock = new SpacelessBlock(body); + setStartFromToken(spacelessBlock, token); + setEndFromToken(spacelessBlock, tokens.expect(Types.TAG_END)); + + return spacelessBlock; + }, +}; diff --git a/src/melody/melody-extension-core/parser/use.js b/src/melody/melody-extension-core/parser/use.js new file mode 100644 index 00000000..007a990b --- /dev/null +++ b/src/melody/melody-extension-core/parser/use.js @@ -0,0 +1,58 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Identifier } from 'melody-types'; +import { + Types, + setStartFromToken, + setEndFromToken, + copyStart, + copyEnd, + createNode, +} from 'melody-parser'; +import { AliasExpression, UseStatement } from './../types'; + +export const UseParser = { + name: 'use', + parse(parser, token) { + const tokens = parser.tokens; + + const source = parser.matchExpression(), + aliases = []; + + if (tokens.nextIf(Types.SYMBOL, 'with')) { + do { + const nameToken = tokens.expect(Types.SYMBOL), + name = createNode(Identifier, nameToken, nameToken.text); + let alias = name; + if (tokens.nextIf(Types.SYMBOL, 'as')) { + const aliasToken = tokens.expect(Types.SYMBOL); + alias = createNode(Identifier, aliasToken, aliasToken.text); + } + const aliasExpression = new AliasExpression(name, alias); + copyStart(aliasExpression, name); + copyEnd(aliasExpression, alias); + aliases.push(aliasExpression); + } while (tokens.nextIf(Types.COMMA)); + } + + const useStatement = new UseStatement(source, aliases); + + setStartFromToken(useStatement, token); + setEndFromToken(useStatement, tokens.expect(Types.TAG_END)); + + return useStatement; + }, +}; diff --git a/src/melody/melody-extension-core/types.js b/src/melody/melody-extension-core/types.js new file mode 100644 index 00000000..6b25894d --- /dev/null +++ b/src/melody/melody-extension-core/types.js @@ -0,0 +1,311 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Node, + Identifier, + SequenceExpression, + type, + alias, + visitor, +} from 'melody-types'; +import type { StringLiteral } from 'melody-types'; + +export class AutoescapeBlock extends Node { + constructor(type: String | boolean, expressions?: Array) { + super(); + this.escapeType = type; + this.expressions = expressions; + } +} +type(AutoescapeBlock, 'AutoescapeBlock'); +alias(AutoescapeBlock, 'Block', 'Escape'); +visitor(AutoescapeBlock, 'expressions'); + +export class BlockStatement extends Node { + constructor(name: Identifier, body: Node) { + super(); + this.name = name; + this.body = body; + } +} +type(BlockStatement, 'BlockStatement'); +alias(BlockStatement, 'Statement', 'Scope', 'RootScope'); +visitor(BlockStatement, 'body'); + +export class BlockCallExpression extends Node { + constructor(callee: StringLiteral, args: Array = []) { + super(); + this.callee = callee; + this.arguments = args; + } +} +type(BlockCallExpression, 'BlockCallExpression'); +alias(BlockCallExpression, 'Expression', 'FunctionInvocation'); +visitor(BlockCallExpression, 'arguments'); + +export class MountStatement extends Node { + constructor( + name?: Identifier, + source?: String, + key?: Node, + argument?: Node, + ) { + super(); + this.name = name; + this.source = source; + this.key = key; + this.argument = argument; + } +} +type(MountStatement, 'MountStatement'); +alias(MountStatement, 'Statement'); +visitor(MountStatement, 'name', 'source', 'key', 'argument'); + +export class DoStatement extends Node { + constructor(expression: Node) { + super(); + this.value = expression; + } +} +type(DoStatement, 'DoStatement'); +alias(DoStatement, 'Statement'); +visitor(DoStatement, 'value'); + +export class EmbedStatement extends Node { + constructor(parent: Node) { + super(); + this.parent = parent; + this.argument = null; + this.contextFree = false; + // when `true`, missing templates will be ignored + this.ignoreMissing = false; + this.blocks = null; + } +} +type(EmbedStatement, 'EmbedStatement'); +alias(EmbedStatement, 'Statement', 'Include'); +visitor(EmbedStatement, 'argument', 'blocks'); + +export class ExtendsStatement extends Node { + constructor(parentName: Node) { + super(); + this.parentName = parentName; + } +} +type(ExtendsStatement, 'ExtendsStatement'); +alias(ExtendsStatement, 'Statement', 'Include'); +visitor(ExtendsStatement, 'parentName'); + +export class FilterBlockStatement extends Node { + constructor(filterExpression: Node, body: Node) { + super(); + this.filterExpression = filterExpression; + this.body = body; + } +} +type(FilterBlockStatement, 'FilterBlockStatement'); +alias(FilterBlockStatement, 'Statement', 'Block'); +visitor(FilterBlockStatement, 'filterExpression', 'body'); + +export class FlushStatement extends Node { + constructor() { + super(); + } +} +type(FlushStatement, 'FlushStatement'); +alias(FlushStatement, 'Statement'); + +export class ForStatement extends Node { + constructor( + keyTarget?: Identifier = null, + valueTarget?: Identifier = null, + sequence?: Node = null, + condition?: Node = null, + body?: Node = null, + otherwise?: Node = null, + ) { + super(); + this.keyTarget = keyTarget; + this.valueTarget = valueTarget; + this.sequence = sequence; + this.condition = condition; + this.body = body; + this.otherwise = otherwise; + } +} +type(ForStatement, 'ForStatement'); +alias(ForStatement, 'Statement', 'Scope', 'Loop'); +visitor( + ForStatement, + 'keyTarget', + 'valueTarget', + 'sequence', + 'condition', + 'body', + 'otherwise', +); + +export class ImportDeclaration extends Node { + constructor(key: Node, alias: Identifier) { + super(); + this.key = key; + this.alias = alias; + } +} +type(ImportDeclaration, 'ImportDeclaration'); +alias(ImportDeclaration, 'VariableDeclaration'); +visitor(ImportDeclaration, 'key', 'value'); + +export class FromStatement extends Node { + constructor(source: Node, imports: Array) { + super(); + this.source = source; + this.imports = imports; + } +} +type(FromStatement, 'FromStatement'); +alias(FromStatement, 'Statement'); +visitor(FromStatement, 'source', 'imports'); + +export class IfStatement extends Node { + constructor(test: Node, consequent?: Node = null, alternate?: Node = null) { + super(); + this.test = test; + this.consequent = consequent; + this.alternate = alternate; + } +} +type(IfStatement, 'IfStatement'); +alias(IfStatement, 'Statement', 'Conditional'); +visitor(IfStatement, 'test', 'consequent', 'alternate'); + +export class IncludeStatement extends Node { + constructor(source: Node) { + super(); + this.source = source; + this.argument = null; + this.contextFree = false; + // when `true`, missing templates will be ignored + this.ignoreMissing = false; + } +} +type(IncludeStatement, 'IncludeStatement'); +alias(IncludeStatement, 'Statement', 'Include'); +visitor(IncludeStatement, 'source', 'argument'); + +export class MacroDeclarationStatement extends Node { + constructor(name: Identifier, args: Array, body: SequenceExpression) { + super(); + this.name = name; + this.arguments = args; + this.body = body; + } +} +type(MacroDeclarationStatement, 'MacroDeclarationStatement'); +alias(MacroDeclarationStatement, 'Statement', 'Scope', 'RootScope'); +visitor(MacroDeclarationStatement, 'name', 'arguments', 'body'); + +export class VariableDeclarationStatement extends Node { + constructor(name: Identifier, value: Node) { + super(); + this.name = name; + this.value = value; + } +} +type(VariableDeclarationStatement, 'VariableDeclarationStatement'); +alias(VariableDeclarationStatement, 'Statement'); +visitor(VariableDeclarationStatement, 'name', 'value'); + +export class SetStatement extends Node { + constructor(assignments: Array) { + super(); + this.assignments = assignments; + } +} +type(SetStatement, 'SetStatement'); +alias(SetStatement, 'Statement', 'ContextMutation'); +visitor(SetStatement, 'assignments'); + +export class SpacelessBlock extends Node { + constructor(body?: Node = null) { + super(); + this.body = body; + } +} +type(SpacelessBlock, 'SpacelessBlock'); +alias(SpacelessBlock, 'Statement', 'Block'); +visitor(SpacelessBlock, 'body'); + +export class AliasExpression extends Node { + constructor(name: Identifier, alias: Identifier) { + super(); + this.name = name; + this.alias = alias; + } +} +type(AliasExpression, 'AliasExpression'); +alias(AliasExpression, 'Expression'); +visitor(AliasExpression, 'name', 'alias'); + +export class UseStatement extends Node { + constructor(source: Node, aliases: Array) { + super(); + this.source = source; + this.aliases = aliases; + } +} +type(UseStatement, 'UseStatement'); +alias(UseStatement, 'Statement', 'Include'); +visitor(UseStatement, 'source', 'aliases'); + +export { + UnaryNotExpression, + UnaryNeqExpression, + UnaryPosExpression, + BinaryOrExpression, + BinaryAndExpression, + BitwiseOrExpression, + BitwiseXorExpression, + BitwiseAndExpression, + BinaryEqualsExpression, + BinaryNotEqualsExpression, + BinaryLessThanExpression, + BinaryGreaterThanExpression, + BinaryLessThanOrEqualExpression, + BinaryGreaterThanOrEqualExpression, + BinaryNotInExpression, + BinaryInExpression, + BinaryMatchesExpression, + BinaryStartsWithExpression, + BinaryEndsWithExpression, + BinaryRangeExpression, + BinaryAddExpression, + BinaryMulExpression, + BinaryDivExpression, + BinaryFloorDivExpression, + BinaryModExpression, + BinaryPowerExpression, + BinaryNullCoalesceExpression, + TestEvenExpression, + TestOddExpression, + TestDefinedExpression, + TestSameAsExpression, + TestNullExpression, + TestDivisibleByExpression, + TestConstantExpression, + TestEmptyExpression, + TestIterableExpression, +} from './operators'; diff --git a/src/melody/melody-extension-core/visitors/filters.js b/src/melody/melody-extension-core/visitors/filters.js new file mode 100644 index 00000000..ae644009 --- /dev/null +++ b/src/melody/melody-extension-core/visitors/filters.js @@ -0,0 +1,154 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as t from 'babel-types'; +import template from 'babel-template'; + +// use default value if var is null, undefined or an empty string +// but use var if value is 0, false, an empty array or an empty object +const defaultFilter = template("VAR != null && VAR !== '' ? VAR : DEFAULT"); + +export default { + capitalize: 'lodash', + first: 'lodash', + last: 'lodash', + keys: 'lodash', + default(path) { + // babel-template transforms it to an expression statement + // but we really need an expression here, so unwrap it + path.replaceWithJS( + defaultFilter({ + VAR: path.node.target, + DEFAULT: path.node.arguments[0] || t.stringLiteral(''), + }).expression, + ); + }, + abs(path) { + // todo throw error if arguments exist + path.replaceWithJS( + t.callExpression( + t.memberExpression(t.identifier('Math'), t.identifier('abs')), + [path.node.target], + ), + ); + }, + join(path) { + path.replaceWithJS( + t.callExpression( + t.memberExpression(path.node.target, t.identifier('join')), + path.node.arguments, + ), + ); + }, + json_encode(path) { + // todo: handle arguments + path.replaceWithJS( + t.callExpression( + t.memberExpression( + t.identifier('JSON'), + t.identifier('stringify'), + ), + [path.node.target], + ), + ); + }, + length(path) { + path.replaceWithJS( + t.memberExpression(path.node.target, t.identifier('length')), + ); + }, + lower(path) { + path.replaceWithJS( + t.callExpression( + t.memberExpression( + path.node.target, + t.identifier('toLowerCase'), + ), + [], + ), + ); + }, + upper(path) { + path.replaceWithJS( + t.callExpression( + t.memberExpression( + path.node.target, + t.identifier('toUpperCase'), + ), + [], + ), + ); + }, + slice(path) { + path.replaceWithJS( + t.callExpression( + t.memberExpression(path.node.target, t.identifier('slice')), + path.node.arguments, + ), + ); + }, + sort(path) { + path.replaceWithJS( + t.callExpression( + t.memberExpression(path.node.target, t.identifier('sort')), + path.node.arguments, + ), + ); + }, + split(path) { + path.replaceWithJS( + t.callExpression( + t.memberExpression(path.node.target, t.identifier('split')), + path.node.arguments, + ), + ); + }, + trim(path) { + path.replaceWithJS( + t.callExpression( + t.memberExpression(path.node.target, t.identifier('trim')), + path.node.arguments, + ), + ); + }, + convert_encoding(path) { + // encoding conversion is not supported + path.replaceWith(path.node.target); + }, + date_modify(path) { + path.replaceWithJS( + t.callExpression( + t.identifier( + path.state.addImportFrom('melody-runtime', 'strtotime'), + ), + [path.node.arguments[0], path.node.target], + ), + ); + }, + date(path) { + // Not really happy about this since moment.js could well be incompatible with + // the default twig behaviour + // might need to switch to an actual strftime implementation + path.repalceWithJS( + t.callExpression( + t.callExpression( + t.identifier(path.state.addDefaultImportFrom('moment')), + [path.node.target], + ), + [path.node.arguments[0]], + ), + ); + }, +}; diff --git a/src/melody/melody-extension-core/visitors/for.js b/src/melody/melody-extension-core/visitors/for.js new file mode 100644 index 00000000..26e08f6c --- /dev/null +++ b/src/melody/melody-extension-core/visitors/for.js @@ -0,0 +1,392 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { traverse } from 'melody-traverse'; +import * as t from 'babel-types'; +import babelTemplate from 'babel-template'; + +// @param template +// @returns function +// @param context context bindings +// @returns {exprStmt, initDecl, forStmt} +const template = tpl => { + return ctx => parseExpr(babelTemplate(tpl)(ctx)); +}; + +const forWithContext = template(` +{ +let SEQUENCE = SOURCE, +KEY_TARGET = 0, +LENGTH = SEQUENCE.length, +SUB_CONTEXT = CREATE_SUB_CONTEXT(CONTEXT, { + VALUE_TARGET: SEQUENCE[0], + loop: { + index: 1, + index0: 0, + length: LENGTH, + revindex: LENGTH, + revindex0: LENGTH - 1, + first: true, + last: 1 === LENGTH + } +}); +for (; + KEY_TARGET < LENGTH; + KEY_TARGET++ +) { + SUB_CONTEXT.loop.index0++; + SUB_CONTEXT.loop.index++; + SUB_CONTEXT.loop.revindex--; + SUB_CONTEXT.loop.revindex0--; + SUB_CONTEXT.loop.first = false; + SUB_CONTEXT.loop.last = SUB_CONTEXT.loop.revindex === 0; + SUB_CONTEXT.VALUE_TARGET = _sequence[KEY_TARGET + 1]; +} +} +`); + +const basicFor = template(` +{ +let SEQUENCE = SOURCE, +KEY_TARGET = 0, +LENGTH = SEQUENCE.length, +VALUE_TARGET = SEQUENCE[0]; +for (; + KEY_TARGET < LENGTH; + KEY_TARGET++, + VALUE_TARGET = SEQUENCE[_index] +) { +} +} +`); + +const localFor = template(` +{ +let SEQUENCE = SOURCE, +KEY_TARGET = 0, +LENGTH = SEQUENCE.length, +VALUE_TARGET = SEQUENCE[0], +INDEX_BY_1 = 1, +REVERSE_INDEX_BY_1 = LENGTH, +REVERSE_INDEX = LENGTH - 1, +FIRST = true, +LAST = 1 === LENGTH; +for (; + KEY_TARGET < LENGTH; + KEY_TARGET++, + VALUE_TARGET = SEQUENCE[_index] +) { + INDEX_BY_1++; + REVERSE_INDEX_BY_1--; + REVERSE_INDEX--; + FIRST = false; + LAST = REVERSE_INDEX === 0; +} +} +`); + +// returns an object that has the whole expression, init declarations, for loop +// statement in respective properties. +function parseExpr(exprStmt) { + return { + exprStmt: exprStmt, + initDecl: exprStmt.body[0].declarations, + forStmt: exprStmt.body[1], + }; +} + +export default { + analyse: { + ForStatement: { + enter(path) { + const forStmt = path.node, + scope = path.scope; + if (forStmt.keyTarget) { + scope.registerBinding( + forStmt.keyTarget.name, + path.get('keyTarget'), + 'var', + ); + } + if (forStmt.valueTarget) { + scope.registerBinding( + forStmt.valueTarget.name, + path.get('valueTarget'), + 'var', + ); + } + scope.registerBinding('loop', path, 'var'); + }, + exit(path) { + const sequenceName = path.scope.generateUid('sequence'), + lenName = path.scope.generateUid('length'); + path.scope.registerBinding(sequenceName, path, 'var'); + path.scope.registerBinding(lenName, path, 'var'); + let iName; + if (path.node.keyTarget) { + iName = path.node.keyTarget.name; + } else { + iName = path.scope.generateUid('index0'); + path.scope.registerBinding(iName, path, 'var'); + } + path.setData('forStatement.variableLookup', { + sequenceName, + lenName, + iName, + }); + + if (path.scope.escapesContext) { + const contextName = path.scope.generateUid('context'); + path.scope.registerBinding(contextName, path, 'const'); + path.scope.contextName = contextName; + path.scope.getBinding('loop').kind = 'context'; + if (path.node.valueTarget) { + path.scope.getBinding(path.node.valueTarget.name).kind = + 'context'; + } + } else if (path.scope.getBinding('loop').references) { + const indexName = path.scope.generateUid('index'); + path.scope.registerBinding(indexName, path, 'var'); + const revindexName = path.scope.generateUid('revindex'); + path.scope.registerBinding(revindexName, path, 'var'); + const revindex0Name = path.scope.generateUid('revindex0'); + path.scope.registerBinding(revindex0Name, path, 'var'); + const firstName = path.scope.generateUid('first'); + path.scope.registerBinding(firstName, path, 'var'); + const lastName = path.scope.generateUid('last'); + path.scope.registerBinding(lastName, path, 'var'); + + const lookupTable = { + index: indexName, + index0: iName, + length: lenName, + revindex: revindexName, + revindex0: revindex0Name, + first: firstName, + last: lastName, + }; + path.setData('forStatement.loopLookup', lookupTable); + + const loopBinding = path.scope.getBinding('loop'); + for (const loopPath of loopBinding.referencePaths) { + const memExpr = loopPath.parentPath; + + if (memExpr.is('MemberExpression')) { + const typeName = memExpr.node.property.name; + if (typeName === 'index0') { + memExpr.replaceWithJS({ + type: 'BinaryExpression', + operator: '-', + left: { + type: 'Identifier', + name: indexName, + }, + right: { type: 'NumericLiteral', value: 1 }, + extra: { + parenthesized: true, + }, + }); + } else { + memExpr.replaceWithJS({ + type: 'Identifier', + name: lookupTable[typeName], + }); + } + } + } + } + }, + }, + }, + convert: { + ForStatement: { + enter(path) { + if (path.scope.escapesContext) { + var parentContextName = path.scope.parent.contextName; + if (path.node.otherwise) { + const alternate = path.get('otherwise'); + if (alternate.is('Scope')) { + alternate.scope.contextName = parentContextName; + } + } + + const sequence = path.get('sequence'); + + if (sequence.is('Identifier')) { + sequence.setData( + 'Identifier.contextName', + parentContextName, + ); + } else { + traverse(path.node.sequence, { + Identifier(id) { + id.setData( + 'Identifier.contextName', + parentContextName, + ); + }, + }); + } + } + }, + exit(path) { + const node = path.node; + const { sequenceName, lenName, iName } = path.getData( + 'forStatement.variableLookup', + ); + let expr; + if (path.scope.escapesContext) { + const contextName = path.scope.contextName; + expr = forWithContext({ + CONTEXT: t.identifier(path.scope.parent.contextName), + SUB_CONTEXT: t.identifier(contextName), + CREATE_SUB_CONTEXT: t.identifier( + this.addImportFrom( + 'melody-runtime', + 'createSubContext', + ), + ), + KEY_TARGET: t.identifier(iName), + SOURCE: path.get('sequence').node, + SEQUENCE: t.identifier(sequenceName), + LENGTH: t.identifier(lenName), + VALUE_TARGET: node.valueTarget, + }); + if (node.keyTarget) { + expr.forStmt.body.body.push({ + type: 'ExpressionStatement', + expression: { + type: 'AssignmentExpression', + operator: '=', + left: { + type: 'MemberExpression', + object: { + type: 'Identifier', + name: contextName, + }, + property: { + type: 'Identifier', + name: node.keyTarget.name, + }, + computed: false, + }, + right: { + type: 'Identifier', + name: iName, + }, + }, + }); + expr.initDecl[ + expr.initDecl.length - 1 + ].init.arguments[1].properties.push({ + type: 'ObjectProperty', + method: false, + shorthand: false, + computed: false, + key: { + type: 'Identifier', + name: node.keyTarget.name, + }, + value: { + type: 'Identifier', + name: iName, + }, + }); + } + } else if (path.scope.getBinding('loop').references) { + const { + index: indexName, + revindex: revindexName, + revindex0: revindex0Name, + first: firstName, + last: lastName, + } = path.getData('forStatement.loopLookup'); + + expr = localFor({ + KEY_TARGET: t.identifier(iName), + SOURCE: path.get('sequence').node, + SEQUENCE: t.identifier(sequenceName), + LENGTH: t.identifier(lenName), + VALUE_TARGET: node.valueTarget, + INDEX_BY_1: t.identifier(indexName), + REVERSE_INDEX: t.identifier(revindex0Name), + REVERSE_INDEX_BY_1: t.identifier(revindexName), + FIRST: t.identifier(firstName), + LAST: t.identifier(lastName), + }); + } else { + expr = basicFor({ + SEQUENCE: t.identifier(sequenceName), + SOURCE: path.get('sequence').node, + KEY_TARGET: t.identifier(iName), + LENGTH: t.identifier(lenName), + VALUE_TARGET: node.valueTarget, + }); + } + + expr.forStmt.body.body.unshift(...path.get('body').node.body); + + let uniteratedName; + if (node.otherwise) { + uniteratedName = path.scope.generateUid('uniterated'); + path.scope.parent.registerBinding( + uniteratedName, + path, + 'var', + ); + expr.forStmt.body.body.push( + t.expressionStatement( + t.assignmentExpression( + '=', + t.identifier(uniteratedName), + t.booleanLiteral(false), + ), + ), + ); + } + + if (node.condition) { + expr.forStmt.body = t.blockStatement([ + { + type: 'IfStatement', + test: node.condition, + consequent: t.blockStatement( + expr.forStmt.body.body, + ), + }, + ]); + } + + if (uniteratedName) { + path.replaceWithMultipleJS( + t.variableDeclaration('let', [ + t.variableDeclarator( + t.identifier(uniteratedName), + t.booleanLiteral(true), + ), + ]), + expr.exprStmt, + t.ifStatement( + t.identifier(uniteratedName), + node.otherwise, + ), + ); + } else { + path.replaceWithJS(expr.exprStmt); + } + }, + }, + }, +}; diff --git a/src/melody/melody-extension-core/visitors/functions.js b/src/melody/melody-extension-core/visitors/functions.js new file mode 100644 index 00000000..c701cf98 --- /dev/null +++ b/src/melody/melody-extension-core/visitors/functions.js @@ -0,0 +1,100 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as t from 'babel-types'; + +function addOne(expr) { + return t.binaryExpression('+', expr, t.numericLiteral(1)); +} + +export default { + range(path) { + const args = path.node.arguments; + const callArgs = []; + if (args.length === 1) { + callArgs.push(addOne(args[0])); + } else if (args.length === 3) { + callArgs.push(args[0]); + callArgs.push(addOne(args[1])); + callArgs.push(args[2]); + } else if (args.length === 2) { + callArgs.push(args[0], addOne(args[1])); + } else { + path.state.error( + 'Invalid range call', + path.node.pos, + `The range function accepts 1 to 3 arguments but you have specified ${args.length} arguments instead.`, + ); + } + + path.replaceWithJS( + t.callExpression( + t.identifier(path.state.addImportFrom('lodash', 'range')), + callArgs, + ), + ); + }, + // range: 'lodash', + dump(path) { + if (!path.parentPath.is('PrintExpressionStatement')) { + path.state.error( + 'dump must be used in a lone expression', + path.node.pos, + 'The dump function does not have a return value. Thus it must be used as the only expression.', + ); + } + path.parentPath.replaceWithJS( + t.expressionStatement( + t.callExpression( + t.memberExpression( + t.identifier('console'), + t.identifier('log'), + ), + path.node.arguments, + ), + ), + ); + }, + include(path) { + if (!path.parentPath.is('PrintExpressionStatement')) { + path.state.error({ + title: 'Include function does not return value', + pos: path.node.loc.start, + advice: `The include function currently does not return a value. + Thus you must use it like a regular include tag.`, + }); + } + const includeName = path.scope.generateUid('include'); + const importDecl = t.importDeclaration( + [t.importDefaultSpecifier(t.identifier(includeName))], + path.node.arguments[0], + ); + path.state.program.body.splice(0, 0, importDecl); + path.scope.registerBinding(includeName); + + const argument = path.node.arguments[1]; + + const includeCall = t.expressionStatement( + t.callExpression( + t.memberExpression( + t.identifier(includeName), + t.identifier('render'), + ), + argument ? [argument] : [], + ), + ); + path.replaceWithJS(includeCall); + }, +}; diff --git a/src/melody/melody-extension-core/visitors/tests.js b/src/melody/melody-extension-core/visitors/tests.js new file mode 100644 index 00000000..f5359c1f --- /dev/null +++ b/src/melody/melody-extension-core/visitors/tests.js @@ -0,0 +1,127 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as t from 'babel-types'; + +export default { + convert: { + TestEvenExpression: { + exit(path) { + const expr = t.unaryExpression( + '!', + t.binaryExpression( + '%', + path.get('expression').node, + t.numericLiteral(2), + ), + ); + expr.extra = { parenthesizedArgument: true }; + path.replaceWithJS(expr); + }, + }, + TestOddExpression: { + exit(path) { + const expr = t.unaryExpression( + '!', + t.unaryExpression( + '!', + t.binaryExpression( + '%', + path.get('expression').node, + t.numericLiteral(2), + ), + ), + ); + expr.extra = { parenthesizedArgument: true }; + path.replaceWithJS(expr); + }, + }, + TestDefinedExpression: { + exit(path) { + path.replaceWithJS( + t.binaryExpression( + '!==', + t.unaryExpression( + 'typeof', + path.get('expression').node, + ), + t.stringLiteral('undefined'), + ), + ); + }, + }, + TestEmptyExpression: { + exit(path) { + path.replaceWithJS( + t.callExpression( + t.identifier( + this.addImportFrom('melody-runtime', 'isEmpty'), + ), + [path.get('expression').node], + ), + ); + }, + }, + TestSameAsExpression: { + exit(path) { + path.replaceWithJS( + t.binaryExpression( + '===', + path.get('expression').node, + path.get('arguments')[0].node, + ), + ); + }, + }, + TestNullExpression: { + exit(path) { + path.replaceWithJS( + t.binaryExpression( + '===', + path.get('expression').node, + t.nullLiteral(), + ), + ); + }, + }, + TestDivisibleByExpression: { + exit(path) { + path.replaceWithJS( + t.unaryExpression( + '!', + t.binaryExpression( + '%', + path.get('expression').node, + path.node.arguments[0], + ), + ), + ); + }, + }, + TestIterableExpression: { + exit(path) { + path.replaceWithJS( + t.callExpression( + t.memberExpression( + t.identifier('Array'), + t.identifier('isArray'), + ), + [path.node.expression], + ), + ); + }, + }, + }, +}; diff --git a/src/melody/melody-parser/Associativity.js b/src/melody/melody-parser/Associativity.js new file mode 100644 index 00000000..d66f808e --- /dev/null +++ b/src/melody/melody-parser/Associativity.js @@ -0,0 +1,17 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export var LEFT = Symbol(); +export var RIGHT = Symbol(); diff --git a/src/melody/melody-parser/CharStream.js b/src/melody/melody-parser/CharStream.js new file mode 100644 index 00000000..70e5ff1e --- /dev/null +++ b/src/melody/melody-parser/CharStream.js @@ -0,0 +1,82 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const EOF = Symbol(); + +export class CharStream { + constructor(input) { + this.input = String(input); + this.length = this.input.length; + this.index = 0; + this.position = { line: 1, column: 0 }; + } + + get source() { + return this.input; + } + + reset() { + this.rewind({ line: 1, column: 0, index: 0 }); + } + + mark() { + let { line, column } = this.position, + index = this.index; + return { line, column, index }; + } + + rewind(marker) { + this.position.line = marker.line; + this.position.column = marker.column; + this.index = marker.index; + } + + la(offset) { + var index = this.index + offset; + return index < this.length ? this.input.charAt(index) : EOF; + } + + lac(offset) { + var index = this.index + offset; + return index < this.length ? this.input.charCodeAt(index) : EOF; + } + + next() { + if (this.index === this.length) { + return EOF; + } + var ch = this.input.charAt(this.index); + this.index++; + this.position.column++; + if (ch === '\n') { + this.position.line += 1; + this.position.column = 0; + } + return ch; + } + + match(str) { + const start = this.mark(); + for (let i = 0, len = str.length; i < len; i++) { + const ch = this.next(); + if (ch !== str.charAt(i) || ch === EOF) { + this.rewind(start); + return false; + } + } + return true; + } +} diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js new file mode 100644 index 00000000..0880cd8e --- /dev/null +++ b/src/melody/melody-parser/Lexer.js @@ -0,0 +1,602 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as TokenTypes from './TokenTypes'; +import { EOF } from './CharStream'; + +const State = { + TEXT: 'TEXT', + EXPRESSION: 'EXPRESSION', + TAG: 'TAG', + INTERPOLATION: 'INTERPOLATION', + STRING_SINGLE: 'STRING_SINGLE', + STRING_DOUBLE: 'STRING_DOUBLE', + ELEMENT: 'ELEMENT', + ATTRIBUTE_VALUE: 'ATTRIBUTE_VALUE', +}; + +const STATE = Symbol(), + OPERATORS = Symbol(), + STRING_START = Symbol(); + +const CHAR_TO_TOKEN = { + '[': TokenTypes.LBRACE, + ']': TokenTypes.RBRACE, + '(': TokenTypes.LPAREN, + ')': TokenTypes.RPAREN, + '{': TokenTypes.LBRACKET, + '}': TokenTypes.RBRACKET, + ':': TokenTypes.COLON, + '.': TokenTypes.DOT, + '|': TokenTypes.PIPE, + ',': TokenTypes.COMMA, + '?': TokenTypes.QUESTION_MARK, + '=': TokenTypes.ASSIGNMENT, + //'<': TokenTypes.ELEMENT_START, + //'>': TokenTypes.ELEMENT_END, + '/': TokenTypes.SLASH, +}; + +export default class Lexer { + constructor(input) { + this.input = input; + this[STATE] = [State.TEXT]; + this[OPERATORS] = []; + this[STRING_START] = null; + } + + reset() { + this.input.reset(); + this[STATE] = [State.TEXT]; + } + + get source() { + return this.input.source; + } + + addOperators(...ops) { + this[OPERATORS].push(...ops); + this[OPERATORS].sort((a, b) => (a.length > b.length ? -1 : 1)); + } + + get state() { + return this[STATE][this[STATE].length - 1]; + } + + pushState(state) { + this[STATE].push(state); + } + + popState() { + this[STATE].length--; + } + + createToken(type, pos) { + let input = this.input, + endPos = input.mark(), + end = endPos.index; + return { + type, + pos, + endPos, + end, + length: end - pos.index, + source: input.input, + text: input.input.substr(pos.index, end - pos.index), + toString: function() { + return this.text; + }, + }; + } + + next() { + let input = this.input, + pos, + c; + while ((c = input.la(0)) !== EOF) { + pos = input.mark(); + if ( + this.state !== State.TEXT && + this.state !== State.STRING_DOUBLE && + this.state !== State.STRING_SINGLE && + this.state !== State.ATTRIBUTE_VALUE && + isWhitespace(c) + ) { + input.next(); + while ((c = input.la(0)) !== EOF && isWhitespace(c)) { + input.next(); + } + return this.createToken(TokenTypes.WHITESPACE, pos); + } + if (c === '{' && input.la(1) === '#') { + input.next(); + input.next(); + if (input.la(0) === '-') { + input.next(); + } + while ((c = input.la(0)) !== EOF) { + if ( + (c === '#' && input.la(1) === '}') || + (c === '-' && + input.la(1) === '#' && + input.la(2) === '}') + ) { + if (c === '-') { + input.next(); + } + input.next(); + input.next(); + return this.createToken(TokenTypes.COMMENT, pos); + } + input.next(); + } + } + if (this.state === State.TEXT) { + let entityToken; + if (c === '<') { + if ( + input.la(1) === '{' || + isAlpha(input.lac(1)) || + input.la(1) === '/' + ) { + input.next(); + this.pushState(State.ELEMENT); + return this.createToken(TokenTypes.ELEMENT_START, pos); + } else if ( + input.la(1) === '!' && + input.la(2) === '-' && + input.la(3) === '-' + ) { + // match HTML comment + input.next(); // < + input.next(); // ! + input.next(); // - + input.next(); // - + while ((c = input.la(0)) !== EOF) { + if (c === '-' && input.la(1) === '-') { + input.next(); + input.next(); + if (!(c = input.next()) === '>') { + this.error( + 'Unexpected end for HTML comment', + input.mark(), + `Expected comment to end with '>' but found '${c}' instead.`, + ); + } + break; + } + input.next(); + } + return this.createToken(TokenTypes.HTML_COMMENT, pos); + } else { + return this.matchText(pos); + } + } else if (c === '{') { + return this.matchExpressionToken(pos); + } else if (c === '&' && (entityToken = this.matchEntity(pos))) { + return entityToken; + } else { + return this.matchText(pos); + } + } else if (this.state === State.EXPRESSION) { + if ( + (c === '}' && input.la(1) === '}') || + (c === '-' && input.la(1) === '}' && input.la(2) === '}') + ) { + if (c === '-') { + input.next(); + } + input.next(); + input.next(); + this.popState(); + return this.createToken(TokenTypes.EXPRESSION_END, pos); + } + return this.matchExpression(pos); + } else if (this.state === State.TAG) { + if ( + (c === '%' && input.la(1) === '}') || + (c === '-' && input.la(1) === '%' && input.la(2) === '}') + ) { + if (c === '-') { + input.next(); + } + input.next(); + input.next(); + this.popState(); + return this.createToken(TokenTypes.TAG_END, pos); + } + return this.matchExpression(pos); + } else if ( + this.state === State.STRING_SINGLE || + this.state === State.STRING_DOUBLE + ) { + return this.matchString(pos, true); + } else if (this.state === State.INTERPOLATION) { + if (c === '}') { + input.next(); + this.popState(); // pop interpolation + return this.createToken(TokenTypes.INTERPOLATION_END, pos); + } + return this.matchExpression(pos); + } else if (this.state === State.ELEMENT) { + switch (c) { + case '/': + input.next(); + return this.createToken(TokenTypes.SLASH, pos); + case '{': + return this.matchExpressionToken(pos); + case '>': + input.next(); + this.popState(); + return this.createToken(TokenTypes.ELEMENT_END, pos); + case '"': + input.next(); + this.pushState(State.ATTRIBUTE_VALUE); + return this.createToken(TokenTypes.STRING_START, pos); + case '=': + input.next(); + return this.createToken(TokenTypes.ASSIGNMENT, pos); + default: + return this.matchSymbol(pos); + } + } else if (this.state === State.ATTRIBUTE_VALUE) { + if (c === '"') { + input.next(); + this.popState(); + return this.createToken(TokenTypes.STRING_END, pos); + } else { + return this.matchAttributeValue(pos); + } + } else { + return this.error(`Invalid state ${this.state}`, pos); + } + } + return TokenTypes.EOF_TOKEN; + } + + matchExpressionToken(pos) { + const input = this.input; + switch (input.la(1)) { + case '{': + input.next(); + input.next(); + this.pushState(State.EXPRESSION); + if (input.la(0) === '-') { + input.next(); + } + return this.createToken(TokenTypes.EXPRESSION_START, pos); + case '%': + input.next(); + input.next(); + this.pushState(State.TAG); + if (input.la(0) === '-') { + input.next(); + } + return this.createToken(TokenTypes.TAG_START, pos); + case '#': + input.next(); + input.next(); + if (input.la(0) === '-') { + input.next(); + } + return this.matchComment(pos); + default: + return this.matchText(pos); + } + } + + matchExpression(pos) { + let input = this.input, + c = input.la(0); + switch (c) { + case "'": + this.pushState(State.STRING_SINGLE); + input.next(); + return this.createToken(TokenTypes.STRING_START, pos); + case '"': + this.pushState(State.STRING_DOUBLE); + input.next(); + return this.createToken(TokenTypes.STRING_START, pos); + default: { + if (isDigit(input.lac(0))) { + input.next(); + return this.matchNumber(pos); + } + if ( + (c === 't' && input.match('true')) || + (c === 'T' && input.match('TRUE')) + ) { + return this.createToken(TokenTypes.TRUE, pos); + } + if ( + (c === 'f' && input.match('false')) || + (c === 'F' && input.match('FALSE')) + ) { + return this.createToken(TokenTypes.FALSE, pos); + } + if ( + (c === 'n' && + (input.match('null') || input.match('none'))) || + (c === 'N' && (input.match('NULL') || input.match('NONE'))) + ) { + return this.createToken(TokenTypes.NULL, pos); + } + const { + longestMatchingOperator, + longestMatchEndPos, + } = this.findLongestMatchingOperator(); + const cc = input.lac(0); + if (cc === 95 /* _ */ || isAlpha(cc) || isDigit(cc)) { + // okay... this could be either a symbol or an operator + input.next(); + const sym = this.matchSymbol(pos); + if (sym.text.length <= longestMatchingOperator.length) { + // the operator was longer so let's use that + input.rewind(longestMatchEndPos); + return this.createToken(TokenTypes.OPERATOR, pos); + } + // found a symbol + return sym; + } else if (longestMatchingOperator) { + input.rewind(longestMatchEndPos); + return this.createToken(TokenTypes.OPERATOR, pos); + } else if (CHAR_TO_TOKEN.hasOwnProperty(c)) { + input.next(); + return this.createToken(CHAR_TO_TOKEN[c], pos); + } else { + return this.error(`Unknown token ${c}`, pos); + } + } + } + } + + findLongestMatchingOperator() { + const input = this.input, + start = input.mark(); + let longestMatchingOperator = '', + longestMatchEndPos = null; + for (let i = 0, ops = this[OPERATORS], len = ops.length; i < len; i++) { + const op = ops[i]; + if (op.length > longestMatchingOperator.length && input.match(op)) { + longestMatchingOperator = op; + longestMatchEndPos = input.mark(); + input.rewind(start); + } + } + input.rewind(start); + return { longestMatchingOperator, longestMatchEndPos }; + } + + error(message, pos, advice = '') { + const errorToken = this.createToken(TokenTypes.ERROR, pos); + errorToken.message = message; + errorToken.advice = advice; + return errorToken; + } + + matchEntity(pos) { + const input = this.input; + input.next(); // & + if (input.la(0) === '#') { + input.next(); // # + if (input.la(0) === 'x') { + // hexadecimal numeric character reference + input.next(); // x + let c = input.la(0); + while ( + ('a' <= c && c <= 'f') || + ('A' <= c && c <= 'F') || + isDigit(input.lac(0)) + ) { + input.next(); + c = input.la(0); + } + if (input.la(0) === ';') { + input.next(); + } else { + input.rewind(pos); + return null; + } + } else if (isDigit(input.lac(0))) { + // decimal numeric character reference + // consume decimal numbers + do { + input.next(); + } while (isDigit(input.lac(0))); + // check for final ";" + if (input.la(0) === ';') { + input.next(); + } else { + input.rewind(pos); + return null; + } + } else { + input.rewind(pos); + return null; + } + } else { + // match named character reference + while (isAlpha(input.lac(0))) { + input.next(); + } + if (input.la(0) === ';') { + input.next(); + } else { + input.rewind(pos); + return null; + } + } + return this.createToken(TokenTypes.ENTITY, pos); + } + + matchSymbol(pos) { + let input = this.input, + inElement = this.state === State.ELEMENT, + c; + while ( + (c = input.lac(0)) && + (c === 95 || + isAlpha(c) || + isDigit(c) || + (inElement && (c === 45 || c === 58))) + ) { + input.next(); + } + var end = input.mark(); + if (pos.index === end.index) { + return this.error( + 'Expected an Identifier', + pos, + inElement + ? `Expected a valid attribute name, but instead found "${input.la( + 0, + )}", which is not part of a valid attribute name.` + : `Expected letter, digit or underscore but found ${input.la( + 0, + )} instead.`, + ); + } + return this.createToken(TokenTypes.SYMBOL, pos); + } + + matchString(pos, allowInterpolation = true) { + const input = this.input, + start = this.state === State.STRING_SINGLE ? "'" : '"'; + let c; + // string starts with an interpolation + if (allowInterpolation && input.la(0) === '#' && input.la(1) === '{') { + this.pushState(State.INTERPOLATION); + input.next(); + input.next(); + return this.createToken(TokenTypes.INTERPOLATION_START, pos); + } + if (input.la(0) === start) { + input.next(); + this.popState(); + return this.createToken(TokenTypes.STRING_END, pos); + } + while ((c = input.la(0)) !== start && c !== EOF) { + if (c === '\\' && input.la(1) === start) { + // escape sequence for string start + input.next(); + input.next(); + } else if (allowInterpolation && c === '#' && input.la(1) === '{') { + // found interpolation start, string part matched + // next iteration will match the interpolation + break; + } else { + input.next(); + } + } + var result = this.createToken(TokenTypes.STRING, pos); + result.text = result.text.replace('\\', ''); + return result; + } + + matchAttributeValue(pos) { + let input = this.input, + start = this.state === State.STRING_SINGLE ? "'" : '"', + c; + if (input.la(0) === '{') { + return this.matchExpressionToken(pos); + } + while ((c = input.la(0)) !== start && c !== EOF) { + if (c === '\\' && input.la(1) === start) { + input.next(); + input.next(); + } else if (c === '{') { + // interpolation start + break; + } else if (c === start) { + break; + } else { + input.next(); + } + } + var result = this.createToken(TokenTypes.STRING, pos); + result.text = result.text.replace('\\', ''); + return result; + } + + matchNumber(pos) { + let input = this.input, + c; + while ((c = input.lac(0)) !== EOF) { + if (!isDigit(c)) { + break; + } + input.next(); + } + if (input.la(0) === '.') { + input.next(); + while ((c = input.lac(0)) !== EOF) { + if (!isDigit(c)) { + break; + } + input.next(); + } + } + return this.createToken(TokenTypes.NUMBER, pos); + } + + matchText(pos) { + let input = this.input, + exit = false, + c; + while (!exit && ((c = input.la(0)) && c !== EOF)) { + if (c === '{') { + const c2 = input.la(1); + if (c2 === '{' || c2 === '#' || c2 === '%') { + break; + } + } else if (c === '<') { + if (input.la(1) === '/' || isAlpha(input.lac(1))) { + break; + } else if (input.la(1) === '{') { + const c2 = input.la(1); + if (c2 === '{' || c2 === '#' || c2 === '%') { + break; + } + } + } + input.next(); + } + return this.createToken(TokenTypes.TEXT, pos); + } + + matchComment(pos) { + let input = this.input, + c; + while ((c = input.next()) !== EOF) { + if (c === '#' && input.la(0) === '}') { + input.next(); // consume '}' + break; + } + } + return this.createToken(TokenTypes.COMMENT, pos); + } +} + +function isWhitespace(c) { + return c === '\n' || c === ' ' || c === '\t'; +} + +function isAlpha(c) { + return (65 <= c && c <= 90) || (97 <= c && c <= 122); +} + +function isDigit(c) { + return 48 <= c && c <= 57; +} diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js new file mode 100644 index 00000000..ebb85a84 --- /dev/null +++ b/src/melody/melody-parser/Parser.js @@ -0,0 +1,707 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as n from 'melody-types'; +import * as Types from './TokenTypes'; +import { LEFT, RIGHT } from './Associativity'; +import { + setStartFromToken, + setEndFromToken, + copyStart, + copyEnd, + copyLoc, + createNode, +} from './util'; +import { voidElements } from './elementInfo'; +import * as he from 'he'; + +type UnaryOperator = { + text: String, + precendence: Number, + createNode: Function, +}; + +type BinaryOperator = { + text: String, + precendence: Number, + createNode: Function, + associativity: LEFT | RIGHT, + parse: Function, +}; + +const UNARY = Symbol(), + BINARY = Symbol(), + TAG = Symbol(), + TEST = Symbol(); +export default class Parser { + constructor(tokenStream) { + this.tokens = tokenStream; + this[UNARY] = {}; + this[BINARY] = {}; + this[TAG] = {}; + this[TEST] = {}; + } + + addUnaryOperator(op: UnaryOperator) { + this[UNARY][op.text] = op; + return this; + } + + addBinaryOperator(op: BinaryOperator) { + this[BINARY][op.text] = op; + return this; + } + + addTag(tag) { + this[TAG][tag.name] = tag; + return this; + } + + addTest(test) { + this[TEST][test.text] = test; + } + + hasTest(test) { + return !!this[TEST][test]; + } + + getTest(test) { + return this[TEST][test]; + } + + isUnary(token) { + return token.type === Types.OPERATOR && !!this[UNARY][token.text]; + } + + getBinaryOperator(token) { + return token.type === Types.OPERATOR && this[BINARY][token.text]; + } + + parse(test = null) { + let tokens = this.tokens, + p = setStartFromToken(new n.SequenceExpression(), tokens.la(0)); + while (!tokens.test(Types.EOF)) { + const token = tokens.next(); + if (!p) { + p = setStartFromToken(new n.SequenceExpression(), token); + } + if (test && test(tokens.la(0).text, token, tokens)) { + setEndFromToken(p, token); + return p; + } + switch (token.type) { + case Types.EXPRESSION_START: { + const expression = this.matchExpression(); + p.add( + copyLoc( + new n.PrintExpressionStatement(expression), + expression, + ), + ); + setEndFromToken(p, tokens.expect(Types.EXPRESSION_END)); + break; + } + case Types.TAG_START: + p.add(this.matchTag()); + break; + case Types.TEXT: + p.add( + createNode( + n.PrintTextStatement, + token, + createNode(n.StringLiteral, token, token.text), + ), + ); + break; + case Types.ENTITY: + p.add( + createNode( + n.PrintTextStatement, + token, + createNode( + n.StringLiteral, + token, + he.decode(token.text), + ), + ), + ); + break; + case Types.ELEMENT_START: + p.add(this.matchElement()); + break; + } + } + return p; + } + + /** + * matchElement = '<' SYMBOL attributes* '/'? '>' (children)* '<' '/' SYMBOL '>' + * attributes = SYMBOL '=' (matchExpression | matchString) + * | matchExpression + */ + matchElement() { + let tokens = this.tokens, + elementStartToken = tokens.la(0), + elementName, + element; + if (!(elementName = tokens.nextIf(Types.SYMBOL))) { + this.error({ + title: 'Expected element start', + pos: elementStartToken.pos, + advice: + tokens.lat(0) === Types.SLASH + ? `Unexpected closing "${tokens.la(1) + .text}" tag. Seems like your DOM is out of control.` + : 'Expected an element to start', + }); + } + + element = new n.Element(elementName.text); + setStartFromToken(element, elementStartToken); + + this.matchAttributes(element, tokens); + + if (tokens.nextIf(Types.SLASH)) { + tokens.expect(Types.ELEMENT_END); + element.selfClosing = true; + } else { + tokens.expect(Types.ELEMENT_END); + if (voidElements[elementName.text]) { + element.selfClosing = true; + } else { + element.children = this.parse(function(_, token, tokens) { + if ( + token.type === Types.ELEMENT_START && + tokens.lat(0) === Types.SLASH + ) { + const name = tokens.la(1); + if ( + name.type === Types.SYMBOL && + name.text === elementName.text + ) { + tokens.next(); // SLASH + tokens.next(); // elementName + tokens.expect(Types.ELEMENT_END); + return true; + } + } + return false; + }).expressions; + } + } + setEndFromToken(element, tokens.la(-1)); + return element; + } + + matchAttributes(element, tokens) { + while ( + tokens.lat(0) !== Types.SLASH && + tokens.lat(0) !== Types.ELEMENT_END + ) { + const key = tokens.nextIf(Types.SYMBOL); + if (key) { + const keyNode = new n.Identifier(key.text); + setStartFromToken(keyNode, key); + setEndFromToken(keyNode, key); + + // match an attribute + if (tokens.nextIf(Types.ASSIGNMENT)) { + const start = tokens.expect(Types.STRING_START); + let canBeString = true, + nodes = [], + token; + while (!tokens.test(Types.STRING_END)) { + if ( + canBeString && + (token = tokens.nextIf(Types.STRING)) + ) { + nodes[nodes.length] = createNode( + n.StringLiteral, + token, + token.text, + ); + canBeString = false; + } else if ( + (token = tokens.nextIf(Types.EXPRESSION_START)) + ) { + nodes[nodes.length] = this.matchExpression(); + tokens.expect(Types.EXPRESSION_END); + canBeString = true; + } else { + break; + } + } + tokens.expect(Types.STRING_END); + if (!nodes.length) { + nodes.push(createNode(n.StringLiteral, start, '')); + } + + let expr = nodes[0]; + for (let i = 1, len = nodes.length; i < len; i++) { + const { line, column } = expr.loc.start; + expr = new n.BinaryConcatExpression(expr, nodes[i]); + expr.loc.start.line = line; + expr.loc.start.column = column; + copyEnd(expr, expr.right); + } + const attr = new n.Attribute(keyNode, expr); + copyStart(attr, keyNode); + copyEnd(attr, expr); + element.attributes.push(attr); + } else { + element.attributes.push( + copyLoc(new n.Attribute(keyNode), keyNode), + ); + } + } else if (tokens.nextIf(Types.EXPRESSION_START)) { + element.attributes.push(this.matchExpression()); + tokens.expect(Types.EXPRESSION_END); + } else { + this.error({ + title: 'Invalid token', + pos: tokens.la(0).pos, + advice: + 'A tag must consist of attributes or expressions. Twig Tags are not allowed.', + }); + } + } + } + + error(options) { + this.tokens.error(options.title, options.pos, options.advice); + } + + matchTag() { + let tokens = this.tokens, + tag = tokens.expect(Types.SYMBOL), + parser = this[TAG][tag.text]; + if (!parser) { + tokens.error( + `Unknown tag "${tag.text}"`, + tag.pos, + `Expected a known tag such as\n- ${Object.getOwnPropertyNames( + this[TAG], + ).join('\n- ')}`, + tag.length, + ); + } + return parser.parse(this, tag); + } + + matchExpression(precedence = 0) { + let expr = this.getPrimary(), + tokens = this.tokens, + token, + op; + while ( + (token = tokens.la(0)) && + token.type !== Types.EOF && + (op = this.getBinaryOperator(token)) && + op.precedence >= precedence + ) { + const opToken = tokens.next(); // consume the operator + if (op.parse) { + expr = op.parse(this, opToken, expr); + } else { + const expr1 = this.matchExpression( + op.associativity === LEFT + ? op.precedence + 1 + : op.precedence, + ); + expr = op.createNode(token, expr, expr1); + } + token = tokens.la(0); + } + + return precedence === 0 ? this.matchConditionalExpression(expr) : expr; + } + + getPrimary() { + let tokens = this.tokens, + token = tokens.la(0); + if (this.isUnary(token)) { + const op = this[UNARY][token.text]; + tokens.next(); // consume operator + const expr = this.matchExpression(op.precedence); + return this.matchPostfixExpression(op.createNode(token, expr)); + } else if (tokens.test(Types.LPAREN)) { + tokens.next(); // consume '(' + const expr = this.matchExpression(); + tokens.expect(Types.RPAREN); + return this.matchPostfixExpression(expr); + } + + return this.matchPrimaryExpression(); + } + + matchPrimaryExpression() { + let tokens = this.tokens, + token = tokens.la(0), + node; + switch (token.type) { + case Types.NULL: + node = createNode(n.NullLiteral, tokens.next()); + break; + case Types.FALSE: + node = createNode(n.BooleanLiteral, tokens.next(), false); + break; + case Types.TRUE: + node = createNode(n.BooleanLiteral, tokens.next(), true); + break; + case Types.SYMBOL: + tokens.next(); + if (tokens.test(Types.LPAREN)) { + // SYMBOL '(' arguments* ')' + node = new n.CallExpression( + createNode(n.Identifier, token, token.text), + this.matchArguments(), + ); + copyStart(node, node.callee); + setEndFromToken(node, tokens.la(-1)); // ')' + } else { + node = createNode(n.Identifier, token, token.text); + } + break; + case Types.NUMBER: + node = createNode( + n.NumericLiteral, + token, + Number(tokens.next()), + ); + break; + case Types.STRING_START: + node = this.matchStringExpression(); + break; + // potentially missing: OPERATOR type + default: + if (token.type === Types.LBRACE) { + node = this.matchArray(); + } else if (token.type === Types.LBRACKET) { + node = this.matchMap(); + } else { + this.error({ + title: + 'Unexpected token "' + + token.type + + '" of value "' + + token.text + + '"', + pos: token.pos, + }); + } + break; + } + + return this.matchPostfixExpression(node); + } + + matchStringExpression() { + let tokens = this.tokens, + nodes = [], + canBeString = true, + token, + stringStart, + stringEnd; + stringStart = tokens.expect(Types.STRING_START); + while (!tokens.test(Types.STRING_END)) { + if (canBeString && (token = tokens.nextIf(Types.STRING))) { + nodes[nodes.length] = createNode( + n.StringLiteral, + token, + token.text, + ); + canBeString = false; + } else if ((token = tokens.nextIf(Types.INTERPOLATION_START))) { + nodes[nodes.length] = this.matchExpression(); + tokens.expect(Types.INTERPOLATION_END); + canBeString = true; + } else { + break; + } + } + stringEnd = tokens.expect(Types.STRING_END); + + if (!nodes.length) { + return setEndFromToken( + createNode(n.StringLiteral, stringStart, ''), + stringEnd, + ); + } + + let expr = nodes[0]; + for (let i = 1, len = nodes.length; i < len; i++) { + const { line, column } = expr.loc.start; + expr = new n.BinaryConcatExpression(expr, nodes[i]); + expr.loc.start.line = line; + expr.loc.start.column = column; + copyEnd(expr, expr.right); + } + + return expr; + } + + matchConditionalExpression(test: Node) { + let tokens = this.tokens, + condition = test, + consequent, + alternate; + while (tokens.nextIf(Types.QUESTION_MARK)) { + if (!tokens.nextIf(Types.COLON)) { + consequent = this.matchExpression(); + if (tokens.nextIf(Types.COLON)) { + alternate = this.matchExpression(); + } else { + alternate = null; + } + } else { + consequent = null; + alternate = this.matchExpression(); + } + const { line, column } = condition.loc.start; + condition = new n.ConditionalExpression( + condition, + consequent, + alternate, + ); + condition.loc.start = { line, column }; + copyEnd(condition, alternate || consequent); + } + return condition; + } + + matchArray() { + let tokens = this.tokens, + array = new n.ArrayExpression(), + start = tokens.expect(Types.LBRACE); + setStartFromToken(array, start); + while (!tokens.test(Types.RBRACE) && !tokens.test(Types.EOF)) { + array.elements.push(this.matchExpression()); + if (!tokens.test(Types.RBRACE)) { + tokens.expect(Types.COMMA); + // support trailing commas + if (tokens.test(Types.RBRACE)) { + break; + } + } + } + setEndFromToken(array, tokens.expect(Types.RBRACE)); + return array; + } + + matchMap() { + let tokens = this.tokens, + token, + obj = new n.ObjectExpression(), + startToken = tokens.expect(Types.LBRACKET); + setStartFromToken(obj, startToken); + while (!tokens.test(Types.RBRACKET) && !tokens.test(Types.EOF)) { + let computed = false, + key, + value; + if (tokens.test(Types.STRING_START)) { + key = this.matchStringExpression(); + if (!n.is('StringLiteral', key)) { + computed = true; + } + } else if ((token = tokens.nextIf(Types.SYMBOL))) { + key = createNode(n.Identifier, token, token.text); + } else if ((token = tokens.nextIf(Types.NUMBER))) { + key = createNode(n.NumericLiteral, token, Number(token.text)); + } else if (tokens.test(Types.LPAREN)) { + key = this.matchExpression(); + computed = true; + } else { + this.error({ + title: 'Invalid map key', + pos: tokens.la(0).pos, + advice: + 'Key must be a string, symbol or a number but was ' + + tokens.next(), + }); + } + tokens.expect(Types.COLON); + value = this.matchExpression(); + const prop = new n.ObjectProperty(key, value, computed); + copyStart(prop, key); + copyEnd(prop, value); + obj.properties.push(prop); + if (!tokens.test(Types.RBRACKET)) { + tokens.expect(Types.COMMA); + // support trailing comma + if (tokens.test(Types.RBRACKET)) { + break; + } + } + } + setEndFromToken(obj, tokens.expect(Types.RBRACKET)); + return obj; + } + + matchPostfixExpression(expr) { + const tokens = this.tokens; + let node = expr; + while (!tokens.test(Types.EOF)) { + if (tokens.test(Types.DOT) || tokens.test(Types.LBRACE)) { + node = this.matchSubscriptExpression(node); + } else if (tokens.test(Types.PIPE)) { + tokens.next(); + node = this.matchFilterExpression(node); + } else { + break; + } + } + + return node; + } + + matchSubscriptExpression(node) { + let tokens = this.tokens, + op = tokens.next(); + if (op.type === Types.DOT) { + let token = tokens.next(), + computed = false, + property; + if (token.type === Types.SYMBOL) { + property = createNode(n.Identifier, token, token.text); + } else if (token.type === Types.NUMBER) { + property = createNode( + n.NumericLiteral, + token, + Number(token.text), + ); + computed = true; + } else { + this.error({ + title: 'Invalid token', + pos: token.pos, + advice: + 'Expected number or symbol, found ' + + token + + ' instead', + }); + } + + const memberExpr = new n.MemberExpression(node, property, computed); + copyStart(memberExpr, node); + copyEnd(memberExpr, property); + if (tokens.test(Types.LPAREN)) { + const callExpr = new n.CallExpression( + memberExpr, + this.matchArguments(), + ); + copyStart(callExpr, memberExpr); + setEndFromToken(callExpr, tokens.la(-1)); + return callExpr; + } + return memberExpr; + } else { + let arg, start; + if (tokens.test(Types.COLON)) { + // slice + tokens.next(); + start = null; + } else { + arg = this.matchExpression(); + if (tokens.test(Types.COLON)) { + start = arg; + arg = null; + tokens.next(); + } + } + + if (arg) { + return setEndFromToken( + copyStart(new n.MemberExpression(node, arg, true), node), + tokens.expect(Types.RBRACE), + ); + } else { + // slice + const result = new n.SliceExpression( + node, + start, + tokens.test(Types.RBRACE) ? null : this.matchExpression(), + ); + copyStart(result, node); + setEndFromToken(result, tokens.expect(Types.RBRACE)); + return result; + } + } + } + + matchFilterExpression(node) { + let tokens = this.tokens, + target = node; + while (!tokens.test(Types.EOF)) { + let token = tokens.expect(Types.SYMBOL), + name = createNode(n.Identifier, token, token.text), + args; + if (tokens.test(Types.LPAREN)) { + args = this.matchArguments(); + } else { + args = []; + } + const newTarget = new n.FilterExpression(target, name, args); + copyStart(newTarget, target); + if (newTarget.arguments.length) { + copyEnd( + newTarget, + newTarget.arguments[newTarget.arguments.length - 1], + ); + } else { + copyEnd(newTarget, target); + } + target = newTarget; + + if (!tokens.test(Types.PIPE) || tokens.test(Types.EOF)) { + break; + } + + tokens.next(); // consume '|' + } + return target; + } + + matchArguments() { + let tokens = this.tokens, + args = []; + tokens.expect(Types.LPAREN); + while (!tokens.test(Types.RPAREN) && !tokens.test(Types.EOF)) { + if ( + tokens.test(Types.SYMBOL) && + tokens.lat(1) === Types.ASSIGNMENT + ) { + const name = tokens.next(); + tokens.next(); + const value = this.matchExpression(); + const arg = new n.NamedArgumentExpression( + createNode(n.Identifier, name, name.text), + value, + ); + copyEnd(arg, value); + args.push(arg); + } else { + args.push(this.matchExpression()); + } + + if (!tokens.test(Types.COMMA)) { + tokens.expect(Types.RPAREN); + return args; + } + tokens.expect(Types.COMMA); + } + tokens.expect(Types.RPAREN); + return args; + } +} diff --git a/src/melody/melody-parser/TokenStream.js b/src/melody/melody-parser/TokenStream.js new file mode 100644 index 00000000..05427c11 --- /dev/null +++ b/src/melody/melody-parser/TokenStream.js @@ -0,0 +1,177 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + EOF_TOKEN, + ERROR, + ERROR_TABLE, + COMMENT, + WHITESPACE, + HTML_COMMENT, + TAG_START, + TAG_END, + EXPRESSION_START, + EXPRESSION_END, + TEXT, + STRING, +} from './TokenTypes'; +import trimEnd from 'lodash/trimEnd'; +import trimStart from 'lodash/trimStart'; +import codeFrame from 'melody-code-frame'; + +const TOKENS = Symbol(), + LENGTH = Symbol(); + +export default class TokenStream { + constructor( + lexer, + options = { ignoreComments: true, ignoreWhitespace: true }, + ) { + this.input = lexer; + this.index = 0; + this.options = options; + this[TOKENS] = getAllTokens(lexer, options); + this[LENGTH] = this[TOKENS].length; + + if ( + this[TOKENS].length && + this[TOKENS][this[TOKENS].length - 1].type === ERROR + ) { + const errorToken = this[TOKENS][this[TOKENS].length - 1]; + this.error( + errorToken.message, + errorToken.pos, + errorToken.advice, + errorToken.endPos.index - errorToken.pos.index || 1, + ); + } + } + + la(offset) { + var index = this.index + offset; + return index < this[LENGTH] ? this[TOKENS][index] : EOF_TOKEN; + } + + lat(offset) { + return this.la(offset).type; + } + + test(type, text) { + const token = this.la(0); + return token.type === type && (!text || token.text === text); + } + + next() { + if (this.index === this[LENGTH]) { + return EOF_TOKEN; + } + const token = this[TOKENS][this.index]; + this.index++; + return token; + } + + nextIf(type, text) { + if (this.test(type, text)) { + return this.next(); + } + return false; + } + + expect(type, text) { + const token = this.la(0); + if (token.type === type && (!text || token.text === text)) { + return this.next(); + } + this.error( + 'Invalid Token', + token.pos, + `Expected ${ERROR_TABLE[type] || + type || + text} but found ${ERROR_TABLE[token.type] || + token.type || + token.text} instead.`, + token.length, + ); + } + + error(message, pos, advice, length = 1) { + let errorMessage = `ERROR: ${message}\n`; + errorMessage += codeFrame({ + rawLines: this.input.source, + lineNumber: pos.line, + colNumber: pos.column, + length, + tokens: getAllTokens(this.input, { + ignoreWhitespace: false, + ignoreComments: false, + ignoreHtmlComments: false, + }), + }); + if (advice) { + errorMessage += '\n\n' + advice; + } + throw new Error(errorMessage); + } +} + +function getAllTokens(lexer, options) { + let token, + tokens = [], + acceptWhitespaceControl = false, + trimNext = false; + while ((token = lexer.next()) !== EOF_TOKEN) { + const shouldTrimNext = trimNext; + trimNext = false; + if (acceptWhitespaceControl) { + switch (token.type) { + case EXPRESSION_START: + case TAG_START: + if (token.text[token.text.length - 1] === '-') { + tokens[tokens.length - 1].text = trimEnd( + tokens[tokens.length - 1].text, + ); + } + break; + case EXPRESSION_END: + case TAG_END: + if (token.text[0] === '-') { + trimNext = true; + } + break; + case COMMENT: + if (tokens[tokens.length - 1].type === TEXT) { + tokens[tokens.length - 1].text = trimEnd(tokens.text); + } + trimNext = true; + break; + } + } + if (shouldTrimNext && (token.type === TEXT || token.type === STRING)) { + token.text = trimStart(token.text); + } + if ( + (token.type !== COMMENT || !options.ignoreComments) && + (token.type !== WHITESPACE || !options.ignoreWhitespace) && + (token.type !== HTML_COMMENT || !options.ignoreHtmlComments) + ) { + tokens[tokens.length] = token; + } + acceptWhitespaceControl = true; + if (token.type === ERROR) { + return tokens; + } + } + return tokens; +} diff --git a/src/melody/melody-parser/TokenTypes.js b/src/melody/melody-parser/TokenTypes.js new file mode 100644 index 00000000..639c1127 --- /dev/null +++ b/src/melody/melody-parser/TokenTypes.js @@ -0,0 +1,73 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export const EXPRESSION_START = 'expressionStart'; +export const EXPRESSION_END = 'expressionEnd'; +export const TAG_START = 'tagStart'; +export const TAG_END = 'tagEnd'; +export const INTERPOLATION_START = 'interpolationStart'; +export const INTERPOLATION_END = 'interpolationEnd'; +export const STRING_START = 'stringStart'; +export const STRING_END = 'stringEnd'; +export const COMMENT = 'comment'; +export const WHITESPACE = 'whitespace'; +export const HTML_COMMENT = 'htmlComment'; +export const TEXT = 'text'; +export const ENTITY = 'entity'; +export const SYMBOL = 'symbol'; +export const STRING = 'string'; +export const OPERATOR = 'operator'; +export const TRUE = 'true'; +export const FALSE = 'false'; +export const NULL = 'null'; +export const LBRACE = '['; +export const RBRACE = ']'; +export const LPAREN = '('; +export const RPAREN = ')'; +export const LBRACKET = '{'; +export const RBRACKET = '}'; +export const COLON = ':'; +export const COMMA = ','; +export const DOT = '.'; +export const PIPE = '|'; +export const QUESTION_MARK = '?'; +export const ASSIGNMENT = '='; +export const ELEMENT_START = '<'; +export const SLASH = '/'; +export const ELEMENT_END = '>'; +export const NUMBER = 'number'; +export const EOF = 'EOF'; +export const ERROR = 'ERROR'; +export const EOF_TOKEN = { + type: EOF, + pos: { + index: -1, + line: -1, + pos: -1, + }, + end: -1, + length: 0, + source: null, + text: '', +}; + +export const ERROR_TABLE = { + [EXPRESSION_END]: 'expression end "}}"', + [EXPRESSION_START]: 'expression start "{{"', + [TAG_START]: 'tag start "{%"', + [TAG_END]: 'tag end "%}"', + [INTERPOLATION_START]: 'interpolation start "#{"', + [INTERPOLATION_END]: 'interpolation end "}"', +}; diff --git a/src/melody/melody-parser/elementInfo.js b/src/melody/melody-parser/elementInfo.js new file mode 100644 index 00000000..a3f3fc6d --- /dev/null +++ b/src/melody/melody-parser/elementInfo.js @@ -0,0 +1,43 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// https://www.w3.org/TR/html5/syntax.html#void-elements +export const voidElements = { + area: true, + base: true, + br: true, + col: true, + embed: true, + hr: true, + img: true, + input: true, + keygen: true, + link: true, + meta: true, + param: true, + source: true, + track: true, + wbr: true, +}; + +export const rawTextElements = { + script: true, + style: true, +}; + +export const escapableRawTextElements = { + textarea: true, + title: true, +}; diff --git a/src/melody/melody-parser/index.js b/src/melody/melody-parser/index.js new file mode 100644 index 00000000..25a28e97 --- /dev/null +++ b/src/melody/melody-parser/index.js @@ -0,0 +1,52 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Parser from './Parser'; +import TokenStream from './TokenStream'; +import * as Types from './TokenTypes'; +import Lexer from './Lexer'; +import { EOF, CharStream } from './CharStream'; +import { LEFT, RIGHT } from './Associativity'; +import { + setStartFromToken, + setEndFromToken, + copyStart, + copyEnd, + copyLoc, + createNode, +} from './util'; + +function parse(code) { + const p = new Parser(new TokenStream(new Lexer(new CharStream(code)))); + return p.parse(); +} + +export { + Parser, + TokenStream, + Lexer, + EOF, + CharStream, + LEFT, + RIGHT, + parse, + setStartFromToken, + setEndFromToken, + copyStart, + copyEnd, + copyLoc, + createNode, + Types, +}; diff --git a/src/melody/melody-parser/util.js b/src/melody/melody-parser/util.js new file mode 100644 index 00000000..98acc03c --- /dev/null +++ b/src/melody/melody-parser/util.js @@ -0,0 +1,56 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export function setStartFromToken(node, { pos: { index, line, column } }) { + node.loc.start = { line, column, index }; + return node; +} + +export function setEndFromToken(node, { pos: { line, column }, end }) { + node.loc.end = { line, column, index: end }; + return node; +} + +export function copyStart(node, { loc: { start: { line, column, index } } }) { + node.loc.start.line = line; + node.loc.start.column = column; + node.loc.start.index = index; + return node; +} + +export function copyEnd(node, end) { + node.loc.end.line = end.loc.end.line; + node.loc.end.column = end.loc.end.column; + node.loc.end.index = end.loc.end.index; + return node; +} + +export function copyLoc(node, { loc: { start, end } }) { + node.loc.start.line = start.line; + node.loc.start.column = start.column; + node.loc.start.index = start.index; + node.loc.end.line = end.line; + node.loc.end.column = end.column; + node.loc.end.index = end.index; + return node; +} + +export function createNode(Type, token, ...args) { + return setEndFromToken(setStartFromToken(new Type(...args), token), token); +} + +export function startNode(Type, token, ...args) { + return setStartFromToken(new Type(...args), token); +} diff --git a/src/melody/melody-traverse/Binding.js b/src/melody/melody-traverse/Binding.js new file mode 100644 index 00000000..48a09a76 --- /dev/null +++ b/src/melody/melody-traverse/Binding.js @@ -0,0 +1,61 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export class Binding { + constructor(identifier, scope, path, kind = 'global') { + this.identifier = identifier; + this.scope = scope; + this.path = path; + this.kind = kind; + + this.referenced = false; + this.references = 0; + this.referencePaths = []; + this.definitionPaths = []; + this.shadowedBinding = null; + this.contextual = false; + + this.data = Object.create(null); + } + + getData(key) { + return this.data[key]; + } + + setData(key, value) { + this.data[key] = value; + } + + reference(path) { + this.referenced = true; + this.references++; + this.referencePaths.push(path); + } + + // dereference(path) { + // if (path) { + // this.referencePaths.splice(this.referencePaths.indexOf(path), 1); + // } + // this.references--; + // this.referenced = !!this.references; + // } + + getRootDefinition() { + if (this.shadowedBinding) { + return this.shadowedBinding.getRootDefinition(); + } + return this; + } +} diff --git a/src/melody/melody-traverse/Path.js b/src/melody/melody-traverse/Path.js new file mode 100644 index 00000000..81c6231b --- /dev/null +++ b/src/melody/melody-traverse/Path.js @@ -0,0 +1,403 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PATH_CACHE_KEY } from 'melody-types'; +import { Node, is } from 'melody-types'; +import { visit } from './traverse'; +import Scope from './Scope'; + +export default class Path { + //region Path creation + constructor(parent) { + this.parent = parent; + + this.inList = false; + this.listKey = null; + this.parentKey = null; + this.container = null; + this.parentPath = null; + + this.key = null; + this.node = null; + this.type = null; + + this.state = null; + + this.data = Object.create(null); + this.contexts = []; + this.scope = null; + this.visitor = null; + + this.shouldSkip = false; + this.shouldStop = false; + this.removed = false; + } + + static get({ parentPath, parent, container, listKey, key }): Path { + const targetNode = container[key], + paths = + (parent && parent[PATH_CACHE_KEY]) || + (parent ? (parent[PATH_CACHE_KEY] = []) : []); + let path; + + for (let i = 0, len = paths.length; i < len; i++) { + const candidate = paths[i]; + if (candidate.node === targetNode) { + path = candidate; + break; + } + } + + if (!path) { + path = new Path(parent); + } + + path.inList = !!listKey; + path.listKey = listKey; + path.parentKey = listKey || key; + path.container = container; + path.parentPath = parentPath || path.parentPath; + + path.key = key; + path.node = path.container[path.key]; + path.type = path.node && path.node.type; + + if (!path.node) { + /*eslint no-console: off*/ + console.log( + 'Path has no node ' + path.parentKey + ' > ' + path.key, + ); + } + paths.push(path); + + return path; + } + + //endregion + + //region Generic data + setData(key: string, val: any): any { + return (this.data[key] = val); + } + + getData(key: string, def?: any): any { + const val = this.data[key]; + if (!val && def) { + return (this.data[key] = def); + } + return val; + } + //endregion + + //region Context + pushContext(context) { + this.contexts.push(context); + this.setContext(context); + } + + popContext() { + this.contexts.pop(); + this.setContext(this.contexts[this.contexts.length - 1]); + } + + setContext(context) { + this.shouldSkip = false; + this.shouldStop = false; + this.removed = false; + //this.skipKeys = {}; + + if (context) { + this.context = context; + this.state = context.state; + this.visitor = context.visitor; + } + + this.setScope(); + return this; + } + + getScope(scope: Scope) { + if (Node.isScope(this.node)) { + if (this.node.type === 'BlockStatement') { + return Scope.get(this, scope.getRootScope()); + } + return Scope.get(this, scope); + } + return scope; + } + + setScope() { + let target = this.context && this.context.scope; + + if (!target) { + let path = this.parentPath; + while (path && !target) { + target = path.scope; + path = path.parentPath; + } + } + + this.scope = this.getScope(target); + } + + visit(): boolean { + if (!this.node) { + return false; + } + + if (call(this, 'enter') || this.shouldSkip) { + return this.shouldStop; + } + + visit(this.node, this.visitor, this.scope, this.state, this); + + call(this, 'exit'); + + return this.shouldStop; + } + + skip() { + this.shouldSkip = true; + } + + stop() { + this.shouldStop = true; + this.shouldSkip = true; + } + + resync() { + if (this.removed) { + return; + } + + if (this.parentPath) { + this.parent = this.parentPath.node; + } + + if (this.parent && this.inList) { + const newContainer = this.parent[this.listKey]; + if (this.container !== newContainer) { + this.container = newContainer || null; + } + } + + if (this.container && this.node !== this.container[this.key]) { + this.key = null; + if (Array.isArray(this.container)) { + let i, len; + for (i = 0, len = this.container.length; i < len; i++) { + if (this.container[i] === this.node) { + this.setKey(i); + break; + } + } + } else { + let key; + for (key in this.container) { + if (this.container[key] === this.node) { + this.setKey(key); + break; + } + } + } + } + } + + setKey(key) { + this.key = key; + this.node = this.container[this.key]; + this.type = this.node && this.node.type; + } + + requeue(path = this) { + if (path.removed) { + return; + } + + for (const context of this.contexts) { + context.maybeQueue(path); + } + } + //endregion + + //region Modification + replaceWith(value) { + this.resync(); + + const replacement = value instanceof Path ? value.node : value; + + if (this.node === replacement) { + return; + } + + replaceWith(this, replacement); + this.type = replacement.type; + this.resync(); + this.setScope(); + this.requeue(); + } + + replaceWithJS(replacement) { + this.resync(); + replaceWith(this, replacement); + this.type = replacement.type; + this.resync(); + this.setScope(); + } + + replaceWithMultipleJS(...replacements) { + this.resync(); + + if (!this.container) { + throw new Error('Path does not have a container'); + } + if (!Array.isArray(this.container)) { + throw new Error('Container of path is not an array'); + } + + this.container.splice(this.key, 1, ...replacements); + this.resync(); + this.updateSiblingKeys(this.key, replacements.length - 1); + markRemoved(this); + //this.node = replacements[0]; + } + + remove() { + this.resync(); + + if (Array.isArray(this.container)) { + this.container.splice(this.key, 1); + this.updateSiblingKeys(this.key, -1); + } else { + replaceWith(this, null); + } + + markRemoved(this); + } + + updateSiblingKeys(fromIndex, incrementBy) { + if (!this.parent) { + return; + } + + const paths: Array = this.parent[PATH_CACHE_KEY]; + for (const path of paths) { + if (path.key >= fromIndex) { + path.key += incrementBy; + } + } + } + //endregion + + is(type: String) { + return is(this.node, type); + } + + findParentPathOfType(type: String) { + let path = this.parentPath; + while (path && !path.is(type)) { + path = path.parentPath; + } + return path && path.type === type ? path : null; + } + + get(key) { + let parts: Array = key.split('.'), + context = this.context; + if (parts.length === 1) { + let node = this.node, + container = node[key]; + if (Array.isArray(container)) { + return container.map((_, i) => + Path.get({ + listKey: key, + parentPath: this, + parent: node, + container, + key: i, + }).setContext(context), + ); + } else { + return Path.get({ + parentPath: this, + parent: node, + container: node, + key, + }).setContext(context); + } + } else { + let path = this; + for (const part of parts) { + if (Array.isArray(path)) { + path = path[part]; + } else { + path = path.get(part); + } + } + return path; + } + } +} + +function markRemoved(path) { + path.shouldSkip = true; + path.removed = true; + path.node = null; +} + +function replaceWith(path, node) { + if (!path.container) { + throw new Error('Path does not have a container'); + } + + path.node = path.container[path.key] = node; +} + +function call(path, key): boolean { + if (!path.node) { + return false; + } + + const visitor = path.visitor[path.node.type]; + if (!visitor || !visitor[key]) { + return false; + } + + const fns: Array = visitor[key]; + for (let i = 0, len = fns.length; i < len; i++) { + const fn = fns[i]; + if (!fn) { + continue; + } + + const node = path.node; + if (!node) { + return true; + } + + fn.call(path.state, path, path.state); + + // node has been replaced, requeue + if (path.node !== node) { + return true; + } + + if (path.shouldStop || path.shouldSkip || path.removed) { + return true; + } + } + + return false; +} diff --git a/src/melody/melody-traverse/Scope.js b/src/melody/melody-traverse/Scope.js new file mode 100644 index 00000000..62badf31 --- /dev/null +++ b/src/melody/melody-traverse/Scope.js @@ -0,0 +1,183 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Binding } from './Binding'; +import type Path from './Path'; +const CACHE_KEY = Symbol(); +let uid = 0; + +export default class Scope { + constructor(path: Path, parentScope?: Scope) { + this.uid = uid++; + this.parent = parentScope; + + this.parentBlock = path.parent; + this.block = path.node; + this.path = path; + + this.references = Object.create(null); + this.bindings = Object.create(null); + this.globals = Object.create(null); + this.uids = Object.create(null); + this.escapesContext = false; + this._contextName = null; + this.mutated = false; + //this.contextName = parentScope && parentScope.contextName || '_context'; + } + + set contextName(val) { + this._contextName = val; + } + + get contextName() { + if (this._contextName) { + return this._contextName; + } + if (this.parent) { + return this.parent.contextName || '_context'; + } + return '_context'; + } + + static get(path: Path, parentScope?: Scope) { + if (parentScope && parentScope.block == path.node) { + return parentScope; + } + + const cached = getCache(path.node); + if (cached) { + return cached; + } + + const scope = new Scope(path, parentScope); + path.node[CACHE_KEY] = scope; + return scope; + } + + get needsSubContext() { + return this.escapesContext && this.hasCustomBindings; + } + + get hasCustomBindings() { + return !!Object.keys(this.bindings).length; + } + + getBinding(name: string) { + let scope = this; + + do { + const binding = scope.getOwnBinding(name); + if (binding) { + return binding; + } + if (scope.path.is('RootScope')) { + return; + } + } while ((scope = scope.parent)); + } + + getOwnBinding(name: string) { + return this.bindings[name]; + } + + hasOwnBinding(name: string) { + return !!this.getOwnBinding(name); + } + + hasBinding(name: string) { + return !name + ? false + : !!(this.hasOwnBinding(name) || this.parentHasBinding(name)); + } + + getRootScope() { + let scope = this; + while (scope.parent) { + scope = scope.parent; + } + return scope; + } + + registerBinding(name: string, path: Path = null, kind: string = 'context') { + let scope = this; + if (kind === 'global' && path === null) { + scope = this.getRootScope(); + } else if (kind === 'const') { + while (scope.parent) { + scope = scope.parent; + if (scope.path.is('RootScope')) { + break; + } + } + } + // todo identify if we need to be able to differentiate between binding kinds + // if (scope.bindings[name]) { + // todo: warn about colliding binding or fix it + // } + if (this.path.state) { + this.path.state.markIdentifier(name); + } + return (scope.bindings[name] = new Binding(name, this, path, kind)); + } + + reference(name: string, path: Path) { + let binding = this.getBinding(name); + if (!binding) { + binding = this.registerBinding(name); + } + binding.reference(path); + } + + parentHasBinding(name: string) { + return this.parent && this.parent.hasBinding(name); + } + + generateUid(nameHint: string = 'temp') { + const name = toIdentifier(nameHint); + + let uid, + i = 0; + do { + uid = generateUid(name, i); + i++; + } while (this.hasBinding(uid)); + + return uid; + } +} + +function getCache(node) { + return node[CACHE_KEY]; +} + +function toIdentifier(nameHint) { + let name = nameHint + ''; + name = name.replace(/[^a-zA-Z0-9$_]/g, ''); + + name = name.replace(/^[-0-9]+/, ''); + name = name.replace(/[-\s]+(.)?/, function(match, c) { + return c ? c.toUpperCase() : ''; + }); + + name = name.replace(/^_+/, '').replace(/[0-9]+$/, ''); + return name; +} + +function generateUid(name, i) { + if (i > 0) { + return `_${name}$${i}`; + } + return `_${name}`; +} diff --git a/src/melody/melody-traverse/TraversalContext.js b/src/melody/melody-traverse/TraversalContext.js new file mode 100644 index 00000000..35dc7682 --- /dev/null +++ b/src/melody/melody-traverse/TraversalContext.js @@ -0,0 +1,151 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Path from './Path'; + +export default class TraversalContext { + constructor(scope, visitor, state, parentPath) { + this.parentPath = parentPath; + this.scope = scope; + this.state = state; + this.visitor = visitor; + + this.queue = null; + this.priorityQueue = null; + } + + create(parent, container, key, listKey): Path { + return Path.get({ + parentPath: this.parentPath, + parent, + container, + key, + listKey, + }); + } + + shouldVisit(node): boolean { + const visitor = this.visitor; + + if (visitor[node.type]) { + return true; + } + + const keys: Array = node.visitorKeys; + // this node doesn't have any children + if (!keys || !keys.length) { + return false; + } + + let i, len; + for (i = 0, len = keys.length; i < len; i++) { + // check if some of its visitor keys have a value, + // if so, we need to visit it + if (node[keys[i]]) { + return true; + } + } + + return false; + } + + visit(node, key) { + var nodes = node[key]; + if (!nodes) { + return false; + } + + if (Array.isArray(nodes)) { + return this.visitMultiple(nodes, node, key); + } else { + return this.visitSingle(node, key); + } + } + + visitSingle(node, key): boolean { + if (this.shouldVisit(node[key])) { + return this.visitQueue([this.create(node, node, key)]); + } else { + return false; + } + } + + visitMultiple(container, parent, listKey) { + if (!container.length) { + return false; + } + + const queue = []; + + for (let i = 0, len = container.length; i < len; i++) { + const node = container[i]; + if (node && this.shouldVisit(node)) { + queue.push(this.create(parent, container, i, listKey)); + } + } + + return this.visitQueue(queue); + } + + visitQueue(queue: Array) { + this.queue = queue; + this.priorityQueue = []; + + let visited = [], + stop = false; + + for (const path of queue) { + path.resync(); + path.pushContext(this); + + if (visited.indexOf(path.node) >= 0) { + continue; + } + visited.push(path.node); + + if (path.visit()) { + stop = true; + break; + } + + if (this.priorityQueue.length) { + stop = this.visitQueue(this.priorityQueue); + this.priorityQueue = []; + this.queue = queue; + if (stop) { + break; + } + } + } + + for (const path of queue) { + path.popContext(); + } + + this.queue = null; + + return stop; + } + + maybeQueue(path, notPriority?: boolean) { + if (this.queue) { + if (notPriority) { + this.queue.push(path); + } else { + this.priorityQueue.push(path); + } + } + } +} diff --git a/src/melody/melody-traverse/index.js b/src/melody/melody-traverse/index.js new file mode 100644 index 00000000..24d2a14e --- /dev/null +++ b/src/melody/melody-traverse/index.js @@ -0,0 +1,21 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Path from './Path'; +import Scope from './Scope'; +export { Scope, Path }; +export * from './Scope'; +export { merge, explode } from './visitors'; +export { traverse, visit } from './traverse'; diff --git a/src/melody/melody-traverse/traverse.js b/src/melody/melody-traverse/traverse.js new file mode 100644 index 00000000..7d31c973 --- /dev/null +++ b/src/melody/melody-traverse/traverse.js @@ -0,0 +1,47 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { explode } from './visitors'; +import TraversalContext from './TraversalContext'; + +export function traverse( + parentNode, + visitor: Object, + scope?: Object, + state?: Object = {}, + parentPath?: Object, +) { + if (!parentNode) { + return; + } + + explode(visitor); + visit(parentNode, visitor, scope, state, parentPath); +} + +export function visit(node, visitor, scope, state, parentPath) { + const keys: Array = node.visitorKeys; + if (!keys || !keys.length) { + return; + } + + const context = new TraversalContext(scope, visitor, state, parentPath); + for (let i = 0, len = keys.length; i < len; i++) { + const key = keys[i]; + if (context.visit(node, key)) { + return; + } + } +} diff --git a/src/melody/melody-traverse/visitors.js b/src/melody/melody-traverse/visitors.js new file mode 100644 index 00000000..0f40f706 --- /dev/null +++ b/src/melody/melody-traverse/visitors.js @@ -0,0 +1,109 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ALIAS_TO_TYPE } from 'melody-types'; + +const EXPLODED = Symbol(); + +export function explode(visitor) { + if (visitor[EXPLODED]) { + return visitor; + } + visitor[EXPLODED] = true; + + for (const key of Object.getOwnPropertyNames(visitor)) { + // make sure all members are objects with enter and exit methods + let fns = visitor[key]; + if (typeof fns === 'function') { + fns = visitor[key] = { enter: fns }; + } + + // make sure enter and exit are arrays + if (fns.enter && !Array.isArray(fns.enter)) { + fns.enter = [fns.enter]; + } + if (fns.exit && !Array.isArray(fns.exit)) { + fns.exit = [fns.exit]; + } + } + + let j = 0; + const visitorKeys = Object.getOwnPropertyNames(visitor); + const visitorKeyLength = visitorKeys.length; + for (; j < visitorKeyLength; j++) { + const key = visitorKeys[j]; + // manage aliases + if (ALIAS_TO_TYPE[key]) { + let i = 0; + for ( + const types = ALIAS_TO_TYPE[key], len = types.length; + i < len; + i++ + ) { + const type = types[i]; + if (!visitor[type]) { + visitor[type] = { enter: [] }; + } + if (visitor[key].enter) { + visitor[type].enter.push(...visitor[key].enter); + } + if (visitor[key].exit) { + if (!visitor[type].exit) { + visitor[type].exit = []; + } + visitor[type].exit.push(...visitor[key].exit); + } + } + delete visitor[key]; + } + } +} + +export function merge(...visitors: Array) { + const rootVisitor = {}; + + let i = 0; + for (const len = visitors.length; i < len; i++) { + const visitor = visitors[i]; + explode(visitor); + + let j = 0; + const visitorTypes = Object.getOwnPropertyNames(visitor); + for ( + const numberOfTypes = visitorTypes.length; + j < numberOfTypes; + j++ + ) { + const key = visitorTypes[j]; + const visitorType = visitor[key]; + + if (!rootVisitor[key]) { + rootVisitor[key] = {}; + } + + const nodeVisitor = rootVisitor[key]; + nodeVisitor.enter = [].concat( + nodeVisitor.enter || [], + visitorType.enter || [], + ); + nodeVisitor.exit = [].concat( + nodeVisitor.exit || [], + visitorType.exit || [], + ); + } + } + + return rootVisitor; +} diff --git a/src/melody/melody-types/index.js b/src/melody/melody-types/index.js new file mode 100644 index 00000000..9a72c550 --- /dev/null +++ b/src/melody/melody-types/index.js @@ -0,0 +1,358 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as t from 'babel-types'; + +export const TYPE_MAP = Object.create(null); +export const ALIAS_TO_TYPE = Object.create(null); +export const PATH_CACHE_KEY = Symbol(); + +const IS_ALIAS_OF = Object.create(null); + +export class Node { + constructor() { + this.loc = { + source: null, + start: { line: 0, column: 0 }, + end: { line: 0, column: 0 }, + }; + this[PATH_CACHE_KEY] = []; + } + + toJSON() { + return Object.getOwnPropertyNames(this).reduce( + (acc, name) => { + if (name === 'loc' || name === 'parent') { + return acc; + } + const value = this[name]; + if (Array.isArray(value)) { + acc[name] = value.map(val => val.toJSON()); + } else { + acc[name] = value && value.toJSON ? value.toJSON() : value; + } + return acc; + }, + { + type: this.type, + }, + ); + } + + static registerType(type) { + if (Node['is' + type]) { + return; + } + + Node['is' + type] = function(node) { + return is(node, type); + }; + + // Node['assert' + type] = function(node) { + // if (!is(node, type)) { + // throw new Error('Expected node to be of type ' + type + ' but was ' + (node ? node.type : 'undefined') + ' instead'); + // } + // }; + } +} +Node.registerType('Scope'); + +export function is(node, type) { + return ( + node && + (node.type === type || + (IS_ALIAS_OF[type] && IS_ALIAS_OF[type][node.type]) || + t.is(node, type)) + ); +} + +export function visitor(type, ...fields: String) { + type.prototype.visitorKeys = fields; +} + +export function alias(type, ...aliases: String) { + type.prototype.aliases = aliases; + for (let i = 0, len = aliases.length; i < len; i++) { + const alias = aliases[i]; + if (!ALIAS_TO_TYPE[alias]) { + ALIAS_TO_TYPE[alias] = []; + } + ALIAS_TO_TYPE[alias].push(type.prototype.type); + if (!IS_ALIAS_OF[alias]) { + IS_ALIAS_OF[alias] = {}; + } + IS_ALIAS_OF[alias][type.prototype.type] = true; + Node.registerType(alias); + } +} + +export function type(Type, type: String) { + Type.prototype.type = type; + TYPE_MAP[type] = Type; + + Node.registerType(type); +} + +export class Fragment extends Node { + constructor(expression: Node) { + super(); + this.value = expression; + } +} +type(Fragment, 'Fragment'); +alias(Fragment, 'Statement'); +visitor(Fragment, 'value'); + +export class PrintExpressionStatement extends Node { + constructor(expression: Node) { + super(); + this.value = expression; + } +} +type(PrintExpressionStatement, 'PrintExpressionStatement'); +alias(PrintExpressionStatement, 'Statement', 'PrintStatement'); +visitor(PrintExpressionStatement, 'value'); + +export class PrintTextStatement extends Node { + constructor(text: StringLiteral) { + super(); + this.value = text; + } +} +type(PrintTextStatement, 'PrintTextStatement'); +alias(PrintTextStatement, 'Statement', 'PrintStatement'); +visitor(PrintTextStatement, 'value'); + +export class ConstantValue extends Node { + constructor(value) { + super(); + this.value = value; + } + + toString() { + return `Const(${this.value})`; + } +} +alias(ConstantValue, 'Expression', 'Literal', 'Immutable'); + +export class StringLiteral extends ConstantValue {} +type(StringLiteral, 'StringLiteral'); +alias(StringLiteral, 'Expression', 'Literal', 'Immutable'); + +export class NumericLiteral extends ConstantValue {} +type(NumericLiteral, 'NumericLiteral'); +alias(NumericLiteral, 'Expression', 'Literal', 'Immutable'); + +export class BooleanLiteral extends ConstantValue { + constructor(value) { + super(value); + } +} +type(BooleanLiteral, 'BooleanLiteral'); +alias(BooleanLiteral, 'Expression', 'Literal', 'Immutable'); + +export class NullLiteral extends ConstantValue { + constructor() { + super(null); + } +} +type(NullLiteral, 'NullLiteral'); +alias(NullLiteral, 'Expression', 'Literal', 'Immutable'); + +export class Identifier extends Node { + constructor(name) { + super(); + this.name = name; + } +} +type(Identifier, 'Identifier'); +alias(Identifier, 'Expression'); + +export class UnaryExpression extends Node { + constructor(operator: String, argument: Node) { + super(); + this.operator = operator; + this.argument = argument; + } +} +type(UnaryExpression, 'UnaryExpression'); +alias(UnaryExpression, 'Expression', 'UnaryLike'); +visitor(UnaryExpression, 'argument'); + +export class BinaryExpression extends Node { + constructor(operator: String, left: Node, right: Node) { + super(); + this.operator = operator; + this.left = left; + this.right = right; + } +} +type(BinaryExpression, 'BinaryExpression'); +alias(BinaryExpression, 'Binary', 'Expression'); +visitor(BinaryExpression, 'left', 'right'); + +export class BinaryConcatExpression extends BinaryExpression { + constructor(left: Node, right: Node) { + super('~', left, right); + } +} +type(BinaryConcatExpression, 'BinaryConcatExpression'); +alias(BinaryConcatExpression, 'BinaryExpression', 'Binary', 'Expression'); +visitor(BinaryConcatExpression, 'left', 'right'); + +export class ConditionalExpression extends Node { + constructor(test: Node, consequent: Node, alternate: Node) { + super(); + this.test = test; + this.consequent = consequent; + this.alternate = alternate; + } +} +type(ConditionalExpression, 'ConditionalExpression'); +alias(ConditionalExpression, 'Expression', 'Conditional'); +visitor(ConditionalExpression, 'test', 'consequent', 'alternate'); + +export class ArrayExpression extends Node { + constructor(elements = []) { + super(); + this.elements = elements; + } +} +type(ArrayExpression, 'ArrayExpression'); +alias(ArrayExpression, 'Expression'); +visitor(ArrayExpression, 'elements'); + +export class MemberExpression extends Node { + constructor(object: Node, property: Node, computed: boolean) { + super(); + this.object = object; + this.property = property; + this.computed = computed; + } +} +type(MemberExpression, 'MemberExpression'); +alias(MemberExpression, 'Expression', 'LVal'); +visitor(MemberExpression, 'object', 'property'); + +export class CallExpression extends Node { + constructor(callee: Node, args: Array) { + super(); + this.callee = callee; + this.arguments = args; + } +} +type(CallExpression, 'CallExpression'); +alias(CallExpression, 'Expression', 'FunctionInvocation'); +visitor(CallExpression, 'callee', 'arguments'); + +export class NamedArgumentExpression extends Node { + constructor(name: Identifier, value: Node) { + super(); + this.name = name; + this.value = value; + } +} +type(NamedArgumentExpression, 'NamedArgumentExpression'); +alias(NamedArgumentExpression, 'Expression'); +visitor(NamedArgumentExpression, 'name', 'value'); + +export class ObjectExpression extends Node { + constructor(properties: Array = []) { + super(); + this.properties = properties; + } +} +type(ObjectExpression, 'ObjectExpression'); +alias(ObjectExpression, 'Expression'); +visitor(ObjectExpression, 'properties'); + +export class ObjectProperty extends Node { + constructor(key: Node, value: Node, computed: boolean) { + super(); + this.key = key; + this.value = value; + this.computed = computed; + } +} +type(ObjectProperty, 'ObjectProperty'); +alias(ObjectProperty, 'Property', 'ObjectMember'); +visitor(ObjectProperty, 'key', 'value'); + +export class SequenceExpression extends Node { + constructor(expressions: Array = []) { + super(); + this.expressions = expressions; + } + + add(child: Node) { + this.expressions.push(child); + this.loc.end = child.loc.end; + } +} +type(SequenceExpression, 'SequenceExpression'); +alias(SequenceExpression, 'Expression', 'Scope'); +visitor(SequenceExpression, 'expressions'); + +export class SliceExpression extends Node { + constructor(target: Node, start: Node, end: Node) { + super(); + this.target = target; + this.start = start; + this.end = end; + } +} +type(SliceExpression, 'SliceExpression'); +alias(SliceExpression, 'Expression'); +visitor(SliceExpression, 'source', 'start', 'end'); + +export class FilterExpression extends Node { + constructor(target: Node, name: Identifier, args: Array) { + super(); + this.target = target; + this.name = name; + this.arguments = args; + } +} +type(FilterExpression, 'FilterExpression'); +alias(FilterExpression, 'Expression'); +visitor(FilterExpression, 'target', 'arguments'); + +export class Element extends Node { + constructor(name: String) { + super(); + this.name = name; + this.attributes = []; + this.children = []; + this.selfClosing = false; + } +} +type(Element, 'Element'); +alias(Element, 'Expression'); +visitor(Element, 'attributes', 'children'); + +export class Attribute extends Node { + constructor(name: Node, value: Node = null) { + super(); + this.name = name; + this.value = value; + } + + isImmutable() { + return is(this.name, 'Identifier') && is(this.value, 'Immutable'); + } +} +type(Attribute, 'Attribute'); +visitor(Attribute, 'name', 'value'); From bf2c0d87298ccbc96bc6463381a87dae176e0877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Efe=20G=C3=BCrkan=20YALAMAN?= Date: Wed, 25 Oct 2017 10:21:20 +0200 Subject: [PATCH 02/27] Chore - Setup travis (#1) - Setup travis - Fix broken dependencies - Change prettier to use es5 as trailing comma strategy - Change min node version to 6 --- src/melody/melody-code-frame/index.js | 4 +- src/melody/melody-extension-core/operators.js | 18 +++--- .../parser/autoescape.js | 2 +- .../melody-extension-core/parser/block.js | 6 +- .../melody-extension-core/parser/embed.js | 2 +- .../melody-extension-core/parser/filter.js | 2 +- .../melody-extension-core/parser/for.js | 8 +-- .../melody-extension-core/parser/from.js | 2 +- src/melody/melody-extension-core/parser/if.js | 4 +- .../melody-extension-core/parser/import.js | 2 +- .../melody-extension-core/parser/macro.js | 4 +- .../melody-extension-core/parser/set.js | 2 +- src/melody/melody-extension-core/types.js | 6 +- .../melody-extension-core/visitors/filters.js | 58 +++++++++---------- .../melody-extension-core/visitors/for.js | 30 +++++----- .../visitors/functions.js | 24 ++++---- .../melody-extension-core/visitors/tests.js | 42 +++++++------- src/melody/melody-parser/Lexer.js | 8 +-- src/melody/melody-parser/Parser.js | 46 +++++++-------- src/melody/melody-parser/TokenStream.js | 8 +-- src/melody/melody-traverse/Path.js | 4 +- src/melody/melody-traverse/traverse.js | 2 +- src/melody/melody-traverse/visitors.js | 4 +- src/melody/melody-types/index.js | 2 +- 24 files changed, 145 insertions(+), 145 deletions(-) diff --git a/src/melody/melody-code-frame/index.js b/src/melody/melody-code-frame/index.js index 3e219859..a0245d7d 100644 --- a/src/melody/melody-code-frame/index.js +++ b/src/melody/melody-code-frame/index.js @@ -36,10 +36,10 @@ export default function({ rawLines, lineNumber, colNumber, length }) { //params.line = params.line.substring(0, colNumber) + chalk.underline(params.line.substring(colNumber, colNumber + length)) + params.line.substring(colNumber + length, params.line.length - 1); params.line += `\n${params.before}${repeat( ' ', - params.width, + params.width )}${params.after}${repeat(' ', colNumber)}${repeat( '^', - length, + length )}`; } diff --git a/src/melody/melody-extension-core/operators.js b/src/melody/melody-extension-core/operators.js index b4a65ae2..a475eba2 100644 --- a/src/melody/melody-extension-core/operators.js +++ b/src/melody/melody-extension-core/operators.js @@ -40,17 +40,17 @@ export const tests = []; export const UnaryNotExpression = createUnaryOperator( 'not', 'UnaryNotExpression', - 50, + 50 ); export const UnaryNeqExpression = createUnaryOperator( '-', 'UnaryNeqExpression', - 500, + 500 ); export const UnaryPosExpression = createUnaryOperator( '+', 'UnaryPosExpression', - 500, + 500 ); //endregion @@ -233,7 +233,7 @@ binaryOperators.push({ if (not) { return copyLoc( new UnaryNotExpression(testExpression), - testExpression, + testExpression ); } return testExpression; @@ -283,11 +283,11 @@ export const TestEvenExpression = createTest('even', 'TestEvenExpression'); export const TestOddExpression = createTest('odd', 'TestOddExpression'); export const TestDefinedExpression = createTest( 'defined', - 'TestDefinedExpression', + 'TestDefinedExpression' ); export const TestSameAsExpression = createTest( 'same as', - 'TestSameAsExpression', + 'TestSameAsExpression' ); tests.push({ text: 'sameas', @@ -305,7 +305,7 @@ tests.push({ }); export const TestDivisibleByExpression = createTest( 'divisible by', - 'TestDivisibleByExpression', + 'TestDivisibleByExpression' ); tests.push({ text: 'divisibleby', @@ -316,12 +316,12 @@ tests.push({ }); export const TestConstantExpression = createTest( 'constant', - 'TestConstantExpression', + 'TestConstantExpression' ); export const TestEmptyExpression = createTest('empty', 'TestEmptyExpression'); export const TestIterableExpression = createTest( 'iterable', - 'TestIterableExpression', + 'TestIterableExpression' ); //endregion diff --git a/src/melody/melody-extension-core/parser/autoescape.js b/src/melody/melody-extension-core/parser/autoescape.js index 99d41222..adcb3991 100644 --- a/src/melody/melody-extension-core/parser/autoescape.js +++ b/src/melody/melody-extension-core/parser/autoescape.js @@ -46,7 +46,7 @@ I expected the current string to end with a ${stringStartToken.text} but instead title: 'Invalid autoescape type declaration', pos: tokens.la(0).pos, advice: `Expected type of autoescape to be a string, boolean or not specified. Found ${tokens.la( - 0, + 0 ).type} instead.`, }); } diff --git a/src/melody/melody-extension-core/parser/block.js b/src/melody/melody-extension-core/parser/block.js index 1f626c5d..4c9e2465 100644 --- a/src/melody/melody-extension-core/parser/block.js +++ b/src/melody/melody-extension-core/parser/block.js @@ -37,7 +37,7 @@ export const BlockParser = { token.type === Types.TAG_START && tokens.nextIf(Types.SYMBOL, 'endblock') ); - }).expressions, + }).expressions ); if (tokens.nextIf(Types.SYMBOL, nameToken.text)) { @@ -49,7 +49,7 @@ export const BlockParser = { advice: unexpectedToken.type == Types.SYMBOL ? `Expected end of block ${nameToken.text} but instead found end of block ${tokens.la( - 0, + 0 ).text}.` : `endblock must be followed by either '%}' or the name of the open block. Found a token of type ${Types .ERROR_TABLE[unexpectedToken.type] || @@ -60,7 +60,7 @@ export const BlockParser = { } else { blockStatement = new BlockStatement( createNode(Identifier, nameToken, nameToken.text), - new PrintExpressionStatement(parser.matchExpression()), + new PrintExpressionStatement(parser.matchExpression()) ); } diff --git a/src/melody/melody-extension-core/parser/embed.js b/src/melody/melody-extension-core/parser/embed.js index d7e89287..bd4bc6e7 100644 --- a/src/melody/melody-extension-core/parser/embed.js +++ b/src/melody/melody-extension-core/parser/embed.js @@ -47,7 +47,7 @@ export const EmbedParser = { tokens.nextIf(Types.SYMBOL, 'endembed') ); }).expressions, - Node.isBlockStatement, + Node.isBlockStatement ); setStartFromToken(embedStatement, token); diff --git a/src/melody/melody-extension-core/parser/filter.js b/src/melody/melody-extension-core/parser/filter.js index 0c08836d..fdd01e62 100644 --- a/src/melody/melody-extension-core/parser/filter.js +++ b/src/melody/melody-extension-core/parser/filter.js @@ -38,7 +38,7 @@ export const FilterParser = { const filterBlockStatement = new FilterBlockStatement( filterExpression, - body, + body ); setStartFromToken(filterBlockStatement, token); setEndFromToken(filterBlockStatement, tokens.expect(Types.TAG_END)); diff --git a/src/melody/melody-extension-core/parser/for.js b/src/melody/melody-extension-core/parser/for.js index f934f89e..86807ef2 100644 --- a/src/melody/melody-extension-core/parser/for.js +++ b/src/melody/melody-extension-core/parser/for.js @@ -33,20 +33,20 @@ export const ForParser = { forStatement.keyTarget = createNode( Identifier, keyTarget, - keyTarget.text, + keyTarget.text ); const valueTarget = tokens.expect(Types.SYMBOL); forStatement.valueTarget = createNode( Identifier, valueTarget, - valueTarget.text, + valueTarget.text ); } else { forStatement.keyTarget = null; forStatement.valueTarget = createNode( Identifier, keyTarget, - keyTarget.text, + keyTarget.text ); } @@ -76,7 +76,7 @@ export const ForParser = { token.type === Types.TAG_START && tokens.test(Types.SYMBOL, 'endfor') ); - }, + } ); } tokens.expect(Types.SYMBOL, 'endfor'); diff --git a/src/melody/melody-extension-core/parser/from.js b/src/melody/melody-extension-core/parser/from.js index cd51781b..fd6a3d95 100644 --- a/src/melody/melody-extension-core/parser/from.js +++ b/src/melody/melody-extension-core/parser/from.js @@ -41,7 +41,7 @@ export const FromParser = { const importDeclaration = new ImportDeclaration( createNode(Identifier, name, name.text), - createNode(Identifier, alias, alias.text), + createNode(Identifier, alias, alias.text) ); setStartFromToken(importDeclaration, name); setEndFromToken(importDeclaration, alias); diff --git a/src/melody/melody-extension-core/parser/if.js b/src/melody/melody-extension-core/parser/if.js index 03ac44db..37d215ac 100644 --- a/src/melody/melody-extension-core/parser/if.js +++ b/src/melody/melody-extension-core/parser/if.js @@ -27,14 +27,14 @@ export const IfParser = { const ifStatement = new IfStatement( test, - parser.parse(matchConsequent).expressions, + parser.parse(matchConsequent).expressions ); do { if (tokens.nextIf(Types.SYMBOL, 'else')) { tokens.expect(Types.TAG_END); (alternate || ifStatement).alternate = parser.parse( - matchAlternate, + matchAlternate ).expressions; } else if (tokens.nextIf(Types.SYMBOL, 'elseif')) { test = parser.matchExpression(); diff --git a/src/melody/melody-extension-core/parser/import.js b/src/melody/melody-extension-core/parser/import.js index 73c57213..17e6ca6b 100644 --- a/src/melody/melody-extension-core/parser/import.js +++ b/src/melody/melody-extension-core/parser/import.js @@ -33,7 +33,7 @@ export const ImportParser = { const importStatement = new ImportDeclaration( source, - createNode(Identifier, alias, alias.text), + createNode(Identifier, alias, alias.text) ); setStartFromToken(importStatement, token); diff --git a/src/melody/melody-extension-core/parser/macro.js b/src/melody/melody-extension-core/parser/macro.js index f8415760..6ec0aafd 100644 --- a/src/melody/melody-extension-core/parser/macro.js +++ b/src/melody/melody-extension-core/parser/macro.js @@ -67,13 +67,13 @@ export const MacroParser = { const macroDeclarationStatement = new MacroDeclarationStatement( createNode(Identifier, nameToken, nameToken.text), args, - body, + body ); setStartFromToken(macroDeclarationStatement, token); setEndFromToken( macroDeclarationStatement, - tokens.expect(Types.TAG_END), + tokens.expect(Types.TAG_END) ); return macroDeclarationStatement; diff --git a/src/melody/melody-extension-core/parser/set.js b/src/melody/melody-extension-core/parser/set.js index 66550703..c9fa8d7e 100644 --- a/src/melody/melody-extension-core/parser/set.js +++ b/src/melody/melody-extension-core/parser/set.js @@ -72,7 +72,7 @@ ${names.length} variable names and ${values.length} values.`, for (let i = 0, len = names.length; i < len; i++) { assignments[i] = new VariableDeclarationStatement( names[i], - values[i], + values[i] ); } diff --git a/src/melody/melody-extension-core/types.js b/src/melody/melody-extension-core/types.js index 6b25894d..f8f0ecf7 100644 --- a/src/melody/melody-extension-core/types.js +++ b/src/melody/melody-extension-core/types.js @@ -61,7 +61,7 @@ export class MountStatement extends Node { name?: Identifier, source?: String, key?: Node, - argument?: Node, + argument?: Node ) { super(); this.name = name; @@ -135,7 +135,7 @@ export class ForStatement extends Node { sequence?: Node = null, condition?: Node = null, body?: Node = null, - otherwise?: Node = null, + otherwise?: Node = null ) { super(); this.keyTarget = keyTarget; @@ -155,7 +155,7 @@ visitor( 'sequence', 'condition', 'body', - 'otherwise', + 'otherwise' ); export class ImportDeclaration extends Node { diff --git a/src/melody/melody-extension-core/visitors/filters.js b/src/melody/melody-extension-core/visitors/filters.js index ae644009..46501359 100644 --- a/src/melody/melody-extension-core/visitors/filters.js +++ b/src/melody/melody-extension-core/visitors/filters.js @@ -32,7 +32,7 @@ export default { defaultFilter({ VAR: path.node.target, DEFAULT: path.node.arguments[0] || t.stringLiteral(''), - }).expression, + }).expression ); }, abs(path) { @@ -40,16 +40,16 @@ export default { path.replaceWithJS( t.callExpression( t.memberExpression(t.identifier('Math'), t.identifier('abs')), - [path.node.target], - ), + [path.node.target] + ) ); }, join(path) { path.replaceWithJS( t.callExpression( t.memberExpression(path.node.target, t.identifier('join')), - path.node.arguments, - ), + path.node.arguments + ) ); }, json_encode(path) { @@ -58,15 +58,15 @@ export default { t.callExpression( t.memberExpression( t.identifier('JSON'), - t.identifier('stringify'), + t.identifier('stringify') ), - [path.node.target], - ), + [path.node.target] + ) ); }, length(path) { path.replaceWithJS( - t.memberExpression(path.node.target, t.identifier('length')), + t.memberExpression(path.node.target, t.identifier('length')) ); }, lower(path) { @@ -74,10 +74,10 @@ export default { t.callExpression( t.memberExpression( path.node.target, - t.identifier('toLowerCase'), + t.identifier('toLowerCase') ), - [], - ), + [] + ) ); }, upper(path) { @@ -85,42 +85,42 @@ export default { t.callExpression( t.memberExpression( path.node.target, - t.identifier('toUpperCase'), + t.identifier('toUpperCase') ), - [], - ), + [] + ) ); }, slice(path) { path.replaceWithJS( t.callExpression( t.memberExpression(path.node.target, t.identifier('slice')), - path.node.arguments, - ), + path.node.arguments + ) ); }, sort(path) { path.replaceWithJS( t.callExpression( t.memberExpression(path.node.target, t.identifier('sort')), - path.node.arguments, - ), + path.node.arguments + ) ); }, split(path) { path.replaceWithJS( t.callExpression( t.memberExpression(path.node.target, t.identifier('split')), - path.node.arguments, - ), + path.node.arguments + ) ); }, trim(path) { path.replaceWithJS( t.callExpression( t.memberExpression(path.node.target, t.identifier('trim')), - path.node.arguments, - ), + path.node.arguments + ) ); }, convert_encoding(path) { @@ -131,10 +131,10 @@ export default { path.replaceWithJS( t.callExpression( t.identifier( - path.state.addImportFrom('melody-runtime', 'strtotime'), + path.state.addImportFrom('melody-runtime', 'strtotime') ), - [path.node.arguments[0], path.node.target], - ), + [path.node.arguments[0], path.node.target] + ) ); }, date(path) { @@ -145,10 +145,10 @@ export default { t.callExpression( t.callExpression( t.identifier(path.state.addDefaultImportFrom('moment')), - [path.node.target], + [path.node.target] ), - [path.node.arguments[0]], - ), + [path.node.arguments[0]] + ) ); }, }; diff --git a/src/melody/melody-extension-core/visitors/for.js b/src/melody/melody-extension-core/visitors/for.js index 26e08f6c..f254d109 100644 --- a/src/melody/melody-extension-core/visitors/for.js +++ b/src/melody/melody-extension-core/visitors/for.js @@ -117,14 +117,14 @@ export default { scope.registerBinding( forStmt.keyTarget.name, path.get('keyTarget'), - 'var', + 'var' ); } if (forStmt.valueTarget) { scope.registerBinding( forStmt.valueTarget.name, path.get('valueTarget'), - 'var', + 'var' ); } scope.registerBinding('loop', path, 'var'); @@ -227,14 +227,14 @@ export default { if (sequence.is('Identifier')) { sequence.setData( 'Identifier.contextName', - parentContextName, + parentContextName ); } else { traverse(path.node.sequence, { Identifier(id) { id.setData( 'Identifier.contextName', - parentContextName, + parentContextName ); }, }); @@ -244,7 +244,7 @@ export default { exit(path) { const node = path.node; const { sequenceName, lenName, iName } = path.getData( - 'forStatement.variableLookup', + 'forStatement.variableLookup' ); let expr; if (path.scope.escapesContext) { @@ -255,8 +255,8 @@ export default { CREATE_SUB_CONTEXT: t.identifier( this.addImportFrom( 'melody-runtime', - 'createSubContext', - ), + 'createSubContext' + ) ), KEY_TARGET: t.identifier(iName), SOURCE: path.get('sequence').node, @@ -344,16 +344,16 @@ export default { path.scope.parent.registerBinding( uniteratedName, path, - 'var', + 'var' ); expr.forStmt.body.body.push( t.expressionStatement( t.assignmentExpression( '=', t.identifier(uniteratedName), - t.booleanLiteral(false), - ), - ), + t.booleanLiteral(false) + ) + ) ); } @@ -363,7 +363,7 @@ export default { type: 'IfStatement', test: node.condition, consequent: t.blockStatement( - expr.forStmt.body.body, + expr.forStmt.body.body ), }, ]); @@ -374,14 +374,14 @@ export default { t.variableDeclaration('let', [ t.variableDeclarator( t.identifier(uniteratedName), - t.booleanLiteral(true), + t.booleanLiteral(true) ), ]), expr.exprStmt, t.ifStatement( t.identifier(uniteratedName), - node.otherwise, - ), + node.otherwise + ) ); } else { path.replaceWithJS(expr.exprStmt); diff --git a/src/melody/melody-extension-core/visitors/functions.js b/src/melody/melody-extension-core/visitors/functions.js index c701cf98..4fd85436 100644 --- a/src/melody/melody-extension-core/visitors/functions.js +++ b/src/melody/melody-extension-core/visitors/functions.js @@ -35,15 +35,15 @@ export default { path.state.error( 'Invalid range call', path.node.pos, - `The range function accepts 1 to 3 arguments but you have specified ${args.length} arguments instead.`, + `The range function accepts 1 to 3 arguments but you have specified ${args.length} arguments instead.` ); } path.replaceWithJS( t.callExpression( t.identifier(path.state.addImportFrom('lodash', 'range')), - callArgs, - ), + callArgs + ) ); }, // range: 'lodash', @@ -52,7 +52,7 @@ export default { path.state.error( 'dump must be used in a lone expression', path.node.pos, - 'The dump function does not have a return value. Thus it must be used as the only expression.', + 'The dump function does not have a return value. Thus it must be used as the only expression.' ); } path.parentPath.replaceWithJS( @@ -60,11 +60,11 @@ export default { t.callExpression( t.memberExpression( t.identifier('console'), - t.identifier('log'), + t.identifier('log') ), - path.node.arguments, - ), - ), + path.node.arguments + ) + ) ); }, include(path) { @@ -79,7 +79,7 @@ export default { const includeName = path.scope.generateUid('include'); const importDecl = t.importDeclaration( [t.importDefaultSpecifier(t.identifier(includeName))], - path.node.arguments[0], + path.node.arguments[0] ); path.state.program.body.splice(0, 0, importDecl); path.scope.registerBinding(includeName); @@ -90,10 +90,10 @@ export default { t.callExpression( t.memberExpression( t.identifier(includeName), - t.identifier('render'), + t.identifier('render') ), - argument ? [argument] : [], - ), + argument ? [argument] : [] + ) ); path.replaceWithJS(includeCall); }, diff --git a/src/melody/melody-extension-core/visitors/tests.js b/src/melody/melody-extension-core/visitors/tests.js index f5359c1f..6811f8a3 100644 --- a/src/melody/melody-extension-core/visitors/tests.js +++ b/src/melody/melody-extension-core/visitors/tests.js @@ -24,8 +24,8 @@ export default { t.binaryExpression( '%', path.get('expression').node, - t.numericLiteral(2), - ), + t.numericLiteral(2) + ) ); expr.extra = { parenthesizedArgument: true }; path.replaceWithJS(expr); @@ -40,9 +40,9 @@ export default { t.binaryExpression( '%', path.get('expression').node, - t.numericLiteral(2), - ), - ), + t.numericLiteral(2) + ) + ) ); expr.extra = { parenthesizedArgument: true }; path.replaceWithJS(expr); @@ -55,10 +55,10 @@ export default { '!==', t.unaryExpression( 'typeof', - path.get('expression').node, + path.get('expression').node ), - t.stringLiteral('undefined'), - ), + t.stringLiteral('undefined') + ) ); }, }, @@ -67,10 +67,10 @@ export default { path.replaceWithJS( t.callExpression( t.identifier( - this.addImportFrom('melody-runtime', 'isEmpty'), + this.addImportFrom('melody-runtime', 'isEmpty') ), - [path.get('expression').node], - ), + [path.get('expression').node] + ) ); }, }, @@ -80,8 +80,8 @@ export default { t.binaryExpression( '===', path.get('expression').node, - path.get('arguments')[0].node, - ), + path.get('arguments')[0].node + ) ); }, }, @@ -91,8 +91,8 @@ export default { t.binaryExpression( '===', path.get('expression').node, - t.nullLiteral(), - ), + t.nullLiteral() + ) ); }, }, @@ -104,9 +104,9 @@ export default { t.binaryExpression( '%', path.get('expression').node, - path.node.arguments[0], - ), - ), + path.node.arguments[0] + ) + ) ); }, }, @@ -116,10 +116,10 @@ export default { t.callExpression( t.memberExpression( t.identifier('Array'), - t.identifier('isArray'), + t.identifier('isArray') ), - [path.node.expression], - ), + [path.node.expression] + ) ); }, }, diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js index 0880cd8e..e9ef0e16 100644 --- a/src/melody/melody-parser/Lexer.js +++ b/src/melody/melody-parser/Lexer.js @@ -172,7 +172,7 @@ export default class Lexer { this.error( 'Unexpected end for HTML comment', input.mark(), - `Expected comment to end with '>' but found '${c}' instead.`, + `Expected comment to end with '>' but found '${c}' instead.` ); } break; @@ -461,11 +461,11 @@ export default class Lexer { pos, inElement ? `Expected a valid attribute name, but instead found "${input.la( - 0, + 0 )}", which is not part of a valid attribute name.` : `Expected letter, digit or underscore but found ${input.la( - 0, - )} instead.`, + 0 + )} instead.` ); } return this.createToken(TokenTypes.SYMBOL, pos); diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index ebb85a84..6b583b47 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -107,8 +107,8 @@ export default class Parser { p.add( copyLoc( new n.PrintExpressionStatement(expression), - expression, - ), + expression + ) ); setEndFromToken(p, tokens.expect(Types.EXPRESSION_END)); break; @@ -121,8 +121,8 @@ export default class Parser { createNode( n.PrintTextStatement, token, - createNode(n.StringLiteral, token, token.text), - ), + createNode(n.StringLiteral, token, token.text) + ) ); break; case Types.ENTITY: @@ -133,9 +133,9 @@ export default class Parser { createNode( n.StringLiteral, token, - he.decode(token.text), - ), - ), + he.decode(token.text) + ) + ) ); break; case Types.ELEMENT_START: @@ -230,7 +230,7 @@ export default class Parser { nodes[nodes.length] = createNode( n.StringLiteral, token, - token.text, + token.text ); canBeString = false; } else if ( @@ -262,7 +262,7 @@ export default class Parser { element.attributes.push(attr); } else { element.attributes.push( - copyLoc(new n.Attribute(keyNode), keyNode), + copyLoc(new n.Attribute(keyNode), keyNode) ); } } else if (tokens.nextIf(Types.EXPRESSION_START)) { @@ -292,9 +292,9 @@ export default class Parser { `Unknown tag "${tag.text}"`, tag.pos, `Expected a known tag such as\n- ${Object.getOwnPropertyNames( - this[TAG], + this[TAG] ).join('\n- ')}`, - tag.length, + tag.length ); } return parser.parse(this, tag); @@ -318,7 +318,7 @@ export default class Parser { const expr1 = this.matchExpression( op.associativity === LEFT ? op.precedence + 1 - : op.precedence, + : op.precedence ); expr = op.createNode(token, expr, expr1); } @@ -366,7 +366,7 @@ export default class Parser { // SYMBOL '(' arguments* ')' node = new n.CallExpression( createNode(n.Identifier, token, token.text), - this.matchArguments(), + this.matchArguments() ); copyStart(node, node.callee); setEndFromToken(node, tokens.la(-1)); // ')' @@ -378,7 +378,7 @@ export default class Parser { node = createNode( n.NumericLiteral, token, - Number(tokens.next()), + Number(tokens.next()) ); break; case Types.STRING_START: @@ -420,7 +420,7 @@ export default class Parser { nodes[nodes.length] = createNode( n.StringLiteral, token, - token.text, + token.text ); canBeString = false; } else if ((token = tokens.nextIf(Types.INTERPOLATION_START))) { @@ -436,7 +436,7 @@ export default class Parser { if (!nodes.length) { return setEndFromToken( createNode(n.StringLiteral, stringStart, ''), - stringEnd, + stringEnd ); } @@ -473,7 +473,7 @@ export default class Parser { condition = new n.ConditionalExpression( condition, consequent, - alternate, + alternate ); condition.loc.start = { line, column }; copyEnd(condition, alternate || consequent); @@ -579,7 +579,7 @@ export default class Parser { property = createNode( n.NumericLiteral, token, - Number(token.text), + Number(token.text) ); computed = true; } else { @@ -599,7 +599,7 @@ export default class Parser { if (tokens.test(Types.LPAREN)) { const callExpr = new n.CallExpression( memberExpr, - this.matchArguments(), + this.matchArguments() ); copyStart(callExpr, memberExpr); setEndFromToken(callExpr, tokens.la(-1)); @@ -624,14 +624,14 @@ export default class Parser { if (arg) { return setEndFromToken( copyStart(new n.MemberExpression(node, arg, true), node), - tokens.expect(Types.RBRACE), + tokens.expect(Types.RBRACE) ); } else { // slice const result = new n.SliceExpression( node, start, - tokens.test(Types.RBRACE) ? null : this.matchExpression(), + tokens.test(Types.RBRACE) ? null : this.matchExpression() ); copyStart(result, node); setEndFromToken(result, tokens.expect(Types.RBRACE)); @@ -657,7 +657,7 @@ export default class Parser { if (newTarget.arguments.length) { copyEnd( newTarget, - newTarget.arguments[newTarget.arguments.length - 1], + newTarget.arguments[newTarget.arguments.length - 1] ); } else { copyEnd(newTarget, target); @@ -687,7 +687,7 @@ export default class Parser { const value = this.matchExpression(); const arg = new n.NamedArgumentExpression( createNode(n.Identifier, name, name.text), - value, + value ); copyEnd(arg, value); args.push(arg); diff --git a/src/melody/melody-parser/TokenStream.js b/src/melody/melody-parser/TokenStream.js index 05427c11..2ae62e82 100644 --- a/src/melody/melody-parser/TokenStream.js +++ b/src/melody/melody-parser/TokenStream.js @@ -37,7 +37,7 @@ const TOKENS = Symbol(), export default class TokenStream { constructor( lexer, - options = { ignoreComments: true, ignoreWhitespace: true }, + options = { ignoreComments: true, ignoreWhitespace: true } ) { this.input = lexer; this.index = 0; @@ -54,7 +54,7 @@ export default class TokenStream { errorToken.message, errorToken.pos, errorToken.advice, - errorToken.endPos.index - errorToken.pos.index || 1, + errorToken.endPos.index - errorToken.pos.index || 1 ); } } @@ -102,7 +102,7 @@ export default class TokenStream { text} but found ${ERROR_TABLE[token.type] || token.type || token.text} instead.`, - token.length, + token.length ); } @@ -140,7 +140,7 @@ function getAllTokens(lexer, options) { case TAG_START: if (token.text[token.text.length - 1] === '-') { tokens[tokens.length - 1].text = trimEnd( - tokens[tokens.length - 1].text, + tokens[tokens.length - 1].text ); } break; diff --git a/src/melody/melody-traverse/Path.js b/src/melody/melody-traverse/Path.js index 81c6231b..96619226 100644 --- a/src/melody/melody-traverse/Path.js +++ b/src/melody/melody-traverse/Path.js @@ -77,7 +77,7 @@ export default class Path { if (!path.node) { /*eslint no-console: off*/ console.log( - 'Path has no node ' + path.parentKey + ' > ' + path.key, + 'Path has no node ' + path.parentKey + ' > ' + path.key ); } paths.push(path); @@ -327,7 +327,7 @@ export default class Path { parent: node, container, key: i, - }).setContext(context), + }).setContext(context) ); } else { return Path.get({ diff --git a/src/melody/melody-traverse/traverse.js b/src/melody/melody-traverse/traverse.js index 7d31c973..b4a0c88d 100644 --- a/src/melody/melody-traverse/traverse.js +++ b/src/melody/melody-traverse/traverse.js @@ -21,7 +21,7 @@ export function traverse( visitor: Object, scope?: Object, state?: Object = {}, - parentPath?: Object, + parentPath?: Object ) { if (!parentNode) { return; diff --git a/src/melody/melody-traverse/visitors.js b/src/melody/melody-traverse/visitors.js index 0f40f706..917001e5 100644 --- a/src/melody/melody-traverse/visitors.js +++ b/src/melody/melody-traverse/visitors.js @@ -96,11 +96,11 @@ export function merge(...visitors: Array) { const nodeVisitor = rootVisitor[key]; nodeVisitor.enter = [].concat( nodeVisitor.enter || [], - visitorType.enter || [], + visitorType.enter || [] ); nodeVisitor.exit = [].concat( nodeVisitor.exit || [], - visitorType.exit || [], + visitorType.exit || [] ); } } diff --git a/src/melody/melody-types/index.js b/src/melody/melody-types/index.js index 9a72c550..668a04a0 100644 --- a/src/melody/melody-types/index.js +++ b/src/melody/melody-types/index.js @@ -48,7 +48,7 @@ export class Node { }, { type: this.type, - }, + } ); } From cab483059ac5369a77fadb3e03047b6a19ca14fb Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Sat, 6 Jan 2018 16:01:53 +0530 Subject: [PATCH 03/27] test: melody code frame (#16) Looks very good. Thank you for taking care of this one! --- src/melody/melody-code-frame/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/melody/melody-code-frame/index.js b/src/melody/melody-code-frame/index.js index a0245d7d..a921e618 100644 --- a/src/melody/melody-code-frame/index.js +++ b/src/melody/melody-code-frame/index.js @@ -33,7 +33,6 @@ export default function({ rawLines, lineNumber, colNumber, length }) { } if (colNumber) { - //params.line = params.line.substring(0, colNumber) + chalk.underline(params.line.substring(colNumber, colNumber + length)) + params.line.substring(colNumber + length, params.line.length - 1); params.line += `\n${params.before}${repeat( ' ', params.width From 1fd4d98b9b44733d3e60f5082bae0e918b41587b Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Sat, 13 Jan 2018 02:24:01 +0530 Subject: [PATCH 04/27] fix: adds missing constant type (#21) --- src/melody/melody-types/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/melody/melody-types/index.js b/src/melody/melody-types/index.js index 668a04a0..668d1371 100644 --- a/src/melody/melody-types/index.js +++ b/src/melody/melody-types/index.js @@ -146,6 +146,7 @@ export class ConstantValue extends Node { return `Const(${this.value})`; } } +type(ConstantValue, 'ConstantValue'); alias(ConstantValue, 'Expression', 'Literal', 'Immutable'); export class StringLiteral extends ConstantValue {} From c6ba301659a1d0a7d1f07406a9b9b76fde08f7fb Mon Sep 17 00:00:00 2001 From: Tim De Groote Date: Thu, 1 Feb 2018 10:52:31 +0100 Subject: [PATCH 05/27] Fix not in operator (#26) * Add tests for the 'not in' operator * Don't match operators that are followed by an alphanumeric character This prevents issues like `not invalid` matching the `not in` operator. We only check for operators that contain a whitespace as those that don't are already matched as a symbol. --- src/melody/melody-parser/Lexer.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js index e9ef0e16..6b2f70d1 100644 --- a/src/melody/melody-parser/Lexer.js +++ b/src/melody/melody-parser/Lexer.js @@ -370,8 +370,15 @@ export default class Lexer { for (let i = 0, ops = this[OPERATORS], len = ops.length; i < len; i++) { const op = ops[i]; if (op.length > longestMatchingOperator.length && input.match(op)) { - longestMatchingOperator = op; - longestMatchEndPos = input.mark(); + const cc = input.lac(0); + + // prevent mixing up operators with symbols (e.g. matching + // 'not in' in 'not invalid'). + if (op.indexOf(' ') === -1 || !(isAlpha(cc) || isDigit(cc))) { + longestMatchingOperator = op; + longestMatchEndPos = input.mark(); + } + input.rewind(start); } } From be700683ae5fc1285f4becd8c2a5995c75cf12fe Mon Sep 17 00:00:00 2001 From: Ian Devlin Date: Tue, 3 Apr 2018 16:28:31 +0200 Subject: [PATCH 06/27] Fixed Typo in call to date filter (#35) - Added `filters.template` file to test all Twig filters and updated snapshot --- src/melody/melody-extension-core/visitors/filters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/melody/melody-extension-core/visitors/filters.js b/src/melody/melody-extension-core/visitors/filters.js index 46501359..e3693d00 100644 --- a/src/melody/melody-extension-core/visitors/filters.js +++ b/src/melody/melody-extension-core/visitors/filters.js @@ -141,7 +141,7 @@ export default { // Not really happy about this since moment.js could well be incompatible with // the default twig behaviour // might need to switch to an actual strftime implementation - path.repalceWithJS( + path.replaceWithJS( t.callExpression( t.callExpression( t.identifier(path.state.addDefaultImportFrom('moment')), From 512d11d8e391d53964b86a85112e072b8ac8bf87 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Wed, 15 Aug 2018 09:10:02 +0200 Subject: [PATCH 07/27] fix: generates esm modules using rollup (#42) --- src/melody/melody-parser/Lexer.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js index 6b2f70d1..b2d5929d 100644 --- a/src/melody/melody-parser/Lexer.js +++ b/src/melody/melody-parser/Lexer.js @@ -560,9 +560,8 @@ export default class Lexer { matchText(pos) { let input = this.input, - exit = false, c; - while (!exit && ((c = input.la(0)) && c !== EOF)) { + while ((c = input.la(0)) && c !== EOF) { if (c === '{') { const c2 = input.la(1); if (c2 === '{' || c2 === '#' || c2 === '%') { From 8f4de038283deac502ffa393c01d37914f0ed516 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Tue, 21 Aug 2018 16:24:57 +0200 Subject: [PATCH 08/27] fix: incorrect `is` method call in melody-types (#20) (#52) --- src/melody/melody-parser/Parser.js | 2 +- src/melody/melody-types/index.js | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 6b583b47..cdfabde7 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -512,7 +512,7 @@ export default class Parser { value; if (tokens.test(Types.STRING_START)) { key = this.matchStringExpression(); - if (!n.is('StringLiteral', key)) { + if (!n.is(key, 'StringLiteral')) { computed = true; } } else if ((token = tokens.nextIf(Types.SYMBOL))) { diff --git a/src/melody/melody-types/index.js b/src/melody/melody-types/index.js index 668d1371..77fa9f30 100644 --- a/src/melody/melody-types/index.js +++ b/src/melody/melody-types/index.js @@ -71,11 +71,12 @@ export class Node { Node.registerType('Scope'); export function is(node, type) { + if (!node) return false; + return ( - node && - (node.type === type || - (IS_ALIAS_OF[type] && IS_ALIAS_OF[type][node.type]) || - t.is(node, type)) + node.type === type || + (IS_ALIAS_OF[type] && IS_ALIAS_OF[type][node.type]) || + t.is(type, node) ); } From 216645024f11b87f4fad33b30a9840f778476e65 Mon Sep 17 00:00:00 2001 From: Sergio Santoro Date: Tue, 18 Sep 2018 12:56:59 +0200 Subject: [PATCH 09/27] Add implementation for filter 'trim' to be compatible with Twig (#64) Add implementation for filter 'trim' to be compatible with Twig --- src/melody/melody-extension-core/index.js | 1 + src/melody/melody-extension-core/visitors/filters.js | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/melody/melody-extension-core/index.js b/src/melody/melody-extension-core/index.js index 51ce6229..0cc049f5 100644 --- a/src/melody/melody-extension-core/index.js +++ b/src/melody/melody-extension-core/index.js @@ -54,6 +54,7 @@ const filterMap = [ 'striptags', 'title', 'url_encode', + 'trim', ].reduce((map, filterName) => { map[filterName] = 'melody-runtime'; return map; diff --git a/src/melody/melody-extension-core/visitors/filters.js b/src/melody/melody-extension-core/visitors/filters.js index e3693d00..66a55637 100644 --- a/src/melody/melody-extension-core/visitors/filters.js +++ b/src/melody/melody-extension-core/visitors/filters.js @@ -115,14 +115,6 @@ export default { ) ); }, - trim(path) { - path.replaceWithJS( - t.callExpression( - t.memberExpression(path.node.target, t.identifier('trim')), - path.node.arguments - ) - ); - }, convert_encoding(path) { // encoding conversion is not supported path.replaceWith(path.node.target); From 927c739ba040f6d0753f2676d2f918e3fe43d0ca Mon Sep 17 00:00:00 2001 From: Patrick Gotthardt Date: Fri, 11 Jan 2019 05:26:14 +0100 Subject: [PATCH 10/27] Add initial draft of async mounting (#82) * Add initial draft of async mounting * Fixed handling the default import * Update test * Update changelog * Add more tests * Add test for async component which is unmounted before its loaded --- src/melody/melody-code-frame/index.js | 10 +-- .../parser/autoescape.js | 12 ++-- .../melody-extension-core/parser/block.js | 8 ++- src/melody/melody-extension-core/parser/if.js | 3 +- .../melody-extension-core/parser/macro.js | 4 +- .../melody-extension-core/parser/mount.js | 64 ++++++++++++++++++- src/melody/melody-extension-core/types.js | 21 +++++- .../visitors/functions.js | 4 +- src/melody/melody-parser/Parser.js | 5 +- src/melody/melody-parser/util.js | 9 ++- 10 files changed, 115 insertions(+), 25 deletions(-) diff --git a/src/melody/melody-code-frame/index.js b/src/melody/melody-code-frame/index.js index a921e618..7b37dbec 100644 --- a/src/melody/melody-code-frame/index.js +++ b/src/melody/melody-code-frame/index.js @@ -33,13 +33,9 @@ export default function({ rawLines, lineNumber, colNumber, length }) { } if (colNumber) { - params.line += `\n${params.before}${repeat( - ' ', - params.width - )}${params.after}${repeat(' ', colNumber)}${repeat( - '^', - length - )}`; + params.line += `\n${params.before}${repeat(' ', params.width)}${ + params.after + }${repeat(' ', colNumber)}${repeat('^', length)}`; } params.before = params.before.replace(/^./, '>'); diff --git a/src/melody/melody-extension-core/parser/autoescape.js b/src/melody/melody-extension-core/parser/autoescape.js index adcb3991..0d718bcd 100644 --- a/src/melody/melody-extension-core/parser/autoescape.js +++ b/src/melody/melody-extension-core/parser/autoescape.js @@ -33,8 +33,10 @@ export const AutoescapeParser = { 'autoescape type declaration must be a simple string', pos: tokens.la(0).pos, advice: `The type declaration for autoescape must be a simple string such as 'html' or 'js'. -I expected the current string to end with a ${stringStartToken.text} but instead found ${Types - .ERROR_TABLE[tokens.lat(0)] || tokens.lat(0)}.`, +I expected the current string to end with a ${ + stringStartToken.text + } but instead found ${Types.ERROR_TABLE[tokens.lat(0)] || + tokens.lat(0)}.`, }); } } else if (tokens.nextIf(Types.FALSE)) { @@ -45,9 +47,9 @@ I expected the current string to end with a ${stringStartToken.text} but instead parser.error({ title: 'Invalid autoescape type declaration', pos: tokens.la(0).pos, - advice: `Expected type of autoescape to be a string, boolean or not specified. Found ${tokens.la( - 0 - ).type} instead.`, + advice: `Expected type of autoescape to be a string, boolean or not specified. Found ${ + tokens.la(0).type + } instead.`, }); } diff --git a/src/melody/melody-extension-core/parser/block.js b/src/melody/melody-extension-core/parser/block.js index 4c9e2465..f8b6c309 100644 --- a/src/melody/melody-extension-core/parser/block.js +++ b/src/melody/melody-extension-core/parser/block.js @@ -48,9 +48,11 @@ export const BlockParser = { pos: unexpectedToken.pos, advice: unexpectedToken.type == Types.SYMBOL - ? `Expected end of block ${nameToken.text} but instead found end of block ${tokens.la( - 0 - ).text}.` + ? `Expected end of block ${ + nameToken.text + } but instead found end of block ${ + tokens.la(0).text + }.` : `endblock must be followed by either '%}' or the name of the open block. Found a token of type ${Types .ERROR_TABLE[unexpectedToken.type] || unexpectedToken.type} instead.`, diff --git a/src/melody/melody-extension-core/parser/if.js b/src/melody/melody-extension-core/parser/if.js index 37d215ac..17e3c1d5 100644 --- a/src/melody/melody-extension-core/parser/if.js +++ b/src/melody/melody-extension-core/parser/if.js @@ -40,7 +40,8 @@ export const IfParser = { test = parser.matchExpression(); tokens.expect(Types.TAG_END); const consequent = parser.parse(matchConsequent).expressions; - alternate = (alternate || ifStatement + alternate = ( + alternate || ifStatement ).alternate = new IfStatement(test, consequent); } diff --git a/src/melody/melody-extension-core/parser/macro.js b/src/melody/melody-extension-core/parser/macro.js index 6ec0aafd..8420109d 100644 --- a/src/melody/melody-extension-core/parser/macro.js +++ b/src/melody/melody-extension-core/parser/macro.js @@ -58,7 +58,9 @@ export const MacroParser = { var nameEndToken = tokens.next(); if (nameToken.text !== nameEndToken.text) { parser.error({ - title: `Macro name mismatch, expected "${nameToken.text}" but found "${nameEndToken.text}"`, + title: `Macro name mismatch, expected "${ + nameToken.text + }" but found "${nameEndToken.text}"`, pos: nameEndToken.pos, }); } diff --git a/src/melody/melody-extension-core/parser/mount.js b/src/melody/melody-extension-core/parser/mount.js index a6d4b3d1..5e06e585 100644 --- a/src/melody/melody-extension-core/parser/mount.js +++ b/src/melody/melody-extension-core/parser/mount.js @@ -30,8 +30,19 @@ export const MountParser = { let name = null, source = null, key = null, + async = false, + delayBy = 0, argument = null; + if (tokens.test(Types.SYMBOL, 'async')) { + // we might be looking at an async mount + const nextToken = tokens.la(1); + if (nextToken.type === Types.STRING_START) { + async = true; + tokens.next(); + } + } + if (tokens.test(Types.STRING_START)) { source = parser.matchStringExpression(); } else { @@ -50,7 +61,58 @@ export const MountParser = { argument = parser.matchExpression(); } - const mountStatement = new MountStatement(name, source, key, argument); + if (async) { + if (tokens.nextIf(Types.SYMBOL, 'delay')) { + tokens.expect(Types.SYMBOL, 'placeholder'); + tokens.expect(Types.SYMBOL, 'by'); + delayBy = Number.parseInt(tokens.expect(Types.NUMBER).text, 10); + if (tokens.nextIf(Types.SYMBOL, 's')) { + delayBy *= 1000; + } else { + tokens.expect(Types.SYMBOL, 'ms'); + } + } + } + + const mountStatement = new MountStatement( + name, + source, + key, + argument, + async, + delayBy + ); + + if (async) { + tokens.expect(Types.TAG_END); + + mountStatement.body = parser.parse((tokenText, token, tokens) => { + return ( + token.type === Types.TAG_START && + (tokens.test(Types.SYMBOL, 'catch') || + tokens.test(Types.SYMBOL, 'endmount')) + ); + }); + + if (tokens.nextIf(Types.SYMBOL, 'catch')) { + const errorVariableName = tokens.expect(Types.SYMBOL); + mountStatement.errorVariableName = createNode( + Identifier, + errorVariableName, + errorVariableName.text + ); + tokens.expect(Types.TAG_END); + mountStatement.otherwise = parser.parse( + (tokenText, token, tokens) => { + return ( + token.type === Types.TAG_START && + tokens.test(Types.SYMBOL, 'endmount') + ); + } + ); + } + tokens.expect(Types.SYMBOL, 'endmount'); + } setStartFromToken(mountStatement, token); setEndFromToken(mountStatement, tokens.expect(Types.TAG_END)); diff --git a/src/melody/melody-extension-core/types.js b/src/melody/melody-extension-core/types.js index f8f0ecf7..4b42278f 100644 --- a/src/melody/melody-extension-core/types.js +++ b/src/melody/melody-extension-core/types.js @@ -61,18 +61,33 @@ export class MountStatement extends Node { name?: Identifier, source?: String, key?: Node, - argument?: Node + argument?: Node, + async?: Boolean, + delayBy?: Number ) { super(); this.name = name; this.source = source; this.key = key; this.argument = argument; + this.async = async; + this.delayBy = delayBy; + this.errorVariableName = null; + this.body = null; + this.otherwise = null; } } type(MountStatement, 'MountStatement'); -alias(MountStatement, 'Statement'); -visitor(MountStatement, 'name', 'source', 'key', 'argument'); +alias(MountStatement, 'Statement', 'Scope'); +visitor( + MountStatement, + 'name', + 'source', + 'key', + 'argument', + 'body', + 'otherwise' +); export class DoStatement extends Node { constructor(expression: Node) { diff --git a/src/melody/melody-extension-core/visitors/functions.js b/src/melody/melody-extension-core/visitors/functions.js index 4fd85436..eb1f1ac3 100644 --- a/src/melody/melody-extension-core/visitors/functions.js +++ b/src/melody/melody-extension-core/visitors/functions.js @@ -35,7 +35,9 @@ export default { path.state.error( 'Invalid range call', path.node.pos, - `The range function accepts 1 to 3 arguments but you have specified ${args.length} arguments instead.` + `The range function accepts 1 to 3 arguments but you have specified ${ + args.length + } arguments instead.` ); } diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index cdfabde7..73c3c1a5 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -162,8 +162,9 @@ export default class Parser { pos: elementStartToken.pos, advice: tokens.lat(0) === Types.SLASH - ? `Unexpected closing "${tokens.la(1) - .text}" tag. Seems like your DOM is out of control.` + ? `Unexpected closing "${ + tokens.la(1).text + }" tag. Seems like your DOM is out of control.` : 'Expected an element to start', }); } diff --git a/src/melody/melody-parser/util.js b/src/melody/melody-parser/util.js index 98acc03c..96269619 100644 --- a/src/melody/melody-parser/util.js +++ b/src/melody/melody-parser/util.js @@ -23,7 +23,14 @@ export function setEndFromToken(node, { pos: { line, column }, end }) { return node; } -export function copyStart(node, { loc: { start: { line, column, index } } }) { +export function copyStart( + node, + { + loc: { + start: { line, column, index }, + }, + } +) { node.loc.start.line = line; node.loc.start.column = column; node.loc.start.index = index; From 880a5fc053147cf9ece76aa518f608de2e0b2104 Mon Sep 17 00:00:00 2001 From: Behrang Yarahmadi <6979966+byara@users.noreply.github.com> Date: Tue, 23 Apr 2019 09:54:24 +0200 Subject: [PATCH 11/27] Warn about usage of non-breaking space (#107) * Throws better error for NBSP tokens [#85] Co-Authored-By: byara <6979966+byara@users.noreply.github.com> --- src/melody/melody-parser/Lexer.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js index b2d5929d..596fb981 100644 --- a/src/melody/melody-parser/Lexer.js +++ b/src/melody/melody-parser/Lexer.js @@ -355,6 +355,11 @@ export default class Lexer { } else if (CHAR_TO_TOKEN.hasOwnProperty(c)) { input.next(); return this.createToken(CHAR_TO_TOKEN[c], pos); + } else if (c === '\xa0') { + return this.error( + 'Unsupported token: Non-breaking space', + pos + ); } else { return this.error(`Unknown token ${c}`, pos); } From de6ed9f2fbf3a589da6fc17a7beb625a5e40af41 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Mon, 4 Nov 2019 11:50:32 +0100 Subject: [PATCH 12/27] Add twig comment ast node type (#135) * melody-types: Add TwigComment Node type melody-parser: Preserve comments in AST * Add parser option to ignore Twig comments or not Add CHANGELOG entry --- src/melody/melody-parser/Parser.js | 14 +++++++++++++- src/melody/melody-types/index.js | 9 +++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 73c3c1a5..d67cbc7b 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -46,12 +46,13 @@ const UNARY = Symbol(), TAG = Symbol(), TEST = Symbol(); export default class Parser { - constructor(tokenStream) { + constructor(tokenStream, options = { ignoreComments: true }) { this.tokens = tokenStream; this[UNARY] = {}; this[BINARY] = {}; this[TAG] = {}; this[TEST] = {}; + this.options = options; } addUnaryOperator(op: UnaryOperator) { @@ -141,6 +142,17 @@ export default class Parser { case Types.ELEMENT_START: p.add(this.matchElement()); break; + case Types.COMMENT: + if (!this.options.ignoreComments) { + p.add( + createNode( + n.TwigComment, + token, + createNode(n.StringLiteral, token, token.text) + ) + ); + } + break; } } return p; diff --git a/src/melody/melody-types/index.js b/src/melody/melody-types/index.js index 77fa9f30..cc658dd3 100644 --- a/src/melody/melody-types/index.js +++ b/src/melody/melody-types/index.js @@ -358,3 +358,12 @@ export class Attribute extends Node { } type(Attribute, 'Attribute'); visitor(Attribute, 'name', 'value'); + +export class TwigComment extends Node { + constructor(text: StringLiteral) { + super(); + this.value = text; + } +} +type(TwigComment, 'TwigComment'); +visitor(TwigComment, 'value'); From e503ed2b98a2b0faea3bad0bf52b928e8881ad66 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Mon, 4 Nov 2019 12:42:59 +0100 Subject: [PATCH 13/27] Preserve HTML comments (#137) * melody-types: Add TwigComment Node type melody-parser: Preserve comments in AST * Add parser option to ignore Twig comments or not Add CHANGELOG entry * melody-types: Add node type "HtmlComment" melody-parser: Optionally preserve HTML comments --- src/melody/melody-parser/Parser.js | 16 +++++++++++++++- src/melody/melody-types/index.js | 9 +++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index d67cbc7b..0c2599e9 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -46,7 +46,10 @@ const UNARY = Symbol(), TAG = Symbol(), TEST = Symbol(); export default class Parser { - constructor(tokenStream, options = { ignoreComments: true }) { + constructor( + tokenStream, + options = { ignoreComments: true, ignoreHtmlComments: true } + ) { this.tokens = tokenStream; this[UNARY] = {}; this[BINARY] = {}; @@ -153,6 +156,17 @@ export default class Parser { ); } break; + case Types.HTML_COMMENT: + if (!this.options.ignoreHtmlComments) { + p.add( + createNode( + n.HtmlComment, + token, + createNode(n.StringLiteral, token, token.text) + ) + ); + } + break; } } return p; diff --git a/src/melody/melody-types/index.js b/src/melody/melody-types/index.js index cc658dd3..9df52b80 100644 --- a/src/melody/melody-types/index.js +++ b/src/melody/melody-types/index.js @@ -367,3 +367,12 @@ export class TwigComment extends Node { } type(TwigComment, 'TwigComment'); visitor(TwigComment, 'value'); + +export class HtmlComment extends Node { + constructor(text: StringLiteral) { + super(); + this.value = text; + } +} +type(HtmlComment, 'HtmlComment'); +visitor(HtmlComment, 'value'); From 59cde7f9ad87de57c400c8decbb212d4f17cbd19 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Mon, 4 Nov 2019 14:11:31 +0100 Subject: [PATCH 14/27] Add option to decode entities (#136) * melody-types: Add TwigComment Node type melody-parser: Preserve comments in AST * Add parser option to ignore Twig comments or not Add CHANGELOG entry * melody-types: Add node type "HtmlComment" melody-parser: Optionally preserve HTML comments * Add option "decodeEntities" * Add some documentation on melody-parser --- src/melody/melody-parser/Parser.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 0c2599e9..84f39059 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -48,7 +48,11 @@ const UNARY = Symbol(), export default class Parser { constructor( tokenStream, - options = { ignoreComments: true, ignoreHtmlComments: true } + options = { + ignoreComments: true, + ignoreHtmlComments: true, + decodeEntites: true, + } ) { this.tokens = tokenStream; this[UNARY] = {}; @@ -137,7 +141,9 @@ export default class Parser { createNode( n.StringLiteral, token, - he.decode(token.text) + this.options.decodeEntites + ? he.decode(token.text) + : token.text ) ) ); From 2eaac1e48b24e05eec69d8b7e24ddbbc066446fb Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Mon, 11 Nov 2019 08:39:32 +0100 Subject: [PATCH 15/27] Optionally skip whitespace trimming in tokenizer (#139) Avoid whitespace trimming (optimization) in TokenStream if "ignoreWhitespace" option is false --- src/melody/melody-parser/TokenStream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/melody/melody-parser/TokenStream.js b/src/melody/melody-parser/TokenStream.js index 2ae62e82..4432d2d4 100644 --- a/src/melody/melody-parser/TokenStream.js +++ b/src/melody/melody-parser/TokenStream.js @@ -168,7 +168,7 @@ function getAllTokens(lexer, options) { ) { tokens[tokens.length] = token; } - acceptWhitespaceControl = true; + acceptWhitespaceControl = options.ignoreWhitespace; if (token.type === ERROR) { return tokens; } From dc8b7d1bbef699d91b4f75d378956662b1e56a66 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Wed, 20 Nov 2019 09:45:19 +0100 Subject: [PATCH 16/27] Tokenizer preserve whitespace (#140) * Avoid whitespace trimming (optimization) in tokenizer if "ignoreWhitespace" option is false * Update packages/melody-parser/src/TokenStream.js Small semantic fix Co-Authored-By: Patrick Gotthardt * Introduce applyWhitespaceTrimming option for TokenStream Improve option handling in Parser and TokenStream Update tests and snapshots * Update CHANGELOG * Add new flags to expression nodes: exprStartBefore exprEndAfter trimLeft trimRight * Avoid re-assignment to function parameter * Remove exprStartBefore and exprEndAfter flags They are probably not needed --- src/melody/melody-parser/Parser.js | 50 ++++++++++++++++++------- src/melody/melody-parser/TokenStream.js | 20 ++++++---- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 84f39059..26d34c65 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -46,20 +46,21 @@ const UNARY = Symbol(), TAG = Symbol(), TEST = Symbol(); export default class Parser { - constructor( - tokenStream, - options = { - ignoreComments: true, - ignoreHtmlComments: true, - decodeEntites: true, - } - ) { + constructor(tokenStream, options) { this.tokens = tokenStream; this[UNARY] = {}; this[BINARY] = {}; this[TAG] = {}; this[TEST] = {}; - this.options = options; + this.options = Object.assign( + {}, + { + ignoreComments: true, + ignoreHtmlComments: true, + decodeEntites: true, + }, + options + ); } addUnaryOperator(op: UnaryOperator) { @@ -334,10 +335,20 @@ export default class Parser { } matchExpression(precedence = 0) { - let expr = this.getPrimary(), - tokens = this.tokens, - token, - op; + const tokens = this.tokens; + let token, + op, + trimLeft = false; + + // Check for {{- (trim preceding whitespace) + if ( + tokens.la(-1).type === Types.EXPRESSION_START && + tokens.la(-1).text.endsWith('-') + ) { + trimLeft = true; + } + + let expr = this.getPrimary(); while ( (token = tokens.la(0)) && token.type !== Types.EOF && @@ -358,7 +369,18 @@ export default class Parser { token = tokens.la(0); } - return precedence === 0 ? this.matchConditionalExpression(expr) : expr; + const result = + precedence === 0 ? this.matchConditionalExpression(expr) : expr; + + // Check for -}} (trim following whitespace) + if (token.type === Types.EXPRESSION_END && token.text.startsWith('-')) { + result.trimRight = true; + } + if (trimLeft) { + result.trimLeft = trimLeft; + } + + return result; } getPrimary() { diff --git a/src/melody/melody-parser/TokenStream.js b/src/melody/melody-parser/TokenStream.js index 4432d2d4..4411be70 100644 --- a/src/melody/melody-parser/TokenStream.js +++ b/src/melody/melody-parser/TokenStream.js @@ -35,14 +35,20 @@ const TOKENS = Symbol(), LENGTH = Symbol(); export default class TokenStream { - constructor( - lexer, - options = { ignoreComments: true, ignoreWhitespace: true } - ) { + constructor(lexer, options) { this.input = lexer; this.index = 0; - this.options = options; - this[TOKENS] = getAllTokens(lexer, options); + const mergedOptions = Object.assign( + {}, + { + ignoreComments: true, + ignoreHtmlComments: true, + ignoreWhitespace: true, + applyWhitespaceTrimming: true, + }, + options + ); + this[TOKENS] = getAllTokens(lexer, mergedOptions); this[LENGTH] = this[TOKENS].length; if ( @@ -168,7 +174,7 @@ function getAllTokens(lexer, options) { ) { tokens[tokens.length] = token; } - acceptWhitespaceControl = options.ignoreWhitespace; + acceptWhitespaceControl = options.applyWhitespaceTrimming; if (token.type === ERROR) { return tokens; } From e3f7640aeeba48bbfe06b0d41473e10dcf39ccc5 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Fri, 22 Nov 2019 09:27:28 +0100 Subject: [PATCH 17/27] Mark BinaryConcatExpressions generated by Melody (#141) * Mark BinaryConcatExpressions generated by Melody with "wasImplicitConcatenation" (true/false) * Update CHANGELOG --- src/melody/melody-parser/Parser.js | 10 ++++++++++ src/melody/melody-types/index.js | 1 + 2 files changed, 11 insertions(+) diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 26d34c65..872febe3 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -290,6 +290,12 @@ export default class Parser { expr.loc.start.column = column; copyEnd(expr, expr.right); } + // Distinguish between BinaryConcatExpression generated by + // this Parser (implicit before parsing), and those that the + // user wrote explicitly. + if (nodes.length > 1) { + expr.wasImplicitConcatenation = true; + } const attr = new n.Attribute(keyNode, expr); copyStart(attr, keyNode); copyEnd(attr, expr); @@ -504,6 +510,10 @@ export default class Parser { copyEnd(expr, expr.right); } + if (nodes.length > 1) { + expr.wasImplicitConcatenation = true; + } + return expr; } diff --git a/src/melody/melody-types/index.js b/src/melody/melody-types/index.js index 9df52b80..6d04b20b 100644 --- a/src/melody/melody-types/index.js +++ b/src/melody/melody-types/index.js @@ -209,6 +209,7 @@ visitor(BinaryExpression, 'left', 'right'); export class BinaryConcatExpression extends BinaryExpression { constructor(left: Node, right: Node) { super('~', left, right); + this.wasImplicitConcatenation = false; } } type(BinaryConcatExpression, 'BinaryConcatExpression'); From 84fd4234bc1721cc007bfa9727d6788f6069fb22 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Tue, 26 Nov 2019 06:26:13 +0100 Subject: [PATCH 18/27] Fix typo bug in Parser option --- src/melody/melody-parser/Parser.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 872febe3..09bc3f86 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -57,7 +57,7 @@ export default class Parser { { ignoreComments: true, ignoreHtmlComments: true, - decodeEntites: true, + decodeEntities: true, }, options ); @@ -142,7 +142,7 @@ export default class Parser { createNode( n.StringLiteral, token, - this.options.decodeEntites + this.options.decodeEntities ? he.decode(token.text) : token.text ) From 657a7a35074cbe47b0fc4cb505d4c15af625672d Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Mon, 16 Dec 2019 19:33:45 +0100 Subject: [PATCH 19/27] Make Lexer interrupt text token when hitting an HTML comment (#145) --- src/melody/melody-parser/Lexer.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js index 596fb981..bc2196f5 100644 --- a/src/melody/melody-parser/Lexer.js +++ b/src/melody/melody-parser/Lexer.js @@ -573,7 +573,12 @@ export default class Lexer { break; } } else if (c === '<') { - if (input.la(1) === '/' || isAlpha(input.lac(1))) { + const nextChar = input.la(1); + if ( + nextChar === '/' || // closing tag + nextChar === '!' || // HTML comment + isAlpha(input.lac(1)) // opening tag + ) { break; } else if (input.la(1) === '{') { const c2 = input.la(1); From 8f26f1ff4f71193029f834254245d3fe4261a21f Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Tue, 17 Dec 2019 12:42:39 +0100 Subject: [PATCH 20/27] Twig tags: Preserve information on whitespace control (#147) * Add trimLeft and trimRight information to Twig tags Add test / update snapshots Add "applyExtension" method to Lexer and Parser * Add whitespace trimming information to autoescape block Update tests * Improve parse() function interface Simplify AutoescapeSpec.js * Add whitespace trimming information to BlockStatement * Add whitespace trimming information to embed * Add whitespace trimming information to filter block * Add whitespace trimming information to for loop * Add whitespace trimming information to if statement * Add whitespace trimming information to macro * Add whitespace information to mount tag * Add whitespace trimming information to set statement * Add whitespace trimming information to spaceless tag * Unify trimLeft... and trimRight... attribute names * Transform autoescape test to use snapshots like the other tests --- .../parser/autoescape.js | 24 +++++++++++- .../melody-extension-core/parser/block.js | 19 +++++++-- .../melody-extension-core/parser/embed.js | 23 ++++++++++- .../melody-extension-core/parser/filter.js | 23 +++++++++-- .../melody-extension-core/parser/for.js | 27 +++++++++++-- src/melody/melody-extension-core/parser/if.js | 37 +++++++++++++++++- .../melody-extension-core/parser/macro.js | 18 ++++++++- .../melody-extension-core/parser/mount.js | 25 ++++++++++++ .../melody-extension-core/parser/set.js | 19 ++++++++- .../melody-extension-core/parser/spaceless.js | 22 ++++++++++- src/melody/melody-parser/Lexer.js | 9 +++++ src/melody/melody-parser/Parser.js | 36 +++++++++++++++-- src/melody/melody-parser/index.js | 39 +++++++++++++++++-- src/melody/melody-parser/util.js | 21 ++++++++++ 14 files changed, 318 insertions(+), 24 deletions(-) diff --git a/src/melody/melody-extension-core/parser/autoescape.js b/src/melody/melody-extension-core/parser/autoescape.js index 0d718bcd..225b6a02 100644 --- a/src/melody/melody-extension-core/parser/autoescape.js +++ b/src/melody/melody-extension-core/parser/autoescape.js @@ -13,7 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { + Types, + setStartFromToken, + setEndFromToken, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, +} from 'melody-parser'; import { AutoescapeBlock } from './../types'; export const AutoescapeParser = { @@ -22,8 +28,11 @@ export const AutoescapeParser = { const tokens = parser.tokens; let escapeType = null, - stringStartToken; + stringStartToken, + openingTagEndToken, + closingTagStartToken; if (tokens.nextIf(Types.TAG_END)) { + openingTagEndToken = tokens.la(-1); escapeType = null; } else if ((stringStartToken = tokens.nextIf(Types.STRING_START))) { escapeType = tokens.expect(Types.STRING).text; @@ -39,10 +48,13 @@ I expected the current string to end with a ${ tokens.lat(0)}.`, }); } + openingTagEndToken = tokens.la(0); } else if (tokens.nextIf(Types.FALSE)) { escapeType = false; + openingTagEndToken = tokens.la(0); } else if (tokens.nextIf(Types.TRUE)) { escapeType = true; + openingTagEndToken = tokens.la(0); } else { parser.error({ title: 'Invalid autoescape type declaration', @@ -61,6 +73,7 @@ I expected the current string to end with a ${ token.type === Types.TAG_START && tokens.nextIf(Types.SYMBOL, 'endautoescape') ) { + closingTagStartToken = token; tagEndToken = tokens.expect(Types.TAG_END); return true; } @@ -68,6 +81,13 @@ I expected the current string to end with a ${ }).expressions; setEndFromToken(autoescape, tagEndToken); + autoescape.trimRightAutoescape = hasTagEndTokenTrimRight( + openingTagEndToken + ); + autoescape.trimLeftEndautoescape = hasTagStartTokenTrimLeft( + closingTagStartToken + ); + return autoescape; }, }; diff --git a/src/melody/melody-extension-core/parser/block.js b/src/melody/melody-extension-core/parser/block.js index f8b6c309..b06e6cdd 100644 --- a/src/melody/melody-extension-core/parser/block.js +++ b/src/melody/melody-extension-core/parser/block.js @@ -19,6 +19,8 @@ import { setStartFromToken, setEndFromToken, createNode, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, } from 'melody-parser'; import { BlockStatement } from './../types'; @@ -28,15 +30,19 @@ export const BlockParser = { const tokens = parser.tokens, nameToken = tokens.expect(Types.SYMBOL); - let blockStatement; - if (tokens.nextIf(Types.TAG_END)) { + let blockStatement, openingTagEndToken, closingTagStartToken; + if ((openingTagEndToken = tokens.nextIf(Types.TAG_END))) { blockStatement = new BlockStatement( createNode(Identifier, nameToken, nameToken.text), parser.parse((tokenText, token, tokens) => { - return !!( + const result = !!( token.type === Types.TAG_START && tokens.nextIf(Types.SYMBOL, 'endblock') ); + if (result) { + closingTagStartToken = token; + } + return result; }).expressions ); @@ -69,6 +75,13 @@ export const BlockParser = { setStartFromToken(blockStatement, token); setEndFromToken(blockStatement, tokens.expect(Types.TAG_END)); + blockStatement.trimRightBlock = + openingTagEndToken && hasTagEndTokenTrimRight(openingTagEndToken); + blockStatement.trimLeftEndblock = !!( + closingTagStartToken && + hasTagStartTokenTrimLeft(closingTagStartToken) + ); + return blockStatement; }, }; diff --git a/src/melody/melody-extension-core/parser/embed.js b/src/melody/melody-extension-core/parser/embed.js index bd4bc6e7..815e7fc7 100644 --- a/src/melody/melody-extension-core/parser/embed.js +++ b/src/melody/melody-extension-core/parser/embed.js @@ -14,7 +14,13 @@ * limitations under the License. */ import { Node } from 'melody-types'; -import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { + Types, + setStartFromToken, + setEndFromToken, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, +} from 'melody-parser'; import { filter } from 'lodash'; import { EmbedStatement } from './../types'; @@ -39,13 +45,19 @@ export const EmbedParser = { } tokens.expect(Types.TAG_END); + const openingTagEndToken = tokens.la(-1); + let closingTagStartToken; embedStatement.blocks = filter( parser.parse((tokenText, token, tokens) => { - return !!( + const result = !!( token.type === Types.TAG_START && tokens.nextIf(Types.SYMBOL, 'endembed') ); + if (result) { + closingTagStartToken = token; + } + return result; }).expressions, Node.isBlockStatement ); @@ -53,6 +65,13 @@ export const EmbedParser = { setStartFromToken(embedStatement, token); setEndFromToken(embedStatement, tokens.expect(Types.TAG_END)); + embedStatement.trimRightEmbed = hasTagEndTokenTrimRight( + openingTagEndToken + ); + embedStatement.trimLeftEndembed = + closingTagStartToken && + hasTagStartTokenTrimLeft(closingTagStartToken); + return embedStatement; }, }; diff --git a/src/melody/melody-extension-core/parser/filter.js b/src/melody/melody-extension-core/parser/filter.js index fdd01e62..535f1716 100644 --- a/src/melody/melody-extension-core/parser/filter.js +++ b/src/melody/melody-extension-core/parser/filter.js @@ -19,6 +19,8 @@ import { setStartFromToken, setEndFromToken, createNode, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, } from 'melody-parser'; import { FilterBlockStatement } from './../types'; @@ -29,11 +31,18 @@ export const FilterParser = { ref = createNode(Identifier, token, 'filter'), filterExpression = parser.matchFilterExpression(ref); tokens.expect(Types.TAG_END); + const openingTagEndToken = tokens.la(-1); + let closingTagStartToken; + const body = parser.parse((text, token, tokens) => { - return ( + const result = token.type === Types.TAG_START && - tokens.nextIf(Types.SYMBOL, 'endfilter') - ); + tokens.nextIf(Types.SYMBOL, 'endfilter'); + + if (result) { + closingTagStartToken = token; + } + return result; }).expressions; const filterBlockStatement = new FilterBlockStatement( @@ -42,6 +51,14 @@ export const FilterParser = { ); setStartFromToken(filterBlockStatement, token); setEndFromToken(filterBlockStatement, tokens.expect(Types.TAG_END)); + + filterBlockStatement.trimRightFilter = hasTagEndTokenTrimRight( + openingTagEndToken + ); + filterBlockStatement.trimLeftEndfilter = + closingTagStartToken && + hasTagStartTokenTrimLeft(closingTagStartToken); + return filterBlockStatement; }, }; diff --git a/src/melody/melody-extension-core/parser/for.js b/src/melody/melody-extension-core/parser/for.js index 86807ef2..2e18704c 100644 --- a/src/melody/melody-extension-core/parser/for.js +++ b/src/melody/melody-extension-core/parser/for.js @@ -19,6 +19,8 @@ import { setStartFromToken, setEndFromToken, createNode, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, } from 'melody-parser'; import { ForStatement } from './../types'; @@ -60,16 +62,23 @@ export const ForParser = { tokens.expect(Types.TAG_END); + const openingTagEndToken = tokens.la(-1); + let elseTagStartToken, elseTagEndToken; + forStatement.body = parser.parse((tokenText, token, tokens) => { - return ( + const result = token.type === Types.TAG_START && (tokens.test(Types.SYMBOL, 'else') || - tokens.test(Types.SYMBOL, 'endfor')) - ); + tokens.test(Types.SYMBOL, 'endfor')); + if (result && tokens.test(Types.SYMBOL, 'else')) { + elseTagStartToken = token; + } + return result; }); if (tokens.nextIf(Types.SYMBOL, 'else')) { tokens.expect(Types.TAG_END); + elseTagEndToken = tokens.la(-1); forStatement.otherwise = parser.parse( (tokenText, token, tokens) => { return ( @@ -79,11 +88,23 @@ export const ForParser = { } ); } + const endforTagStartToken = tokens.la(-1); tokens.expect(Types.SYMBOL, 'endfor'); setStartFromToken(forStatement, token); setEndFromToken(forStatement, tokens.expect(Types.TAG_END)); + forStatement.trimRightFor = hasTagEndTokenTrimRight(openingTagEndToken); + forStatement.trimLeftElse = !!( + elseTagStartToken && hasTagStartTokenTrimLeft(elseTagStartToken) + ); + forStatement.trimRightElse = !!( + elseTagEndToken && hasTagEndTokenTrimRight(elseTagEndToken) + ); + forStatement.trimLeftEndfor = hasTagStartTokenTrimLeft( + endforTagStartToken + ); + return forStatement; }, }; diff --git a/src/melody/melody-extension-core/parser/if.js b/src/melody/melody-extension-core/parser/if.js index 17e3c1d5..3409d5d0 100644 --- a/src/melody/melody-extension-core/parser/if.js +++ b/src/melody/melody-extension-core/parser/if.js @@ -13,7 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { + Types, + setStartFromToken, + setEndFromToken, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, +} from 'melody-parser'; import { IfStatement } from './../types'; export const IfParser = { @@ -24,25 +30,41 @@ export const IfParser = { alternate = null; tokens.expect(Types.TAG_END); + const ifTagEndToken = tokens.la(-1); const ifStatement = new IfStatement( test, parser.parse(matchConsequent).expressions ); + let elseTagStartToken, + elseTagEndToken, + elseifTagStartToken, + elseifTagEndToken; + do { if (tokens.nextIf(Types.SYMBOL, 'else')) { + elseTagStartToken = tokens.la(-2); tokens.expect(Types.TAG_END); + elseTagEndToken = tokens.la(-1); (alternate || ifStatement).alternate = parser.parse( matchAlternate ).expressions; } else if (tokens.nextIf(Types.SYMBOL, 'elseif')) { + elseifTagStartToken = tokens.la(-2); test = parser.matchExpression(); tokens.expect(Types.TAG_END); + elseifTagEndToken = tokens.la(-1); const consequent = parser.parse(matchConsequent).expressions; alternate = ( alternate || ifStatement ).alternate = new IfStatement(test, consequent); + alternate.trimLeft = hasTagStartTokenTrimLeft( + elseifTagStartToken + ); + alternate.trimRightIf = hasTagEndTokenTrimRight( + elseifTagEndToken + ); } if (tokens.nextIf(Types.SYMBOL, 'endif')) { @@ -50,9 +72,22 @@ export const IfParser = { } } while (!tokens.test(Types.EOF)); + const endifTagStartToken = tokens.la(-2); + setStartFromToken(ifStatement, token); setEndFromToken(ifStatement, tokens.expect(Types.TAG_END)); + ifStatement.trimRightIf = hasTagEndTokenTrimRight(ifTagEndToken); + ifStatement.trimLeftElse = !!( + elseTagStartToken && hasTagStartTokenTrimLeft(elseTagStartToken) + ); + ifStatement.trimRightElse = !!( + elseTagEndToken && hasTagEndTokenTrimRight(elseTagEndToken) + ); + ifStatement.trimLeftEndif = hasTagStartTokenTrimLeft( + endifTagStartToken + ); + return ifStatement; }, }; diff --git a/src/melody/melody-extension-core/parser/macro.js b/src/melody/melody-extension-core/parser/macro.js index 8420109d..427692ea 100644 --- a/src/melody/melody-extension-core/parser/macro.js +++ b/src/melody/melody-extension-core/parser/macro.js @@ -19,6 +19,8 @@ import { setStartFromToken, setEndFromToken, createNode, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, } from 'melody-parser'; import { MacroDeclarationStatement } from './../types'; @@ -47,11 +49,18 @@ export const MacroParser = { } tokens.expect(Types.RPAREN); + const openingTagEndToken = tokens.la(0); + let closingTagStartToken; + const body = parser.parse((tokenText, token, tokens) => { - return !!( + const result = !!( token.type === Types.TAG_START && tokens.nextIf(Types.SYMBOL, 'endmacro') ); + if (result) { + closingTagStartToken = token; + } + return result; }); if (tokens.test(Types.SYMBOL)) { @@ -78,6 +87,13 @@ export const MacroParser = { tokens.expect(Types.TAG_END) ); + macroDeclarationStatement.trimRightMacro = hasTagEndTokenTrimRight( + openingTagEndToken + ); + macroDeclarationStatement.trimLeftEndmacro = hasTagStartTokenTrimLeft( + closingTagStartToken + ); + return macroDeclarationStatement; }, }; diff --git a/src/melody/melody-extension-core/parser/mount.js b/src/melody/melody-extension-core/parser/mount.js index 5e06e585..32fafab3 100644 --- a/src/melody/melody-extension-core/parser/mount.js +++ b/src/melody/melody-extension-core/parser/mount.js @@ -20,6 +20,8 @@ import { setStartFromToken, setEndFromToken, createNode, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, } from 'melody-parser'; export const MountParser = { @@ -83,8 +85,14 @@ export const MountParser = { delayBy ); + let openingTagEndToken, + catchTagStartToken, + catchTagEndToken, + endmountTagStartToken; + if (async) { tokens.expect(Types.TAG_END); + openingTagEndToken = tokens.la(-1); mountStatement.body = parser.parse((tokenText, token, tokens) => { return ( @@ -95,6 +103,7 @@ export const MountParser = { }); if (tokens.nextIf(Types.SYMBOL, 'catch')) { + catchTagStartToken = tokens.la(-2); const errorVariableName = tokens.expect(Types.SYMBOL); mountStatement.errorVariableName = createNode( Identifier, @@ -102,6 +111,7 @@ export const MountParser = { errorVariableName.text ); tokens.expect(Types.TAG_END); + catchTagEndToken = tokens.la(-1); mountStatement.otherwise = parser.parse( (tokenText, token, tokens) => { return ( @@ -112,11 +122,26 @@ export const MountParser = { ); } tokens.expect(Types.SYMBOL, 'endmount'); + endmountTagStartToken = tokens.la(-2); } setStartFromToken(mountStatement, token); setEndFromToken(mountStatement, tokens.expect(Types.TAG_END)); + mountStatement.trimRightMount = !!( + openingTagEndToken && hasTagEndTokenTrimRight(openingTagEndToken) + ); + mountStatement.trimLeftCatch = !!( + catchTagStartToken && hasTagStartTokenTrimLeft(catchTagStartToken) + ); + mountStatement.trimRightCatch = !!( + catchTagEndToken && hasTagEndTokenTrimRight(catchTagEndToken) + ); + mountStatement.trimLeftEndmount = !!( + endmountTagStartToken && + hasTagStartTokenTrimLeft(endmountTagStartToken) + ); + return mountStatement; }, }; diff --git a/src/melody/melody-extension-core/parser/set.js b/src/melody/melody-extension-core/parser/set.js index c9fa8d7e..07b36f7e 100644 --- a/src/melody/melody-extension-core/parser/set.js +++ b/src/melody/melody-extension-core/parser/set.js @@ -19,6 +19,8 @@ import { setStartFromToken, setEndFromToken, createNode, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, } from 'melody-parser'; import { VariableDeclarationStatement, SetStatement } from './../types'; @@ -29,6 +31,8 @@ export const SetParser = { names = [], values = []; + let openingTagEndToken, closingTagStartToken; + do { const name = tokens.expect(Types.SYMBOL); names.push(createNode(Identifier, name, name.text)); @@ -48,12 +52,17 @@ export const SetParser = { }); } tokens.expect(Types.TAG_END); + openingTagEndToken = tokens.la(-1); values[0] = parser.parse((tokenText, token, tokens) => { - return !!( + const result = !!( token.type === Types.TAG_START && tokens.nextIf(Types.SYMBOL, 'endset') ); + if (result) { + closingTagStartToken = token; + } + return result; }).expressions; } @@ -81,6 +90,14 @@ ${names.length} variable names and ${values.length} values.`, setStartFromToken(setStatement, token); setEndFromToken(setStatement, tokens.expect(Types.TAG_END)); + setStatement.trimRightSet = !!( + openingTagEndToken && hasTagEndTokenTrimRight(openingTagEndToken) + ); + setStatement.trimLeftEndset = !!( + closingTagStartToken && + hasTagStartTokenTrimLeft(closingTagStartToken) + ); + return setStatement; }, }; diff --git a/src/melody/melody-extension-core/parser/spaceless.js b/src/melody/melody-extension-core/parser/spaceless.js index 1cb154b3..d9dd6a52 100644 --- a/src/melody/melody-extension-core/parser/spaceless.js +++ b/src/melody/melody-extension-core/parser/spaceless.js @@ -13,7 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Types, setStartFromToken, setEndFromToken } from 'melody-parser'; +import { + Types, + setStartFromToken, + setEndFromToken, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, +} from 'melody-parser'; import { SpacelessBlock } from './../types'; export const SpacelessParser = { @@ -22,18 +28,30 @@ export const SpacelessParser = { const tokens = parser.tokens; tokens.expect(Types.TAG_END); + const openingTagEndToken = tokens.la(-1); + let closingTagStartToken; const body = parser.parse((tokenText, token, tokens) => { - return !!( + const result = !!( token.type === Types.TAG_START && tokens.nextIf(Types.SYMBOL, 'endspaceless') ); + closingTagStartToken = token; + return result; }).expressions; const spacelessBlock = new SpacelessBlock(body); setStartFromToken(spacelessBlock, token); setEndFromToken(spacelessBlock, tokens.expect(Types.TAG_END)); + spacelessBlock.trimRightSpaceless = hasTagEndTokenTrimRight( + openingTagEndToken + ); + spacelessBlock.trimLeftEndspaceless = !!( + closingTagStartToken && + hasTagStartTokenTrimLeft(closingTagStartToken) + ); + return spacelessBlock; }, }; diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js index bc2196f5..ed6d2981 100644 --- a/src/melody/melody-parser/Lexer.js +++ b/src/melody/melody-parser/Lexer.js @@ -57,6 +57,15 @@ export default class Lexer { this[STRING_START] = null; } + applyExtension(ext) { + if (ext.unaryOperators) { + this.addOperators(...ext.unaryOperators.map(op => op.text)); + } + if (ext.binaryOperators) { + this.addOperators(...ext.binaryOperators.map(op => op.text)); + } + } + reset() { this.input.reset(); this[STATE] = [State.TEXT]; diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 09bc3f86..f2e04a2e 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -63,6 +63,29 @@ export default class Parser { ); } + applyExtension(ext) { + if (ext.tags) { + for (const tag of ext.tags) { + this.addTag(tag); + } + } + if (ext.unaryOperators) { + for (const op of ext.unaryOperators) { + this.addUnaryOperator(op); + } + } + if (ext.binaryOperators) { + for (const op of ext.binaryOperators) { + this.addBinaryOperator(op); + } + } + if (ext.tests) { + for (const test of ext.tests) { + this.addTest(test); + } + } + } + addUnaryOperator(op: UnaryOperator) { this[UNARY][op.text] = op; return this; @@ -324,8 +347,10 @@ export default class Parser { } matchTag() { - let tokens = this.tokens, - tag = tokens.expect(Types.SYMBOL), + const tokens = this.tokens; + const tagStartToken = tokens.la(-1); + + const tag = tokens.expect(Types.SYMBOL), parser = this[TAG][tag.text]; if (!parser) { tokens.error( @@ -337,7 +362,12 @@ export default class Parser { tag.length ); } - return parser.parse(this, tag); + + const result = parser.parse(this, tag); + const tagEndToken = tokens.la(-1); + result.trimLeft = tagStartToken.text.endsWith('-'); + result.trimRight = tagEndToken.text.startsWith('-'); + return result; } matchExpression(precedence = 0) { diff --git a/src/melody/melody-parser/index.js b/src/melody/melody-parser/index.js index 25a28e97..e5cebc15 100644 --- a/src/melody/melody-parser/index.js +++ b/src/melody/melody-parser/index.js @@ -26,11 +26,40 @@ import { copyEnd, copyLoc, createNode, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, + isMelodyExtension, } from './util'; -function parse(code) { - const p = new Parser(new TokenStream(new Lexer(new CharStream(code)))); - return p.parse(); +function parse(code, options, ...extensions) { + return createExtendedParser(code, options, ...extensions).parse(); +} + +function createExtendedParser(code, options, ...extensions) { + let passedOptions = options; + const passedExtensions = extensions; + if (isMelodyExtension(options)) { + // Variant without options parameter: createExtendedParser(code, ...extensions) + passedOptions = undefined; + passedExtensions.unshift(options); + } + const lexer = createExtendedLexer(code, ...passedExtensions); + const parser = new Parser( + new TokenStream(lexer, passedOptions), + passedOptions + ); + for (const ext of passedExtensions) { + parser.applyExtension(ext); + } + return parser; +} + +function createExtendedLexer(code, ...extensions) { + const lexer = new Lexer(new CharStream(code)); + for (const ext of extensions) { + lexer.applyExtension(ext); + } + return lexer; } export { @@ -42,11 +71,15 @@ export { LEFT, RIGHT, parse, + createExtendedLexer, + createExtendedParser, setStartFromToken, setEndFromToken, copyStart, copyEnd, copyLoc, createNode, + hasTagStartTokenTrimLeft, + hasTagEndTokenTrimRight, Types, }; diff --git a/src/melody/melody-parser/util.js b/src/melody/melody-parser/util.js index 96269619..38d3de18 100644 --- a/src/melody/melody-parser/util.js +++ b/src/melody/melody-parser/util.js @@ -61,3 +61,24 @@ export function createNode(Type, token, ...args) { export function startNode(Type, token, ...args) { return setStartFromToken(new Type(...args), token); } + +export function hasTagStartTokenTrimLeft(token) { + return token.text.endsWith('-'); +} + +export function hasTagEndTokenTrimRight(token) { + return token.text.startsWith('-'); +} + +export function isMelodyExtension(obj) { + return ( + obj && + (Array.isArray(obj.binaryOperators) || + typeof obj.filterMap === 'object' || + typeof obj.functionMap === 'object' || + Array.isArray(obj.tags) || + Array.isArray(obj.tests) || + Array.isArray(obj.unaryOperators) || + Array.isArray(obj.visitors)) + ); +} From 2f32c2dc3d9febce31b2eaa54103bfc02f5541f3 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Wed, 8 Jan 2020 10:20:50 +0100 Subject: [PATCH 21/27] Add parsing of declarations like (#150) * Add parsing of declarations like * Update CHANGELOG --- src/melody/melody-parser/Lexer.js | 27 +++++++++++ src/melody/melody-parser/Parser.js | 67 ++++++++++++++++++++++++++ src/melody/melody-parser/TokenTypes.js | 1 + src/melody/melody-types/index.js | 10 ++++ 4 files changed, 105 insertions(+) diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js index ed6d2981..089a4e74 100644 --- a/src/melody/melody-parser/Lexer.js +++ b/src/melody/melody-parser/Lexer.js @@ -25,6 +25,7 @@ const State = { STRING_DOUBLE: 'STRING_DOUBLE', ELEMENT: 'ELEMENT', ATTRIBUTE_VALUE: 'ATTRIBUTE_VALUE', + DECLARATION: 'DECLARATION', }; const STATE = Symbol(), @@ -189,6 +190,17 @@ export default class Lexer { input.next(); } return this.createToken(TokenTypes.HTML_COMMENT, pos); + } else if ( + input.la(1) === '!' && + (isAlpha(input.lac(2)) || isWhitespace(input.la(2))) + ) { + input.next(); + input.next(); + this.pushState(State.DECLARATION); + return this.createToken( + TokenTypes.DECLARATION_START, + pos + ); } else { return this.matchText(pos); } @@ -268,6 +280,21 @@ export default class Lexer { } else { return this.matchAttributeValue(pos); } + } else if (this.state === State.DECLARATION) { + switch (c) { + case '>': + input.next(); + this.popState(); + return this.createToken(TokenTypes.ELEMENT_END, pos); + case '"': + input.next(); + this.pushState(State.STRING_DOUBLE); + return this.createToken(TokenTypes.STRING_START, pos); + case '{': + return this.matchExpressionToken(pos); + default: + return this.matchSymbol(pos); + } } else { return this.error(`Invalid state ${this.state}`, pos); } diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index f2e04a2e..e5486f59 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -57,6 +57,7 @@ export default class Parser { { ignoreComments: true, ignoreHtmlComments: true, + ignoreDeclarations: true, decodeEntities: true, }, options @@ -175,6 +176,13 @@ export default class Parser { case Types.ELEMENT_START: p.add(this.matchElement()); break; + case Types.DECLARATION_START: { + const declarationNode = this.matchDeclaration(); + if (!this.options.ignoreDeclarations) { + p.add(declarationNode); + } + break; + } case Types.COMMENT: if (!this.options.ignoreComments) { p.add( @@ -202,6 +210,65 @@ export default class Parser { return p; } + /** + * e.g., + */ + matchDeclaration() { + const tokens = this.tokens, + declarationStartToken = tokens.la(-1); + let declarationType = null, + currentToken = null; + + if (!(declarationType = tokens.nextIf(Types.SYMBOL))) { + this.error({ + title: 'Expected declaration start', + pos: declarationStartToken.pos, + advice: + "After '' (children)* '<' '/' SYMBOL '>' * attributes = SYMBOL '=' (matchExpression | matchString) diff --git a/src/melody/melody-parser/TokenTypes.js b/src/melody/melody-parser/TokenTypes.js index 639c1127..49c4e862 100644 --- a/src/melody/melody-parser/TokenTypes.js +++ b/src/melody/melody-parser/TokenTypes.js @@ -21,6 +21,7 @@ export const INTERPOLATION_START = 'interpolationStart'; export const INTERPOLATION_END = 'interpolationEnd'; export const STRING_START = 'stringStart'; export const STRING_END = 'stringEnd'; +export const DECLARATION_START = 'declarationStart'; export const COMMENT = 'comment'; export const WHITESPACE = 'whitespace'; export const HTML_COMMENT = 'htmlComment'; diff --git a/src/melody/melody-types/index.js b/src/melody/melody-types/index.js index 6d04b20b..0e5c8a0a 100644 --- a/src/melody/melody-types/index.js +++ b/src/melody/melody-types/index.js @@ -377,3 +377,13 @@ export class HtmlComment extends Node { } type(HtmlComment, 'HtmlComment'); visitor(HtmlComment, 'value'); + +export class Declaration extends Node { + constructor(declarationType: String) { + super(); + this.declarationType = declarationType; + this.parts = []; + } +} +type(Declaration, 'Declaration'); +visitor(Declaration, 'parts'); From 0c504be840fdaeba957069293b4990333531a56c Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Wed, 8 Jan 2020 11:30:59 +0100 Subject: [PATCH 22/27] Parser: More precise location information (#148) * Improve location information on AST nodes * Add `getNodeSource()` method on the melody-parser package * Update and add test cases --- src/melody/melody-code-frame/index.js | 2 +- src/melody/melody-parser/Parser.js | 150 ++++++++++++++++---------- src/melody/melody-parser/index.js | 2 + src/melody/melody-parser/util.js | 16 +++ 4 files changed, 112 insertions(+), 58 deletions(-) diff --git a/src/melody/melody-code-frame/index.js b/src/melody/melody-code-frame/index.js index 7b37dbec..603cd96d 100644 --- a/src/melody/melody-code-frame/index.js +++ b/src/melody/melody-code-frame/index.js @@ -32,7 +32,7 @@ export default function({ rawLines, lineNumber, colNumber, length }) { return; } - if (colNumber) { + if (typeof colNumber === 'number') { params.line += `\n${params.before}${repeat(' ', params.width)}${ params.after }${repeat(' ', colNumber)}${repeat('^', length)}`; diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index e5486f59..2a5b77f5 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -19,6 +19,7 @@ import { LEFT, RIGHT } from './Associativity'; import { setStartFromToken, setEndFromToken, + setMarkFromToken, copyStart, copyEnd, copyLoc, @@ -137,42 +138,50 @@ export default class Parser { switch (token.type) { case Types.EXPRESSION_START: { const expression = this.matchExpression(); - p.add( - copyLoc( - new n.PrintExpressionStatement(expression), - expression - ) + const statement = new n.PrintExpressionStatement( + expression ); - setEndFromToken(p, tokens.expect(Types.EXPRESSION_END)); + const endToken = tokens.expect(Types.EXPRESSION_END); + setStartFromToken(statement, token); + setEndFromToken(statement, endToken); + setEndFromToken(p, endToken); + p.add(statement); + break; } case Types.TAG_START: p.add(this.matchTag()); break; - case Types.TEXT: - p.add( - createNode( - n.PrintTextStatement, - token, - createNode(n.StringLiteral, token, token.text) - ) + case Types.TEXT: { + const textStringLiteral = createNode( + n.StringLiteral, + token, + token.text ); + const textTextStatement = createNode( + n.PrintTextStatement, + token, + textStringLiteral + ); + p.add(textTextStatement); break; - case Types.ENTITY: - p.add( - createNode( - n.PrintTextStatement, - token, - createNode( - n.StringLiteral, - token, - this.options.decodeEntities - ? he.decode(token.text) - : token.text - ) - ) + } + case Types.ENTITY: { + const entityStringLiteral = createNode( + n.StringLiteral, + token, + this.options.decodeEntities + ? he.decode(token.text) + : token.text + ); + const entityTextStatement = createNode( + n.PrintTextStatement, + token, + entityStringLiteral ); + p.add(entityTextStatement); break; + } case Types.ELEMENT_START: p.add(this.matchElement()); break; @@ -185,24 +194,32 @@ export default class Parser { } case Types.COMMENT: if (!this.options.ignoreComments) { - p.add( - createNode( - n.TwigComment, - token, - createNode(n.StringLiteral, token, token.text) - ) + const stringLiteral = createNode( + n.StringLiteral, + token, + token.text + ); + const twigComment = createNode( + n.TwigComment, + token, + stringLiteral ); + p.add(twigComment); } break; case Types.HTML_COMMENT: if (!this.options.ignoreHtmlComments) { - p.add( - createNode( - n.HtmlComment, - token, - createNode(n.StringLiteral, token, token.text) - ) + const stringLiteral = createNode( + n.StringLiteral, + token, + token.text + ); + const htmlComment = createNode( + n.HtmlComment, + token, + stringLiteral ); + p.add(htmlComment); } break; } @@ -275,14 +292,14 @@ export default class Parser { * | matchExpression */ matchElement() { - let tokens = this.tokens, - elementStartToken = tokens.la(0), - elementName, - element; + const tokens = this.tokens, + elementNameToken = tokens.la(0), + tagStartToken = tokens.la(-1); + let elementName; if (!(elementName = tokens.nextIf(Types.SYMBOL))) { this.error({ title: 'Expected element start', - pos: elementStartToken.pos, + pos: elementNameToken.pos, advice: tokens.lat(0) === Types.SLASH ? `Unexpected closing "${ @@ -292,8 +309,7 @@ export default class Parser { }); } - element = new n.Element(elementName.text); - setStartFromToken(element, elementStartToken); + const element = new n.Element(elementName.text); this.matchAttributes(element, tokens); @@ -325,7 +341,11 @@ export default class Parser { }).expressions; } } + + setStartFromToken(element, tagStartToken); setEndFromToken(element, tokens.la(-1)); + setMarkFromToken(element, 'elementNameLoc', elementNameToken); + return element; } @@ -369,7 +389,8 @@ export default class Parser { } tokens.expect(Types.STRING_END); if (!nodes.length) { - nodes.push(createNode(n.StringLiteral, start, '')); + const node = createNode(n.StringLiteral, start, ''); + nodes.push(node); } let expr = nodes[0]; @@ -415,7 +436,8 @@ export default class Parser { matchTag() { const tokens = this.tokens; - const tagStartToken = tokens.la(-1); + const tagStartToken = tokens.la(-1), + tagNameToken = tokens.la(0); const tag = tokens.expect(Types.SYMBOL), parser = this[TAG][tag.text]; @@ -434,11 +456,17 @@ export default class Parser { const tagEndToken = tokens.la(-1); result.trimLeft = tagStartToken.text.endsWith('-'); result.trimRight = tagEndToken.text.startsWith('-'); + + setStartFromToken(result, tagStartToken); + setEndFromToken(result, tagEndToken); + setMarkFromToken(result, 'tagNameLoc', tagNameToken); + return result; } matchExpression(precedence = 0) { - const tokens = this.tokens; + const tokens = this.tokens, + exprStartToken = tokens.la(0); let token, op, trimLeft = false; @@ -472,6 +500,9 @@ export default class Parser { token = tokens.la(0); } + if (precedence === 0) { + setEndFromToken(expr, tokens.la(-1)); + } const result = precedence === 0 ? this.matchConditionalExpression(expr) : expr; @@ -483,6 +514,10 @@ export default class Parser { result.trimLeft = trimLeft; } + const exprEndToken = tokens.la(-1); + setStartFromToken(result, exprStartToken); + setEndFromToken(result, exprEndToken); + return result; } @@ -566,13 +601,11 @@ export default class Parser { } matchStringExpression() { - let tokens = this.tokens, + let canBeString = true, + token; + const tokens = this.tokens, nodes = [], - canBeString = true, - token, - stringStart, - stringEnd; - stringStart = tokens.expect(Types.STRING_START); + stringStart = tokens.expect(Types.STRING_START); while (!tokens.test(Types.STRING_END)) { if (canBeString && (token = tokens.nextIf(Types.STRING))) { nodes[nodes.length] = createNode( @@ -589,7 +622,7 @@ export default class Parser { break; } } - stringEnd = tokens.expect(Types.STRING_END); + const stringEnd = tokens.expect(Types.STRING_END); if (!nodes.length) { return setEndFromToken( @@ -611,12 +644,15 @@ export default class Parser { expr.wasImplicitConcatenation = true; } + setStartFromToken(expr, stringStart); + setEndFromToken(expr, stringEnd); + return expr; } matchConditionalExpression(test: Node) { - let tokens = this.tokens, - condition = test, + const tokens = this.tokens; + let condition = test, consequent, alternate; while (tokens.nextIf(Types.QUESTION_MARK)) { diff --git a/src/melody/melody-parser/index.js b/src/melody/melody-parser/index.js index e5cebc15..f224444b 100644 --- a/src/melody/melody-parser/index.js +++ b/src/melody/melody-parser/index.js @@ -25,6 +25,7 @@ import { copyStart, copyEnd, copyLoc, + getNodeSource, createNode, hasTagStartTokenTrimLeft, hasTagEndTokenTrimRight, @@ -78,6 +79,7 @@ export { copyStart, copyEnd, copyLoc, + getNodeSource, createNode, hasTagStartTokenTrimLeft, hasTagEndTokenTrimRight, diff --git a/src/melody/melody-parser/util.js b/src/melody/melody-parser/util.js index 38d3de18..fb955289 100644 --- a/src/melody/melody-parser/util.js +++ b/src/melody/melody-parser/util.js @@ -23,6 +23,15 @@ export function setEndFromToken(node, { pos: { line, column }, end }) { return node; } +export function setMarkFromToken( + node, + propertyName, + { pos: { index, line, column } } +) { + node[propertyName] = { line, column, index }; + return node; +} + export function copyStart( node, { @@ -44,6 +53,13 @@ export function copyEnd(node, end) { return node; } +export function getNodeSource(node, entireSource) { + if (entireSource && node.loc.start && node.loc.end) { + return entireSource.substring(node.loc.start.index, node.loc.end.index); + } + return ''; +} + export function copyLoc(node, { loc: { start, end } }) { node.loc.start.line = start.line; node.loc.start.column = start.column; From 673aac62bbd49898678fc146d9085e3245401647 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Thu, 9 Jan 2020 11:25:07 +0100 Subject: [PATCH 23/27] Add missing trim properties on PrintExpressionStatement (#152) * Add missing trimLeft and trimRight properties on PrintExpressionStatement nodes * Add test case covering the changes * Update CHANGELOG --- src/melody/melody-parser/Parser.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 2a5b77f5..a5859e91 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -145,6 +145,8 @@ export default class Parser { setStartFromToken(statement, token); setEndFromToken(statement, endToken); setEndFromToken(p, endToken); + statement.trimLeft = !!expression.trimLeft; + statement.trimRight = !!expression.trimRight; p.add(statement); break; From e636dcb74e62d267e405d4ee9523ceec63fdafda Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Fri, 10 Jan 2020 06:35:11 +0100 Subject: [PATCH 24/27] Bugfix: Prevent Number tokens from ending in a dot "." (#153) * Add failing test case * Keep matchNumber() from ending a Number token with a dot * Update CHANGELOG --- src/melody/melody-parser/Lexer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js index 089a4e74..fe598bf1 100644 --- a/src/melody/melody-parser/Lexer.js +++ b/src/melody/melody-parser/Lexer.js @@ -587,7 +587,7 @@ export default class Lexer { } input.next(); } - if (input.la(0) === '.') { + if (input.la(0) === '.' && isDigit(input.lac(1))) { input.next(); while ((c = input.lac(0)) !== EOF) { if (!isDigit(c)) { From 19695795377b4b9283f8e158e843034a6e91e4c7 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Sat, 11 Jan 2020 23:05:40 +0100 Subject: [PATCH 25/27] Bugfix: Preserve backslashes in string literals (#149) * Fix bug where all backslashes in strings are removed * Introduce "preserveSourceLiterally" option * Update README and CHANGELOG * Add Lexer test case with escaped newline --- src/melody/melody-parser/Lexer.js | 22 +++++++++++++++++++--- src/melody/melody-parser/Parser.js | 8 +++++--- src/melody/melody-parser/index.js | 15 +++++++++++---- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/melody/melody-parser/Lexer.js b/src/melody/melody-parser/Lexer.js index fe598bf1..40dd2d91 100644 --- a/src/melody/melody-parser/Lexer.js +++ b/src/melody/melody-parser/Lexer.js @@ -51,11 +51,15 @@ const CHAR_TO_TOKEN = { }; export default class Lexer { - constructor(input) { + constructor(input, { preserveSourceLiterally = false } = {}) { this.input = input; this[STATE] = [State.TEXT]; this[OPERATORS] = []; this[STRING_START] = null; + this.options = { + preserveSourceLiterally: + preserveSourceLiterally === true ? true : false, + }; } applyExtension(ext) { @@ -549,7 +553,13 @@ export default class Lexer { } } var result = this.createToken(TokenTypes.STRING, pos); - result.text = result.text.replace('\\', ''); + // Replace double backslash before escaped quotes + if (!this.options.preserveSourceLiterally) { + result.text = result.text.replace( + new RegExp('(?:\\\\)(' + start + ')', 'g'), + '$1' + ); + } return result; } @@ -574,7 +584,13 @@ export default class Lexer { } } var result = this.createToken(TokenTypes.STRING, pos); - result.text = result.text.replace('\\', ''); + // Replace double backslash before escaped quotes + if (!this.options.preserveSourceLiterally) { + result.text = result.text.replace( + new RegExp('(?:\\\\)(' + start + ')', 'g'), + '$1' + ); + } return result; } diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index a5859e91..9a093a29 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -60,6 +60,7 @@ export default class Parser { ignoreHtmlComments: true, ignoreDeclarations: true, decodeEntities: true, + preserveSourceLiterally: false, }, options ); @@ -172,9 +173,10 @@ export default class Parser { const entityStringLiteral = createNode( n.StringLiteral, token, - this.options.decodeEntities - ? he.decode(token.text) - : token.text + !this.options.decodeEntities || + this.options.preserveSourceLiterally + ? token.text + : he.decode(token.text) ); const entityTextStatement = createNode( n.PrintTextStatement, diff --git a/src/melody/melody-parser/index.js b/src/melody/melody-parser/index.js index f224444b..d523606e 100644 --- a/src/melody/melody-parser/index.js +++ b/src/melody/melody-parser/index.js @@ -44,7 +44,7 @@ function createExtendedParser(code, options, ...extensions) { passedOptions = undefined; passedExtensions.unshift(options); } - const lexer = createExtendedLexer(code, ...passedExtensions); + const lexer = createExtendedLexer(code, options, ...passedExtensions); const parser = new Parser( new TokenStream(lexer, passedOptions), passedOptions @@ -55,9 +55,16 @@ function createExtendedParser(code, options, ...extensions) { return parser; } -function createExtendedLexer(code, ...extensions) { - const lexer = new Lexer(new CharStream(code)); - for (const ext of extensions) { +function createExtendedLexer(code, options, ...extensions) { + let passedOptions = options; + const passedExtensions = extensions; + if (isMelodyExtension(options)) { + // Variant without options parameter: createExtendedLexer(code, ...extensions) + passedOptions = undefined; + passedExtensions.unshift(options); + } + const lexer = new Lexer(new CharStream(code), passedOptions); + for (const ext of passedExtensions) { lexer.applyExtension(ext); } return lexer; From d0f4e7a415a7c0b8244fe40eca0c73fb3fbbd705 Mon Sep 17 00:00:00 2001 From: Thomas Bartel Date: Thu, 27 Feb 2020 12:54:27 +0100 Subject: [PATCH 26/27] Introduce generic twig tags (#154) * Introduce - GenericTwigTag - GenericTagParser - Option allowUnknownTags * Add test cases (from Craft CMS) Enable "unexpected" tokens in generic tag * Add multi-tag parsing for unknown tags * Fix location information for generic twig tags * Set allowUnknownTags option to true when custom multiTags are provided * Update README to document the use of `allowUnknownTags` and `multiTags` --- .../melody-parser/GenericMultiTagParser.js | 66 ++++++++++++++ src/melody/melody-parser/GenericTagParser.js | 53 +++++++++++ src/melody/melody-parser/Parser.js | 91 +++++++++++++------ src/melody/melody-parser/TokenStream.js | 6 +- src/melody/melody-types/index.js | 19 ++++ 5 files changed, 207 insertions(+), 28 deletions(-) create mode 100644 src/melody/melody-parser/GenericMultiTagParser.js create mode 100644 src/melody/melody-parser/GenericTagParser.js diff --git a/src/melody/melody-parser/GenericMultiTagParser.js b/src/melody/melody-parser/GenericMultiTagParser.js new file mode 100644 index 00000000..6aef6149 --- /dev/null +++ b/src/melody/melody-parser/GenericMultiTagParser.js @@ -0,0 +1,66 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { setStartFromToken, setEndFromToken } from './util'; +import { GenericTagParser } from './GenericTagParser'; +import * as Types from './TokenTypes'; + +const tagMatchesOneOf = (tokenStream, tagNames) => { + for (let i = 0; i < tagNames.length; i++) { + if (tokenStream.test(Types.SYMBOL, tagNames[i])) { + return true; + } + } + return false; +}; + +export const createMultiTagParser = (tagName, subTags = []) => ({ + name: 'genericTwigMultiTag', + parse(parser, token) { + const tokens = parser.tokens, + tagStartToken = tokens.la(-1); + + if (subTags.length === 0) { + subTags.push('end' + tagName); + } + + const twigTag = GenericTagParser.parse(parser, token); + let currentTagName = tagName; + const endTagName = subTags[subTags.length - 1]; + + while (currentTagName !== endTagName) { + // Parse next section + twigTag.sections.push( + parser.parse((tokenText, token, tokens) => { + const hasReachedNextTag = + token.type === Types.TAG_START && + tagMatchesOneOf(tokens, subTags); + return hasReachedNextTag; + }) + ); + tokens.next(); // Get past "{%" + + // Parse next tag + const childTag = GenericTagParser.parse(parser); + twigTag.sections.push(childTag); + currentTagName = childTag.tagName; + } + + setStartFromToken(twigTag, tagStartToken); + setEndFromToken(twigTag, tokens.la(0)); + + return twigTag; + }, +}); diff --git a/src/melody/melody-parser/GenericTagParser.js b/src/melody/melody-parser/GenericTagParser.js new file mode 100644 index 00000000..29a25690 --- /dev/null +++ b/src/melody/melody-parser/GenericTagParser.js @@ -0,0 +1,53 @@ +/** + * Copyright 2017 trivago N.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { hasTagStartTokenTrimLeft, hasTagEndTokenTrimRight } from './util'; +import * as Types from './TokenTypes'; +import * as n from 'melody-types'; + +export const GenericTagParser = { + name: 'genericTwigTag', + parse(parser) { + const tokens = parser.tokens, + tagStartToken = tokens.la(-2); + let currentToken; + + const twigTag = new n.GenericTwigTag(tokens.la(-1).text); + while ((currentToken = tokens.la(0))) { + if (currentToken.type === Types.TAG_END) { + break; + } else { + try { + twigTag.parts.push(parser.matchExpression()); + } catch (e) { + if (e.errorType === 'UNEXPECTED_TOKEN') { + twigTag.parts.push( + new n.GenericToken(e.tokenType, e.tokenText) + ); + tokens.next(); + } else { + throw e; + } + } + } + } + tokens.expect(Types.TAG_END); + + twigTag.trimLeft = hasTagStartTokenTrimLeft(tagStartToken); + twigTag.trimRight = hasTagEndTokenTrimRight(currentToken); + + return twigTag; + }, +}; diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 9a093a29..65fb4d42 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -25,6 +25,8 @@ import { copyLoc, createNode, } from './util'; +import { GenericTagParser } from './GenericTagParser'; +import { createMultiTagParser } from './GenericMultiTagParser'; import { voidElements } from './elementInfo'; import * as he from 'he'; @@ -61,9 +63,15 @@ export default class Parser { ignoreDeclarations: true, decodeEntities: true, preserveSourceLiterally: false, + allowUnknownTags: false, + multiTags: {}, // e.g. { "nav": ["endnav"], "switch": ["case", "default", "endswitch"]} }, options ); + // If there are custom multi tags, then we allow all custom tags + if (Object.keys(this.options.multiTags).length > 0) { + this.options.allowUnknownTags = true; + } } applyExtension(ext) { @@ -434,36 +442,60 @@ export default class Parser { } } - error(options) { - this.tokens.error(options.title, options.pos, options.advice); + error(options, metadata = {}) { + this.tokens.error( + options.title, + options.pos, + options.advice, + 1, + metadata + ); + } + + getGenericParserFor(tagName) { + if (this.options.multiTags[tagName]) { + return createMultiTagParser( + tagName, + this.options.multiTags[tagName] + ); + } else { + return GenericTagParser; + } } matchTag() { const tokens = this.tokens; - const tagStartToken = tokens.la(-1), - tagNameToken = tokens.la(0); + const tagStartToken = tokens.la(-1); - const tag = tokens.expect(Types.SYMBOL), - parser = this[TAG][tag.text]; + const tag = tokens.expect(Types.SYMBOL); + let parser = this[TAG][tag.text]; + let isUsingGenericParser = false; if (!parser) { - tokens.error( - `Unknown tag "${tag.text}"`, - tag.pos, - `Expected a known tag such as\n- ${Object.getOwnPropertyNames( - this[TAG] - ).join('\n- ')}`, - tag.length - ); + if (this.options.allowUnknownTags) { + parser = this.getGenericParserFor(tag.text); + isUsingGenericParser = true; + } else { + tokens.error( + `Unknown tag "${tag.text}"`, + tag.pos, + `Expected a known tag such as\n- ${Object.getOwnPropertyNames( + this[TAG] + ).join('\n- ')}`, + tag.length + ); + } } const result = parser.parse(this, tag); const tagEndToken = tokens.la(-1); - result.trimLeft = tagStartToken.text.endsWith('-'); - result.trimRight = tagEndToken.text.startsWith('-'); + if (!isUsingGenericParser) { + result.trimLeft = tagStartToken.text.endsWith('-'); + result.trimRight = tagEndToken.text.startsWith('-'); + } setStartFromToken(result, tagStartToken); setEndFromToken(result, tagEndToken); - setMarkFromToken(result, 'tagNameLoc', tagNameToken); + setMarkFromToken(result, 'tagNameLoc', tag); return result; } @@ -588,15 +620,22 @@ export default class Parser { } else if (token.type === Types.LBRACKET) { node = this.matchMap(); } else { - this.error({ - title: - 'Unexpected token "' + - token.type + - '" of value "' + - token.text + - '"', - pos: token.pos, - }); + this.error( + { + title: + 'Unexpected token "' + + token.type + + '" of value "' + + token.text + + '"', + pos: token.pos, + }, + { + errorType: 'UNEXPECTED_TOKEN', + tokenText: token.text, + tokenType: token.type, + } + ); } break; } diff --git a/src/melody/melody-parser/TokenStream.js b/src/melody/melody-parser/TokenStream.js index 4411be70..83f66558 100644 --- a/src/melody/melody-parser/TokenStream.js +++ b/src/melody/melody-parser/TokenStream.js @@ -112,7 +112,7 @@ export default class TokenStream { ); } - error(message, pos, advice, length = 1) { + error(message, pos, advice, length = 1, metadata = {}) { let errorMessage = `ERROR: ${message}\n`; errorMessage += codeFrame({ rawLines: this.input.source, @@ -128,7 +128,9 @@ export default class TokenStream { if (advice) { errorMessage += '\n\n' + advice; } - throw new Error(errorMessage); + const result = new Error(errorMessage); + Object.assign(result, metadata); + throw result; } } diff --git a/src/melody/melody-types/index.js b/src/melody/melody-types/index.js index 0e5c8a0a..3283763a 100644 --- a/src/melody/melody-types/index.js +++ b/src/melody/melody-types/index.js @@ -387,3 +387,22 @@ export class Declaration extends Node { } type(Declaration, 'Declaration'); visitor(Declaration, 'parts'); + +export class GenericTwigTag extends Node { + constructor(tagName: String) { + super(); + this.tagName = tagName; + this.parts = []; + this.sections = []; + } +} +type(GenericTwigTag, 'GenericTwigTag'); + +export class GenericToken extends Node { + constructor(tokenType: String, tokenText: String) { + super(); + this.tokenType = tokenType; + this.tokenText = tokenText; + } +} +type(GenericToken, 'GenericToken'); From af7a06757530385241c44f0e34390b5ee21bc18e Mon Sep 17 00:00:00 2001 From: Martin Mai Date: Fri, 29 May 2020 07:21:28 +0200 Subject: [PATCH 27/27] Retain trimming information for conditional expressions (#162) --- src/melody/melody-parser/Parser.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/melody/melody-parser/Parser.js b/src/melody/melody-parser/Parser.js index 65fb4d42..79359565 100644 --- a/src/melody/melody-parser/Parser.js +++ b/src/melody/melody-parser/Parser.js @@ -536,11 +536,13 @@ export default class Parser { token = tokens.la(0); } + var result = expr; if (precedence === 0) { setEndFromToken(expr, tokens.la(-1)); + result = this.matchConditionalExpression(expr); + // Update the local token variable because the stream pointer already advanced. + token = tokens.la(0); } - const result = - precedence === 0 ? this.matchConditionalExpression(expr) : expr; // Check for -}} (trim following whitespace) if (token.type === Types.EXPRESSION_END && token.text.startsWith('-')) {