Below are benchmarks and source code for lexer. It needs build.
Place this script file in the tool
folder and execute it.
A lexer will be generated.
Where it will be generated is specified in the script.
In our case, tool/expr_parser.dart
.
This lexer is very easy to change (improve, expand possibilities).
You can easily add support for variables like foo2
. Currently, this is not supported (it seems to me).
The lexer does not remove spaces, and throws a FormatException
at the location of the error.
In your case, the errors will be primitive, because it is a lexer (lexical analysis), but not a parser (grammar).
Time passed: 0.000, Test 'math_expressions lexer': 2193.664 ms
Time passed: 2.196, Test 'parser_builder lexer': 38.308 ms
Time passed: 2.235, Test 'math_expressions lexer': 2308.093 ms
Time passed: 4.543, Test 'parser_builder lexer': 4.19 ms
Time passed: 4.547, Test 'math_expressions lexer': 2310.533 ms
Time passed: 6.858, Test 'parser_builder lexer': 4.146 ms
Time passed: 6.862, Test 'math_expressions lexer': 2321.319 ms
Time passed: 9.184, Test 'parser_builder lexer': 4.347 ms
Time passed: 9.188, Test 'math_expressions lexer': 2306.278 ms
Time passed: 11.494, Test 'parser_builder lexer': 4.142 ms
import 'package:math_expressions/math_expressions.dart';
import 'package:parser_builder/branch.dart';
import 'package:parser_builder/bytes.dart';
import 'package:parser_builder/character.dart';
import 'package:parser_builder/combinator.dart';
import 'package:parser_builder/fast_build.dart';
import 'package:parser_builder/multi.dart';
import 'package:parser_builder/parser_builder.dart';
import 'package:parser_builder/sequence.dart';
import 'package:parser_builder/transformers.dart';
Future<void> main(List<String> args) async {
final context = Context();
context.optimizeForSize = false;
await fastBuild(context, [_parse], 'tool/expr_parser.dart',
header: __header, publish: {'parse': _parse});
}
const __header = r'''
// ignore_for_file: unused_local_variable
import 'package:math_expressions/math_expressions.dart';
import 'package:source_span/source_span.dart';
import 'package:tuple/tuple.dart';
const _source =
'e^(x+2) +x * 2^2.5 * log(10,100) + (-(3.0^2.0))+- (1) + 1^1^1 * ln(10) / sqrt(10) * nrt(2,10) - ceil(1.2) + foo - baz_';
void main() {
final count = 1000;
final names = ['math_expressions lexer', 'parser_builder lexer'];
final tests = [_test1, _test2];
final sw = Stopwatch();
var repeat = 5;
sw.start();
while (repeat-- > 0) {
for (var i = 0; i < tests.length; i++) {
final name = names[i];
final test = tests[i];
final seconds = (sw.elapsedMilliseconds / 1000).toStringAsFixed(3);
final title = 'Time passed: $seconds, Test \'$name\'';
_measure(title, 1, () => test(count));
}
}
}
void _measure(String name, int count, Function() f) {
final sw = Stopwatch();
sw.start();
for (var i = 0; i < count; i++) {
f();
}
sw.stop();
final time = sw.elapsedMicroseconds / 1000;
print('$name: $time ms');
}
void _test1(int count) {
for (var i = 0; i < count; i++) {
final lexer = Lexer();
final res = lexer.tokenize(_source);
}
}
void _test2(int count) {
for (var i = 0; i < count; i++) {
final res = parse(_source);
}
}
''';
const _eof = Named('_eof', Eof<String>());
const _isIdentEnd = CharClasses([_isIdentStart, CharClass('[_]')]);
const _isIdentStart = CharClass('[a-zA-Z]');
const _number = Named(
'_number',
Map1(
Recognize<String>(
Sequence([
Digit1(),
Opt(Sequence([Tag('.'), Digit1()])),
]),
),
Expr<Token>(['x'], 'Token({{x}}, TokenType.VAL)')));
const _parse = Named('_parse', Delimited(_ws, _tokenize, _eof));
const _token = Named(
'_token',
SwitchTag<Token>({
'+': Skip(1, Expr.value("Token('+', TokenType.PLUS)")),
'-': Skip(1, Expr.value("Token('-', TokenType.MINUS)")),
'*': Skip(1, Expr.value("Token('*', TokenType.TIMES)")),
'/': Skip(1, Expr.value("Token('/', TokenType.DIV)")),
'%': Skip(1, Expr.value("Token('%', TokenType.MOD)")),
'^': Skip(1, Expr.value("Token('^', TokenType.POW)")),
'!': Skip(1, Expr.value("Token('!', TokenType.FACTORIAL)")),
'sqrt': Skip(4, Expr.value("Token('sqrt', TokenType.SQRT)")),
'log': Skip(3, Expr.value("Token('log', TokenType.LOG)")),
'cos': Skip(3, Expr.value("Token('cos', TokenType.COS)")),
'sin': Skip(3, Expr.value("Token('sin', TokenType.SIN)")),
'tan': Skip(3, Expr.value("Token('tan', TokenType.TAN)")),
'arccos': Skip(6, Expr.value("Token('arccos', TokenType.ACOS)")),
'arcsin': Skip(6, Expr.value("Token('arcsin', TokenType.ASIN)")),
'arctan': Skip(6, Expr.value("Token('arctan', TokenType.ATAN)")),
'abs': Skip(3, Expr.value("Token('abs', TokenType.ABS)")),
'ceil': Skip(4, Expr.value("Token('ceil', TokenType.CEIL)")),
'floor': Skip(5, Expr.value("Token('floor', TokenType.FLOOR)")),
'sgn': Skip(3, Expr.value("Token('sgn', TokenType.SGN)")),
'ln': Skip(2, Expr.value("Token('ln', TokenType.LN)")),
'e': Skip(1, Expr.value("Token('e', TokenType.EFUNC)")),
'e^': Skip(2, Expr.value("Token('e', TokenType.EFUNC)")),
'(': Skip(1, Expr.value("Token('(', TokenType.LBRACE)")),
')': Skip(1, Expr.value("Token(')', TokenType.RBRACE)")),
'{': Skip(1, Expr.value("Token('{', TokenType.LBRACE)")),
'}': Skip(1, Expr.value("Token('}', TokenType.RBRACE)")),
',': Skip(1, Expr.value("Token(',', TokenType.SEPAR)")),
null: Alt2(_val, _var),
}));
const _tokenize = Named(
'_tokenize',
Map1(ManyTill(Terminated(_token, _ws), _eof),
Expr<List<Token>>(['x'], '{{x}}.item1')));
const _val = Named('_val', _number);
const _var = Named(
'_var',
Map1(
Recognize<String>(
Sequence([Satisfy(_isIdentStart), SkipWhile(_isIdentEnd)])),
Expr<Token>(['x'], 'Token({{x}}, TokenType.VAR)')));
const _ws = Named('_ws', SkipWhile(CharClass('#x9 | #xA | #xD | #x20')));
typedef Expr<T> = ExprTransformer<T>;