From c3558c8e67d40f125db54e1cf8d35ebd0649e0a5 Mon Sep 17 00:00:00 2001 From: Udo Klimaschewski Date: Sun, 19 Mar 2023 13:50:58 +0100 Subject: [PATCH] 360: BUG: Undefined operator '.' ParseException when letter 'e' is after dot '.' (#362) * 360: Fixes recognition of structure separator vs. decimal point, in combination with scientific number notation. --- .../com/ezylang/evalex/parser/Tokenizer.java | 65 ++++++++++++------- .../ExpressionEvaluatorScientificTest.java | 8 ++- .../ExpressionEvaluatorStructureTest.java | 29 +++++++++ .../evalex/parser/TokenizerStructureTest.java | 45 +++++++++++++ 4 files changed, 124 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/ezylang/evalex/parser/Tokenizer.java b/src/main/java/com/ezylang/evalex/parser/Tokenizer.java index 989f15e..3a6a23b 100644 --- a/src/main/java/com/ezylang/evalex/parser/Tokenizer.java +++ b/src/main/java/com/ezylang/evalex/parser/Tokenizer.java @@ -103,9 +103,9 @@ public class Tokenizer { } return ((previousToken.getType() == BRACE_CLOSE && currentToken.getType() == BRACE_OPEN) - || ((previousToken.getType() == NUMBER_LITERAL - && currentToken.getType() == VARIABLE_OR_CONSTANT)) - || ((previousToken.getType() == NUMBER_LITERAL && currentToken.getType() == BRACE_OPEN))); + || (previousToken.getType() == NUMBER_LITERAL + && currentToken.getType() == VARIABLE_OR_CONSTANT) + || (previousToken.getType() == NUMBER_LITERAL && currentToken.getType() == BRACE_OPEN)); } private void validateToken(Token currentToken) throws ParseException { @@ -150,16 +150,16 @@ public class Tokenizer { } else if (currentChar == ']' && configuration.isArraysAllowed()) { return parseArrayClose(); } else if (currentChar == '.' - && !isNumberChar(peekNextChar()) + && !isNextCharNumberChar() && configuration.isStructuresAllowed()) { return parseStructureSeparator(); } else if (currentChar == ',') { Token token = new Token(currentColumnIndex, ",", TokenType.COMMA); consumeChar(); return token; - } else if (isIdentifierStart(currentChar)) { + } else if (isAtIdentifierStart()) { return parseIdentifier(); - } else if (isNumberStart(currentChar)) { + } else if (isAtNumberStart()) { return parseNumberLiteral(); } else { return parseOperator(); @@ -247,6 +247,8 @@ public class Tokenizer { } else if (operatorDictionary.hasInfixOperator(tokenString)) { OperatorIfc operator = operatorDictionary.getInfixOperator(tokenString); return new Token(tokenStartIndex, tokenString, TokenType.INFIX_OPERATOR, operator); + } else if (tokenString.equals(".") && configuration.isStructuresAllowed()) { + return new Token(tokenStartIndex, tokenString, STRUCTURE_SEPARATOR); } throw new ParseException( tokenStartIndex, @@ -357,14 +359,14 @@ public class Tokenizer { consumeChar(); tokenValue.append((char) currentChar); consumeChar(); - while (currentChar != -1 && isHexChar(currentChar)) { + while (currentChar != -1 && isAtHexChar()) { tokenValue.append((char) currentChar); consumeChar(); } } else { // decimal number int lastChar = -1; - while (currentChar != -1 && isNumberChar(currentChar)) { + while (currentChar != -1 && isAtNumberChar()) { tokenValue.append((char) currentChar); lastChar = currentChar; consumeChar(); @@ -382,7 +384,7 @@ public class Tokenizer { private Token parseIdentifier() throws ParseException { int tokenStartIndex = currentColumnIndex; StringBuilder tokenValue = new StringBuilder(); - while (currentChar != -1 && isIdentifierChar(currentChar)) { + while (currentChar != -1 && isAtIdentifierChar()) { tokenValue.append((char) currentChar); consumeChar(); } @@ -472,24 +474,43 @@ public class Tokenizer { } } - private boolean isNumberStart(int ch) { - if (Character.isDigit(ch)) { + private boolean isAtNumberStart() { + if (Character.isDigit(currentChar)) { return true; } - return ch == '.' && Character.isDigit(peekNextChar()); + return currentChar == '.' && Character.isDigit(peekNextChar()); } - private boolean isNumberChar(int ch) { + private boolean isAtNumberChar() { int previousChar = peekPreviousChar(); + if (previousChar == 'e' || previousChar == 'E') { - return Character.isDigit(ch) || ch == '+' || ch == '-'; - } else { - return Character.isDigit(ch) || ch == '.' || ch == 'e' || ch == 'E'; + return Character.isDigit(currentChar) || currentChar == '+' || currentChar == '-'; + } + + if (previousChar == '.') { + return Character.isDigit(currentChar) || currentChar == 'e' || currentChar == 'E'; + } + + return Character.isDigit(currentChar) + || currentChar == '.' + || currentChar == 'e' + || currentChar == 'E'; + } + + private boolean isNextCharNumberChar() { + if (peekNextChar() == -1) { + return false; } + consumeChar(); + boolean isAtNumber = isAtNumberChar(); + currentColumnIndex--; + currentChar = expressionString.charAt(currentColumnIndex - 1); + return isAtNumber; } - private boolean isHexChar(int ch) { - switch (ch) { + private boolean isAtHexChar() { + switch (currentChar) { case '0': case '1': case '2': @@ -518,12 +539,12 @@ public class Tokenizer { } } - private boolean isIdentifierStart(int ch) { - return Character.isLetter(ch) || ch == '_'; + private boolean isAtIdentifierStart() { + return Character.isLetter(currentChar) || currentChar == '_'; } - private boolean isIdentifierChar(int ch) { - return Character.isLetter(ch) || Character.isDigit(ch) || ch == '_'; + private boolean isAtIdentifierChar() { + return Character.isLetter(currentChar) || Character.isDigit(currentChar) || currentChar == '_'; } private void skipBlanks() { diff --git a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java index de06e96..af6f8f6 100644 --- a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java +++ b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java @@ -37,7 +37,13 @@ class ExpressionEvaluatorScientificTest extends BaseExpressionEvaluatorTest { "0.5e2 : 50", "0.35E4 + 0.5e1 : 3505", "2135E-4 : 0.2135", - "2135E+4 : 21350000" + "2135E+4 : 21350000", + "3.e1: 30", + "3.e-1: 0.3", + "3.e+2: 300", + "3.E1: 30", + "3.E-1: 0.3", + "3.E+2: 300" }) void testScientificLiteralsEvaluation(String expression, String expectedResult) throws ParseException, EvaluationException { diff --git a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorStructureTest.java b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorStructureTest.java index d981a32..9420984 100644 --- a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorStructureTest.java +++ b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorStructureTest.java @@ -26,6 +26,35 @@ import org.junit.jupiter.api.Test; class ExpressionEvaluatorStructureTest extends BaseExpressionEvaluatorTest { + @Test + void testStructureScientificNumberDistinction() throws EvaluationException, ParseException { + Map structure = + new HashMap<>() { + { + put("environment_id", new BigDecimal(12345)); + } + }; + Expression expression = new Expression("order.environment_id").with("order", structure); + + assertThat(expression.evaluate().getStringValue()).isEqualTo("12345"); + } + + @Test + void testStructureScientificNumberDistinctionMultiple() + throws EvaluationException, ParseException { + Map structure1 = new HashMap<>(); + Map structure2 = new HashMap<>(); + Map structure3 = new HashMap<>(); + + structure3.put("e", new BigDecimal("765")); + structure2.put("var_x", structure3); + structure1.put("e_id_e", structure2); + + Expression expression = new Expression("order.e_id_e.var_x.e").with("order", structure1); + + assertThat(expression.evaluate().getStringValue()).isEqualTo("765"); + } + @Test void testSimpleStructure() throws ParseException, EvaluationException { Map structure = diff --git a/src/test/java/com/ezylang/evalex/parser/TokenizerStructureTest.java b/src/test/java/com/ezylang/evalex/parser/TokenizerStructureTest.java index ea63232..a44ce40 100644 --- a/src/test/java/com/ezylang/evalex/parser/TokenizerStructureTest.java +++ b/src/test/java/com/ezylang/evalex/parser/TokenizerStructureTest.java @@ -32,6 +32,51 @@ class TokenizerStructureTest extends BaseParserTest { new Token(3, "b", TokenType.VARIABLE_OR_CONSTANT)); } + @Test + void testStructureLeftIsE() throws ParseException { + assertAllTokensParsedCorrectly( + "e.b", + new Token(1, "e", TokenType.VARIABLE_OR_CONSTANT), + new Token(2, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(3, "b", TokenType.VARIABLE_OR_CONSTANT)); + } + + @Test + void testStructureRightIsE() throws ParseException { + assertAllTokensParsedCorrectly( + "a.e", + new Token(1, "a", TokenType.VARIABLE_OR_CONSTANT), + new Token(2, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(3, "e", TokenType.VARIABLE_OR_CONSTANT)); + } + + @Test + void testStructureBothAreE() throws ParseException { + assertAllTokensParsedCorrectly( + "e.e", + new Token(1, "e", TokenType.VARIABLE_OR_CONSTANT), + new Token(2, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(3, "e", TokenType.VARIABLE_OR_CONSTANT)); + } + + @Test + void testStructureLeftEndsE() throws ParseException { + assertAllTokensParsedCorrectly( + "variable.a", + new Token(1, "variable", TokenType.VARIABLE_OR_CONSTANT), + new Token(9, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(10, "a", TokenType.VARIABLE_OR_CONSTANT)); + } + + @Test + void testStructureRightStartsE() throws ParseException { + assertAllTokensParsedCorrectly( + "a.end", + new Token(1, "a", TokenType.VARIABLE_OR_CONSTANT), + new Token(2, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(3, "end", TokenType.VARIABLE_OR_CONSTANT)); + } + @Test void testStructureSeparatorNotAllowedBegin() { assertThatThrownBy(() -> new Tokenizer(".", configuration).parse()) -- 2.51.2