From 662db5f03ce39f5a64254efe58bcddcc326df4a3 Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Mon, 15 May 2023 17:08:38 +0200 Subject: [PATCH 01/11] fix audit multiline npe --- .../warp10/script/MemoryWarpScriptStack.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index ebd2d9154..b34e0cb04 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -643,27 +643,28 @@ public void exec(String line, long lineNumber) throws WarpScriptException { } else { throw new WarpScriptException("Not inside a multiline."); } - } - inMultiline.set(false); + } else { + inMultiline.set(false); - String mlcontent = multiline.toString(); + String mlcontent = multiline.toString(); - if (null != secureScript) { - secureScript.append(" "); - secureScript.append("'"); - try { - secureScript.append(WarpURLEncoder.encode(mlcontent, StandardCharsets.UTF_8)); - } catch (UnsupportedEncodingException uee) { - } - secureScript.append("'"); - } else { - if (macros.isEmpty()) { - this.push(mlcontent); + if (null != secureScript) { + secureScript.append(" "); + secureScript.append("'"); + try { + secureScript.append(WarpURLEncoder.encode(mlcontent, StandardCharsets.UTF_8)); + } catch (UnsupportedEncodingException uee) { + } + secureScript.append("'"); } else { - macros.get(0).add(mlcontent); + if (macros.isEmpty()) { + this.push(mlcontent); + } else { + macros.get(0).add(mlcontent); + } } + multiline.setLength(0); } - multiline.setLength(0); continue; } else if (inMultiline.get()) { if (multiline.length() > 0) { From d5658602bbd6ffa21a90228051aaebb75acf6fac Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Mon, 22 May 2023 22:39:35 +0200 Subject: [PATCH 02/11] update the WarpScript parser --- .../warp10/script/MemoryWarpScriptStack.java | 307 +++++++++--------- 1 file changed, 162 insertions(+), 145 deletions(-) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index b34e0cb04..ce0a67d48 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicLong; import io.warp10.script.functions.MSGFAIL; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.util.Progressable; import io.warp10.WarpConfig; @@ -564,166 +563,180 @@ public void exec(String line, long lineNumber) throws WarpScriptException { try { recurseIn(); - String[] statements; + int start = 0; - line = line.trim(); - - // - // Replace whitespaces in Strings with '%20' - // - - line = UnsafeString.sanitizeStrings(line); - - if (-1 != line.indexOf(' ') && !inMultiline.get()) { - //statements = line.split(" +"); - statements = UnsafeString.split(line, ' '); - } else { - // We're either in multiline mode or the line had no whitespace inside - statements = new String[1]; - if (inMultiline.get()) { - // If the line only contained the end of multiline indicator with possible wsp on both sides - // then set the statement to that, otherwise set it to the raw line - if(WarpScriptStack.MULTILINE_END.equals(line)) { - statements[0] = line; + // fast path to process multiline strings or comments blocks + if (inMultiline.get()) { + // Do not do try to detect end of multiline when line length >= 100000 (saves trim memory copy in case of a huge line in a multiline) + if (line.length() < 100000) { + line = line.trim(); + } + // End of multiline + if (WarpScriptStack.MULTILINE_END.equals(line)) { + inMultiline.set(false); + String mlcontent = multiline.toString(); + if (null != secureScript) { + secureScript.append(" "); + secureScript.append("'"); + try { + secureScript.append(WarpURLEncoder.encode(mlcontent, StandardCharsets.UTF_8)); + } catch (UnsupportedEncodingException uee) { + } + secureScript.append("'"); } else { - statements[0] = rawline; + if (macros.isEmpty()) { + this.push(mlcontent); + } else { + macros.get(0).add(mlcontent); + } + } + multiline.setLength(0); + } else { + // Append current line to existing multiline + if (multiline.length() > 0) { + multiline.append("\n"); } + multiline.append(rawline); + } + return; + } else if (inComment.get()) { + int end = line.indexOf(COMMENT_END); + if (-1 == end) { + return; // no end of comment in this line, skip it } else { - statements[0] = line; + start = end; // skip the beginning of the line, before */ } } - // - // Report progress - // - - progress(); - - // - // Loop over the statements - // - - for (int st = 0; st < statements.length; st++) { - handleSignal(); + // Process line character by character, looking at block comments, comments, strings, then process statements. + String stmt; + int end = 0; + int pos = 0; + try { + for (pos = start; pos < line.length(); pos++) { - String stmt = statements[st]; + if (line.charAt(pos) <= ' ') { + continue; // ignore spaces (or other control char) + } - try { - // - // Skip empty statements if we are not currently building a multiline - // + // start of comment block /* + if (line.length() - pos > 1 && line.charAt(pos) == '/' && line.charAt(pos + 1) == '*') { + // look at the end of the comment block on the same line + end = line.indexOf(COMMENT_END, pos + 2); + if (-1 != end) { + pos = end + 2; // skip the comment block + continue; + } else { + inComment.set(true); + break; // no need to process the remaining characters on the line + } + } - if (0 == stmt.length() && !inMultiline.get()) { + // end of comment block */ + if (line.length() - pos > 1 && line.charAt(pos) == '*' && line.charAt(pos + 1) == '/') { + if (!inComment.get()) { + if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Not inside a comment.", lineNumber, pos); + macros.get(0).add(err); + addAuditError(err); + } else { + throw new WarpScriptException("Not inside a comment."); + } + } + inComment.set(false); + pos = pos + 2; continue; } - // - // Trim statement - // - - if (!inMultiline.get()) { - stmt = stmt.trim(); + // line comments , // or # , ignore the remaining characters of the line + if (!inComment.get() && (line.charAt(pos) == '#' || (pos < line.length() - 1 && line.charAt(pos) == '/' && line.charAt(pos + 1) == '/'))) { + break; } - // - // End execution on encountering a comment - // - - if (stmt.length() > 0 && (stmt.charAt(0) == '#' || (stmt.charAt(0) == '/' && stmt.length() >= 2 && stmt.charAt(1) == '/')) && !inMultiline.get() && !inComment.get()) { - // Skip comments and blank lines - return; - } + incOps(); + checkOps(); - if (WarpScriptStack.MULTILINE_END.equals(stmt)) { - if (!inMultiline.get()) { + // Detect strings, "xx" or 'xx' + if (line.charAt(pos) == '"' || line.charAt(pos) == '\'') { + // string start, look for the end + end = line.indexOf(line.charAt(pos), pos + 1); + if (end == -1) { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Not inside a multiline.", lineNumber, st); + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Cannot find end of string", lineNumber, pos); macros.get(0).add(err); addAuditError(err); + break; // cannot find the end, do not try to parse the end of line. } else { - throw new WarpScriptException("Not inside a multiline."); + throw new WarpScriptException("Cannot find end of string"); } } else { - inMultiline.set(false); - - String mlcontent = multiline.toString(); - - if (null != secureScript) { - secureScript.append(" "); - secureScript.append("'"); + // the string is valid when it is followed by a separator (or end of line) + if (end == line.length() - 1 || (line.charAt(end + 1) <= ' ')) { + // this is a valid string, we can decode and push it. + String str = line.substring(pos + 1, end); try { - secureScript.append(WarpURLEncoder.encode(mlcontent, StandardCharsets.UTF_8)); - } catch (UnsupportedEncodingException uee) { + str = WarpURLDecoder.decode(str, StandardCharsets.UTF_8); + if (macros.isEmpty()) { + push(str); + } else { + macros.get(0).add(str); + } + } catch (Exception uee) { + // Catch any decode exception, including incomplete (%) patterns + if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, uee.getMessage(), lineNumber, pos); + macros.get(0).add(err); + addAuditError(err); + } else { + throw new WarpScriptException(uee); + } } - secureScript.append("'"); } else { - if (macros.isEmpty()) { - this.push(mlcontent); + if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Missing space after string", lineNumber, pos); + macros.get(0).add(err); + addAuditError(err); } else { - macros.get(0).add(mlcontent); + throw new WarpScriptException("Missing space after string"); } } - multiline.setLength(0); - } - continue; - } else if (inMultiline.get()) { - if (multiline.length() > 0) { - multiline.append("\n"); - } - multiline.append(stmt); - continue; - } else if (!allowLooseBlockComments && WarpScriptStack.COMMENT_END.equals(stmt)) { - // Legacy comments block: Comments block must start with /* and end with */ . - if (!inComment.get()) { - if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Not inside a comment.", lineNumber, st); - macros.get(0).add(err); - addAuditError(err); - } else { - throw new WarpScriptException("Not inside a comment."); - } } - inComment.set(false); - continue; - } else if (allowLooseBlockComments && stmt.startsWith(WarpScriptStack.COMMENT_START) && stmt.endsWith(WarpScriptStack.COMMENT_END)) { - // Single statement case : /*****foo*****/ - continue; - } else if (allowLooseBlockComments && inComment.get() && stmt.endsWith(WarpScriptStack.COMMENT_END)) { - // End of comment, statement may contain characters before : +-+***/ - inComment.set(false); + pos = end + 1; continue; - } else if (inComment.get()) { - continue; - } else if (!allowLooseBlockComments && WarpScriptStack.COMMENT_START.equals(stmt)) { - // Start of comment, statement may contain characters after : /**---- - inComment.set(true); - continue; - } else if (allowLooseBlockComments && stmt.startsWith(WarpScriptStack.COMMENT_START)) { - // Legacy comments block: Comments block must start with /* and end with */ . - inComment.set(true); - continue; - } else if (WarpScriptStack.MULTILINE_START.equals(stmt)) { - if (1 != statements.length) { + } + + // not a comment/multiline, not a string, this is a statement (followed by a space or end of line) + end = pos; + while (end < line.length() && line.charAt(end) > ' ') { // tolerate tabs or other control characters (v2.x tolerate them at start and end, side effect of trim() + end++; + } + stmt = line.substring(pos, end); + + + if (WarpScriptStack.MULTILINE_START.equals(stmt)) { + if (!WarpScriptStack.MULTILINE_START.equals(line.trim())) { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Can only start multiline strings by using " + WarpScriptStack.MULTILINE_START + " on a line by itself.", lineNumber, st); + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Can only start multiline strings by using " + WarpScriptStack.MULTILINE_START + " on a line by itself.", lineNumber, pos); macros.get(0).add(err); addAuditError(err); } else { throw new WarpScriptException("Can only start multiline strings by using " + WarpScriptStack.MULTILINE_START + " on a line by itself."); } + } else { + inMultiline.set(true); + multiline = new StringBuilder(); } - inMultiline.set(true); - multiline = new StringBuilder(); - continue; + break; // nothing more to process, it is either <' on a single line, or a failure. } - incOps(); - checkOps(); + // + // the following code is the same as previous parser version + // if (WarpScriptStack.SECURE_SCRIPT_END.equals(stmt)) { if (null == secureScript) { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Not inside a secure script definition.", lineNumber, st); + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Not inside a secure script definition.", lineNumber, pos); macros.get(0).add(err); addAuditError(err); } else { @@ -739,7 +752,7 @@ public void exec(String line, long lineNumber) throws WarpScriptException { secureScript = new StringBuilder(); } else { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Already inside a secure script definition.", lineNumber, st); + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Already inside a secure script definition.", lineNumber, pos); macros.get(0).add(err); addAuditError(err); } else { @@ -866,7 +879,7 @@ public void exec(String line, long lineNumber) throws WarpScriptException { } else { if (auditMode) { macros.get(0).add(new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_LOAD, - stmt.substring(1), stmt, lineNumber, st)); + stmt.substring(1), stmt, lineNumber, pos)); } else { macros.get(0).add(stmt.substring(1)); macros.get(0).add(WarpScriptLib.getFunction(WarpScriptLib.LOAD)); @@ -876,8 +889,8 @@ public void exec(String line, long lineNumber) throws WarpScriptException { // // This is an immediate variable dereference // - if (auditMode && macros.size() > 1) { - macros.get(0).add(new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EARLY_BINDING, null, stmt.substring(2), lineNumber, st)); + if (auditMode && macros.size() >= 1) { + macros.get(0).add(new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EARLY_BINDING, null, stmt.substring(2), lineNumber, pos)); } else { Object o = load(stmt.substring(2)); @@ -916,7 +929,7 @@ public void exec(String line, long lineNumber) throws WarpScriptException { macros.get(0).add(stmt.substring(1)); if (auditMode) { macros.get(0).add(new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_LOAD, - stmt.substring(1), stmt, lineNumber, st)); + stmt.substring(1), stmt, lineNumber, pos)); } else { macros.get(0).add(WarpScriptLib.getFunction(WarpScriptLib.RUN)); } @@ -928,9 +941,9 @@ public void exec(String line, long lineNumber) throws WarpScriptException { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { try { Object func = findFunction(stmt); - macros.get(0).add(new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.FUNCTION_CALL, func, stmt, lineNumber, st)); + macros.get(0).add(new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.FUNCTION_CALL, func, stmt, lineNumber, pos)); } catch (WarpScriptException e) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.UNKNOWN, null, stmt, lineNumber, st); + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.UNKNOWN, null, stmt, lineNumber, pos); macros.get(0).add(err); addAuditError(err); } @@ -973,28 +986,32 @@ public void exec(String line, long lineNumber) throws WarpScriptException { } } } - } catch (WarpScriptATCException e) { - throw e; - } catch (Exception e) { - StringBuilder errorMessage = new StringBuilder("Exception at '"); - boolean nextStatement = false; - for (int stc = Math.max(0, st - 3); stc < Math.min(statements.length, st + 4); stc++) { - if (nextStatement) { - errorMessage.append(" "); - } else { - nextStatement = true; - } + pos = end; + } + progress(); + } catch (WarpScriptATCException e) { + throw e; + } catch (Exception e) { + StringBuilder errorMessage = new StringBuilder("Exception at '"); + if (pos < 0) { + pos = 0; // should not happen + } + if (pos >= line.length()) { + pos = line.length() - 1; + } + if (pos > end) { + end = pos; + } + if (end > line.length()) { + end = line.length(); + } + errorMessage.append(line, Math.max(0, pos - 30), pos); + errorMessage.append("=>").append(line, pos, end).append("<="); + errorMessage.append(line, end, Math.min(end + 30, line.length())); - if (st == stc) { - errorMessage.append("=>").append(StringUtils.abbreviateMiddle(statements[stc].trim(), "...", 32)).append("<="); - } else { - errorMessage.append(StringUtils.abbreviateMiddle(statements[stc], "...", 32)); - } - } - errorMessage.append("' in section " + sectionName); + errorMessage.append("' in section " + sectionName); - throw new WarpScriptException(errorMessage.toString(), e); - } + throw new WarpScriptException(errorMessage.toString(), e); } return; From ac32ff315cff7b9ef47674b88418d77919fafe66 Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Wed, 24 May 2023 17:46:30 +0200 Subject: [PATCH 03/11] fix string management for better compatibility, and add warning when separator found inside string --- .../warp10/script/MemoryWarpScriptStack.java | 28 ++++++++++++++++++- .../script/WarpScriptAuditStatement.java | 1 + 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index ce0a67d48..97d10eadc 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -657,9 +657,35 @@ public void exec(String line, long lineNumber) throws WarpScriptException { checkOps(); // Detect strings, "xx" or 'xx' + // If the separator is at the end of the line or + // followed by a whitespace then we consider we exited the string, otherwise + // it is just part of the string, but we store a warning flag. if (line.charAt(pos) == '"' || line.charAt(pos) == '\'') { + char sep = line.charAt(pos); + boolean warnSepInclusion = false; // string start, look for the end - end = line.indexOf(line.charAt(pos), pos + 1); + end = -1; + if (pos != line.length() - 1) { // do not look for string end if it starts at line end + int strEnd = pos + 1; + while (strEnd < line.length()) { + if (line.charAt(strEnd) == sep) { + if (strEnd == line.length() - 1 || line.charAt(strEnd + 1) <= ' ') { + // end of line, or followed by a separator + end = strEnd; + break; + } else { + warnSepInclusion = true; // separator found inside the string. legal, but confusing + } + } + strEnd++; + } + } + //end = line.indexOf(line.charAt(pos), pos + 1); + if (auditMode && warnSepInclusion) { + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_WARNING, null, "Separator found inside the string", lineNumber, pos); + macros.get(0).add(err); + addAuditError(err); + } if (end == -1) { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Cannot find end of string", lineNumber, pos); diff --git a/warp10/src/main/java/io/warp10/script/WarpScriptAuditStatement.java b/warp10/src/main/java/io/warp10/script/WarpScriptAuditStatement.java index df3556a68..0b5ba1654 100644 --- a/warp10/src/main/java/io/warp10/script/WarpScriptAuditStatement.java +++ b/warp10/src/main/java/io/warp10/script/WarpScriptAuditStatement.java @@ -37,6 +37,7 @@ public enum STATEMENT_TYPE { WS_EARLY_BINDING, // managed as a simple load for the moment WS_LOAD, // load action is stored as one statement, to store the right statement position WS_RUN, // run action is stored as one statement, to store the right statement position + WS_WARNING, // something legal in WarpScript, but than can be misleading UNKNOWN // syntax error (missing space, [5 is not a function), function call to a REDEF function, or unknown extension. } From b5c539d465204c7ae25111542a2d77dec85129ce Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Thu, 25 May 2023 16:03:43 +0200 Subject: [PATCH 04/11] add statement absolute end position to audit statements --- .../WarpScriptAuditStatementSerializer.java | 3 +- .../warp10/script/MemoryWarpScriptStack.java | 52 +++++++------------ .../script/WarpScriptAuditStatement.java | 22 ++++++-- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/warp10/src/main/java/io/warp10/json/WarpScriptAuditStatementSerializer.java b/warp10/src/main/java/io/warp10/json/WarpScriptAuditStatementSerializer.java index 375c0bd51..1d32a7947 100644 --- a/warp10/src/main/java/io/warp10/json/WarpScriptAuditStatementSerializer.java +++ b/warp10/src/main/java/io/warp10/json/WarpScriptAuditStatementSerializer.java @@ -35,7 +35,8 @@ public void serialize(WarpScriptAuditStatement st, JsonGenerator gen, Serializer gen.writeStartObject(); gen.writeStringField(WarpScriptAuditStatement.KEY_TYPE,st.type.name()); gen.writeNumberField(WarpScriptAuditStatement.KEY_LINE,st.lineNumber); - gen.writeNumberField(WarpScriptAuditStatement.KEY_POSITION,st.positionNumber); + gen.writeNumberField(WarpScriptAuditStatement.KEY_POSITION,st.inLinePositionStart); + gen.writeNumberField(WarpScriptAuditStatement.KEY_POSITION_END,st.inLinePositionEnd); gen.writeStringField(WarpScriptAuditStatement.KEY_STATEMENT,st.statement); gen.writeEndObject(); } diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index 97d10eadc..cbb524728 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -636,7 +636,7 @@ public void exec(String line, long lineNumber) throws WarpScriptException { if (line.length() - pos > 1 && line.charAt(pos) == '*' && line.charAt(pos + 1) == '/') { if (!inComment.get()) { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Not inside a comment.", lineNumber, pos); + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Not inside a comment.", lineNumber, pos, pos + 1); macros.get(0).add(err); addAuditError(err); } else { @@ -680,50 +680,38 @@ public void exec(String line, long lineNumber) throws WarpScriptException { strEnd++; } } - //end = line.indexOf(line.charAt(pos), pos + 1); - if (auditMode && warnSepInclusion) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_WARNING, null, "Separator found inside the string", lineNumber, pos); - macros.get(0).add(err); - addAuditError(err); - } + if (end == -1) { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Cannot find end of string", lineNumber, pos); + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Cannot find end of string", lineNumber, pos, line.length()-1); macros.get(0).add(err); addAuditError(err); - break; // cannot find the end, do not try to parse the end of line. + break; // cannot find the end of the string, do not try to parse the end of current line. } else { throw new WarpScriptException("Cannot find end of string"); } } else { - // the string is valid when it is followed by a separator (or end of line) - if (end == line.length() - 1 || (line.charAt(end + 1) <= ' ')) { - // this is a valid string, we can decode and push it. - String str = line.substring(pos + 1, end); - try { - str = WarpURLDecoder.decode(str, StandardCharsets.UTF_8); - if (macros.isEmpty()) { - push(str); - } else { - macros.get(0).add(str); - } - } catch (Exception uee) { - // Catch any decode exception, including incomplete (%) patterns - if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, uee.getMessage(), lineNumber, pos); - macros.get(0).add(err); - addAuditError(err); - } else { - throw new WarpScriptException(uee); - } + if (auditMode && warnSepInclusion) { + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_WARNING, null, "Separator found inside the string", lineNumber, pos, end + 1); + addAuditError(err); + } + // this is a valid string, we can decode and push it. + String str = line.substring(pos + 1, end); + try { + str = WarpURLDecoder.decode(str, StandardCharsets.UTF_8); + if (macros.isEmpty()) { + push(str); + } else { + macros.get(0).add(str); } - } else { + } catch (Exception uee) { + // Catch any decode exception, including incomplete (%) patterns if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { - WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Missing space after string", lineNumber, pos); + WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, uee.getMessage(), lineNumber, pos, end + 1); macros.get(0).add(err); addAuditError(err); } else { - throw new WarpScriptException("Missing space after string"); + throw new WarpScriptException(uee); } } } diff --git a/warp10/src/main/java/io/warp10/script/WarpScriptAuditStatement.java b/warp10/src/main/java/io/warp10/script/WarpScriptAuditStatement.java index 0b5ba1654..614553668 100644 --- a/warp10/src/main/java/io/warp10/script/WarpScriptAuditStatement.java +++ b/warp10/src/main/java/io/warp10/script/WarpScriptAuditStatement.java @@ -29,6 +29,7 @@ public class WarpScriptAuditStatement implements WarpScriptStackFunction { public static final String KEY_TYPE = "type"; public static final String KEY_LINE = "line"; public static final String KEY_POSITION = "position"; + public static final String KEY_POSITION_END = "position.end"; public static final String KEY_STATEMENT = "statement"; public enum STATEMENT_TYPE { @@ -44,17 +45,27 @@ public enum STATEMENT_TYPE { public STATEMENT_TYPE type; public String statement; public long lineNumber; // first line is numbered 1 - public int positionNumber; // first statement in line is numbered 0 + public int inLinePositionStart; // first character of the line is numbered 0 + public int inLinePositionEnd; // end of the statement or audit issue public Object statementObject; - public WarpScriptAuditStatement(STATEMENT_TYPE type, Object statementObject, String statement, Long lineNumber, int statementNumber) { + public WarpScriptAuditStatement(STATEMENT_TYPE type, Object statementObject, String statement, Long lineNumber, int start) { this.type = type; this.statement = statement; this.lineNumber = lineNumber; - this.positionNumber = statementNumber; + this.inLinePositionStart = start; + this.inLinePositionEnd = statement == null ? start : start + statement.length(); this.statementObject = statementObject; } + public WarpScriptAuditStatement(STATEMENT_TYPE type, Object statementObject, String statement, Long lineNumber, int start, int end) { + this.type = type; + this.statement = statement; + this.lineNumber = lineNumber; + this.inLinePositionStart = start; + this.inLinePositionEnd = end; + this.statementObject = statementObject; + } @Override public Object apply(WarpScriptStack stack) throws WarpScriptException { switch (type) { @@ -101,7 +112,7 @@ public Object apply(WarpScriptStack stack) throws WarpScriptException { * @return " (line x, position y)", where x and y are numbers. */ public String formatPosition() { - return " (line " + lineNumber + ", position " + positionNumber + ")"; + return " (line " + lineNumber + ", position " + inLinePositionStart + ")"; } /** @@ -112,7 +123,8 @@ public Map toMap() { Map m = new LinkedHashMap(); m.put(KEY_TYPE,type.name()); m.put(KEY_LINE,lineNumber); - m.put(KEY_POSITION,((Integer)positionNumber).longValue()); // WarpScript knows LONG + m.put(KEY_POSITION,((Integer) inLinePositionStart).longValue()); // WarpScript knows LONG + m.put(KEY_POSITION_END,((Integer) inLinePositionEnd).longValue()); // WarpScript knows LONG m.put(KEY_STATEMENT,statement); return m; } From 3e82be5145c5875e12ae989d8d22ce36d95602bb Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Thu, 25 May 2023 17:54:50 +0200 Subject: [PATCH 05/11] fix typos --- .../warp10/script/MemoryWarpScriptStack.java | 62 ++++++++++--------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index cbb524728..bc14981ce 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -565,12 +565,11 @@ public void exec(String line, long lineNumber) throws WarpScriptException { int start = 0; - // fast path to process multiline strings or comments blocks + // + // Fast path to process multiline strings or comments blocks + // if (inMultiline.get()) { - // Do not do try to detect end of multiline when line length >= 100000 (saves trim memory copy in case of a huge line in a multiline) - if (line.length() < 100000) { - line = line.trim(); - } + line = line.trim(); // End of multiline if (WarpScriptStack.MULTILINE_END.equals(line)) { inMultiline.set(false); @@ -602,13 +601,15 @@ public void exec(String line, long lineNumber) throws WarpScriptException { } else if (inComment.get()) { int end = line.indexOf(COMMENT_END); if (-1 == end) { - return; // no end of comment in this line, skip it + return; // No end of comment in this line, skip it } else { - start = end; // skip the beginning of the line, before */ + start = end; // Skip the beginning of the line, before */ } } - + + // // Process line character by character, looking at block comments, comments, strings, then process statements. + // String stmt; int end = 0; int pos = 0; @@ -616,15 +617,16 @@ public void exec(String line, long lineNumber) throws WarpScriptException { for (pos = start; pos < line.length(); pos++) { if (line.charAt(pos) <= ' ') { - continue; // ignore spaces (or other control char) + continue; // Ignore spaces (or other control char) } - // start of comment block /* + // + // Start of comment block /* + // if (line.length() - pos > 1 && line.charAt(pos) == '/' && line.charAt(pos + 1) == '*') { - // look at the end of the comment block on the same line - end = line.indexOf(COMMENT_END, pos + 2); + end = line.indexOf(COMMENT_END, pos + 2); // Look at the end of the comment block on the same line if (-1 != end) { - pos = end + 2; // skip the comment block + pos = end + 2; // Skip the comment block continue; } else { inComment.set(true); @@ -632,7 +634,9 @@ public void exec(String line, long lineNumber) throws WarpScriptException { } } - // end of comment block */ + // + // End of comment block */ + // if (line.length() - pos > 1 && line.charAt(pos) == '*' && line.charAt(pos + 1) == '/') { if (!inComment.get()) { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { @@ -648,9 +652,11 @@ public void exec(String line, long lineNumber) throws WarpScriptException { continue; } - // line comments , // or # , ignore the remaining characters of the line + // + // Line comments, // or # + // if (!inComment.get() && (line.charAt(pos) == '#' || (pos < line.length() - 1 && line.charAt(pos) == '/' && line.charAt(pos + 1) == '/'))) { - break; + break; // Ignore the remaining characters of the line } incOps(); @@ -663,18 +669,17 @@ public void exec(String line, long lineNumber) throws WarpScriptException { if (line.charAt(pos) == '"' || line.charAt(pos) == '\'') { char sep = line.charAt(pos); boolean warnSepInclusion = false; - // string start, look for the end end = -1; - if (pos != line.length() - 1) { // do not look for string end if it starts at line end + if (pos != line.length() - 1) { // Do not look for string end if it starts at line end int strEnd = pos + 1; while (strEnd < line.length()) { if (line.charAt(strEnd) == sep) { if (strEnd == line.length() - 1 || line.charAt(strEnd + 1) <= ' ') { - // end of line, or followed by a separator + // End of line, or followed by a separator end = strEnd; break; } else { - warnSepInclusion = true; // separator found inside the string. legal, but confusing + warnSepInclusion = true; // Separator found inside the string. Legal, but may be confusing } } strEnd++; @@ -686,16 +691,15 @@ public void exec(String line, long lineNumber) throws WarpScriptException { WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Cannot find end of string", lineNumber, pos, line.length()-1); macros.get(0).add(err); addAuditError(err); - break; // cannot find the end of the string, do not try to parse the end of current line. + break; // Cannot find the end of the string, do not try to parse the end of current line. } else { throw new WarpScriptException("Cannot find end of string"); } - } else { + } else { // This is a valid string, we can decode and push it. if (auditMode && warnSepInclusion) { WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_WARNING, null, "Separator found inside the string", lineNumber, pos, end + 1); addAuditError(err); } - // this is a valid string, we can decode and push it. String str = line.substring(pos + 1, end); try { str = WarpURLDecoder.decode(str, StandardCharsets.UTF_8); @@ -719,9 +723,11 @@ public void exec(String line, long lineNumber) throws WarpScriptException { continue; } - // not a comment/multiline, not a string, this is a statement (followed by a space or end of line) + // + // Not a comment or multiline, not a string, this is a statement (followed by a space or end of line) + // end = pos; - while (end < line.length() && line.charAt(end) > ' ') { // tolerate tabs or other control characters (v2.x tolerate them at start and end, side effect of trim() + while (end < line.length() && line.charAt(end) > ' ') { // Tolerate tabs or other control characters (v2.x tolerate them at start and end, side effect of trim() end++; } stmt = line.substring(pos, end); @@ -740,11 +746,11 @@ public void exec(String line, long lineNumber) throws WarpScriptException { inMultiline.set(true); multiline = new StringBuilder(); } - break; // nothing more to process, it is either <' on a single line, or a failure. + break; // Nothing more to process, it is either <' on a single line, or a failure. } // - // the following code is the same as previous parser version + // The following code is the same as previous parser version // if (WarpScriptStack.SECURE_SCRIPT_END.equals(stmt)) { @@ -1008,7 +1014,7 @@ public void exec(String line, long lineNumber) throws WarpScriptException { } catch (Exception e) { StringBuilder errorMessage = new StringBuilder("Exception at '"); if (pos < 0) { - pos = 0; // should not happen + pos = 0; // Should not happen } if (pos >= line.length()) { pos = line.length() - 1; From 9c6781b73a4b298c998a35e58b7aa160be81cbb2 Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Sat, 27 May 2023 21:48:20 +0200 Subject: [PATCH 06/11] check for separator after end of comment --- .../warp10/script/MemoryWarpScriptStack.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index bc14981ce..e89dd5f7e 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -624,12 +624,11 @@ public void exec(String line, long lineNumber) throws WarpScriptException { // Start of comment block /* // if (line.length() - pos > 1 && line.charAt(pos) == '/' && line.charAt(pos + 1) == '*') { + inComment.set(true); end = line.indexOf(COMMENT_END, pos + 2); // Look at the end of the comment block on the same line if (-1 != end) { - pos = end + 2; // Skip the comment block - continue; + pos = end; // seek to next comment end, will be evaluated below } else { - inComment.set(true); break; // no need to process the remaining characters on the line } } @@ -647,20 +646,38 @@ public void exec(String line, long lineNumber) throws WarpScriptException { throw new WarpScriptException("Not inside a comment."); } } - inComment.set(false); - pos = pos + 2; - continue; + // is it followed by a space, or by end of line ? + if (pos == line.length() - 2 || line.charAt(pos + 2) == ' ') { + inComment.set(false); + pos = pos + 2; + continue; + } else { + // */x inside a comment is not a valid comment end. look for the next */ + end = line.indexOf(COMMENT_END, pos + 2); + if (-1 == end) { + break; // no need to process the remaining characters on the line + } else { + pos = end - 1; // skip to next */ found (the for loop will increment pos) + continue; + } + } } + if (inComment.get()) { // should never happen + continue; + } + // // Line comments, // or # // - if (!inComment.get() && (line.charAt(pos) == '#' || (pos < line.length() - 1 && line.charAt(pos) == '/' && line.charAt(pos + 1) == '/'))) { + if (line.charAt(pos) == '#' || (pos < line.length() - 1 && line.charAt(pos) == '/' && line.charAt(pos + 1) == '/')) { break; // Ignore the remaining characters of the line } incOps(); checkOps(); + handleSignal(); + progress(); // Detect strings, "xx" or 'xx' // If the separator is at the end of the line or From ad7011dc532803e5947b53d037c8c6eb551c1e05 Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Fri, 2 Jun 2023 11:03:54 +0200 Subject: [PATCH 07/11] better string detection compatibility --- .../warp10/script/MemoryWarpScriptStack.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index e89dd5f7e..b816c61d7 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -646,7 +646,7 @@ public void exec(String line, long lineNumber) throws WarpScriptException { throw new WarpScriptException("Not inside a comment."); } } - // is it followed by a space, or by end of line ? + // is it followed by a space, or by end of line? if (pos == line.length() - 2 || line.charAt(pos + 2) == ' ') { inComment.set(false); pos = pos + 2; @@ -679,19 +679,29 @@ public void exec(String line, long lineNumber) throws WarpScriptException { handleSignal(); progress(); - // Detect strings, "xx" or 'xx' - // If the separator is at the end of the line or + // + // Trim end of line to comply with string detection of the previous parser + // "a"%09a" is a valid single string (separator not followed by a space) + // "a" followed by a tabulation then end of line should also be valid (previous parser trimmed the line) + // + // Then detect strings, "xx" or 'xx' if the separator is at the end of the line or // followed by a whitespace then we consider we exited the string, otherwise // it is just part of the string, but we store a warning flag. + // + int trimmedLength = line.length() - 1; + while (trimmedLength > 0 && line.charAt(trimmedLength) <= ' ') { + trimmedLength--; + } + trimmedLength++; if (line.charAt(pos) == '"' || line.charAt(pos) == '\'') { char sep = line.charAt(pos); boolean warnSepInclusion = false; end = -1; - if (pos != line.length() - 1) { // Do not look for string end if it starts at line end + if (pos != trimmedLength - 1) { // Do not look for string end if it starts at line end int strEnd = pos + 1; - while (strEnd < line.length()) { + while (strEnd < trimmedLength) { if (line.charAt(strEnd) == sep) { - if (strEnd == line.length() - 1 || line.charAt(strEnd + 1) <= ' ') { + if (strEnd == trimmedLength - 1 || line.charAt(strEnd + 1) == ' ') { // End of line, or followed by a separator end = strEnd; break; From 9661e53f23b857178179249ccb0f681f85a28924 Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Fri, 2 Jun 2023 11:14:41 +0200 Subject: [PATCH 08/11] fix comment --- .../src/main/java/io/warp10/script/MemoryWarpScriptStack.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index b816c61d7..d1ed70fd1 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -702,7 +702,7 @@ public void exec(String line, long lineNumber) throws WarpScriptException { while (strEnd < trimmedLength) { if (line.charAt(strEnd) == sep) { if (strEnd == trimmedLength - 1 || line.charAt(strEnd + 1) == ' ') { - // End of line, or followed by a separator + // End of trimmed line, or followed by a whitespace end = strEnd; break; } else { From 0fd17d385ee87382066349c82a389ede148c2b70 Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Fri, 2 Jun 2023 14:19:43 +0200 Subject: [PATCH 09/11] push decoded strings to securescript too --- .../warp10/script/MemoryWarpScriptStack.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index d1ed70fd1..cda247454 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicLong; import io.warp10.script.functions.MSGFAIL; +import io.warp10.script.functions.SNAPSHOT; import org.apache.hadoop.util.Progressable; import io.warp10.WarpConfig; @@ -730,10 +731,16 @@ public void exec(String line, long lineNumber) throws WarpScriptException { String str = line.substring(pos + 1, end); try { str = WarpURLDecoder.decode(str, StandardCharsets.UTF_8); - if (macros.isEmpty()) { - push(str); + if (null != secureScript) { + secureScript.append(" '"); + SNAPSHOT.appendProcessedString(secureScript, str); + secureScript.append("'"); } else { - macros.get(0).add(str); + if (macros.isEmpty()) { + push(str); + } else { + macros.get(0).add(str); + } } } catch (Exception uee) { // Catch any decode exception, including incomplete (%) patterns @@ -790,9 +797,11 @@ public void exec(String line, long lineNumber) throws WarpScriptException { throw new WarpScriptException("Not inside a secure script definition."); } } else { - this.push(secureScript.toString()); - secureScript = null; - new SECURE("SECURESCRIPT").apply(this); + if (!auditMode) { + this.push(secureScript.toString()); + secureScript = null; + new SECURE("SECURESCRIPT").apply(this); + } } } else if (WarpScriptStack.SECURE_SCRIPT_START.equals(stmt)) { if (null == secureScript) { From 2383ebdc603a04af2dde547062df9c8d2fcee881 Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Fri, 2 Jun 2023 15:32:57 +0200 Subject: [PATCH 10/11] parser add an early exit --- .../java/io/warp10/script/MemoryWarpScriptStack.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index cda247454..a3358cc0d 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -598,6 +598,8 @@ public void exec(String line, long lineNumber) throws WarpScriptException { } multiline.append(rawline); } + handleSignal(); + progress(); return; } else if (inComment.get()) { int end = line.indexOf(COMMENT_END); @@ -624,7 +626,7 @@ public void exec(String line, long lineNumber) throws WarpScriptException { // // Start of comment block /* // - if (line.length() - pos > 1 && line.charAt(pos) == '/' && line.charAt(pos + 1) == '*') { + if (pos < line.length() - 1 && line.charAt(pos) == '/' && line.charAt(pos + 1) == '*') { inComment.set(true); end = line.indexOf(COMMENT_END, pos + 2); // Look at the end of the comment block on the same line if (-1 != end) { @@ -637,7 +639,7 @@ public void exec(String line, long lineNumber) throws WarpScriptException { // // End of comment block */ // - if (line.length() - pos > 1 && line.charAt(pos) == '*' && line.charAt(pos + 1) == '/') { + if (pos < line.length() - 1 && line.charAt(pos) == '*' && line.charAt(pos + 1) == '/') { if (!inComment.get()) { if (auditMode && !(macros.isEmpty() || macros.size() == forcedMacro)) { WarpScriptAuditStatement err = new WarpScriptAuditStatement(WarpScriptAuditStatement.STATEMENT_TYPE.WS_EXCEPTION, null, "Not inside a comment.", lineNumber, pos, pos + 1); @@ -694,6 +696,9 @@ public void exec(String line, long lineNumber) throws WarpScriptException { trimmedLength--; } trimmedLength++; + if (0 == trimmedLength) { + break; // empty line + } if (line.charAt(pos) == '"' || line.charAt(pos) == '\'') { char sep = line.charAt(pos); boolean warnSepInclusion = false; From 2c9dfd119ceadff60de52701f7e0b0f38f71906c Mon Sep 17 00:00:00 2001 From: pi-r-p Date: Fri, 2 Jun 2023 15:42:32 +0200 Subject: [PATCH 11/11] add progress call in long block comment --- .../src/main/java/io/warp10/script/MemoryWarpScriptStack.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java index a3358cc0d..bbfa7861a 100644 --- a/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java +++ b/warp10/src/main/java/io/warp10/script/MemoryWarpScriptStack.java @@ -604,6 +604,8 @@ public void exec(String line, long lineNumber) throws WarpScriptException { } else if (inComment.get()) { int end = line.indexOf(COMMENT_END); if (-1 == end) { + handleSignal(); + progress(); return; // No end of comment in this line, skip it } else { start = end; // Skip the beginning of the line, before */