diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java index b79f3176fd3..2c02ea6ec8e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java @@ -206,7 +206,7 @@ public Map getPlans(IIndentStream stream) { stream.append("\"").append(Utilities.escapeDoubleQuotes(e.name())).append("\""); stream.append(": "); RelNode rel = cv.getRel(); - RelJsonWriter planWriter = new RelJsonWriter(result); + RelJsonWriter planWriter = new RelJsonWriter(result, options.ioOptions.verbosity); rel.explain(planWriter); String json = planWriter.asString(); stream.appendIndentedStrings(json); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/RelJsonWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/RelJsonWriter.java index 809b76cc8ac..693ece21eac 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/RelJsonWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/RelJsonWriter.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.UnaryOperator; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; @@ -37,31 +38,38 @@ public class RelJsonWriter implements RelWriter { protected final RelJson relJson; protected final List<@Nullable Object> relList; private final List> values; + public final int verbosity; - public RelJsonWriter(Map relIdMap) { - this(new JsonBuilder(), relIdMap); + public RelJsonWriter(Map relIdMap, int verbosity) { + this(new JsonBuilder(), relIdMap, verbosity); } - public RelJsonWriter(JsonBuilder jsonBuilder, Map relIdMap) { - this(jsonBuilder, UnaryOperator.identity(), relIdMap); + public RelJsonWriter(JsonBuilder jsonBuilder, Map relIdMap, int verbosity) { + this(jsonBuilder, UnaryOperator.identity(), relIdMap, verbosity); } public RelJsonWriter(JsonBuilder jsonBuilder, UnaryOperator relJsonTransform, - Map relIdMap) { + Map relIdMap, + int verbosity) { this.values = new ArrayList<>(); this.jsonBuilder = Objects.requireNonNull(jsonBuilder, "jsonBuilder"); this.relList = this.jsonBuilder.list(); this.relJson = relJsonTransform.apply(RelJson.create().withJsonBuilder(jsonBuilder)); this.relIdMap = relIdMap; + this.verbosity = verbosity; } + final Set highVerbosity = Set.of("exprs", "aggs", "condition"); + protected void explain_(RelNode rel, List> values) { Map map = this.jsonBuilder.map(); map.put("id", null); map.put("relOp", this.relJson.classToTypeName(rel.getClass())); for(Pair value : values) { + if (this.verbosity < 1 && highVerbosity.contains(value.left)) + continue; if (!(value.right instanceof RelNode)) { this.put(map, value.left, value.right); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java index ebab1a78fc1..b2c94809f89 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java @@ -1402,7 +1402,7 @@ public void generateDFRecursiveTest() throws IOException, SQLException { File file = createInputScript(sql); File json = this.createTempJsonFile(); CompilerMessages msg = CompilerMain.execute( - "--dataflow", json.getPath(), "--noRust", file.getPath()); + "-v", "1", "--dataflow", json.getPath(), "--noRust", file.getPath()); Assert.assertEquals(0, msg.exitCode); String jsonContents = Utilities.readFile(json.toPath()); String expected = TestUtil.readStringFromResourceFile("metadataTests-generateDFRecursive.json"); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java index d577244ff6b..9c4b1c4eff3 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java @@ -67,7 +67,9 @@ static void compileToMultiCrate(String file, boolean check) throws SQLException, compileToMultiCrate(file, check, true); } - static void compileProgramToMultiCrate(String sql, boolean check) throws IOException, SQLException, InterruptedException { + @SuppressWarnings("SameParameterValue") + static void compileProgramToMultiCrate(String sql, boolean check) + throws IOException, SQLException, InterruptedException { File file = createInputScript(sql); compileToMultiCrate(file.getAbsolutePath(), check); } @@ -201,12 +203,6 @@ CREATE TABLE T ( compileProgramToMultiCrate(sql, true); } - @Test @Ignore - public void testMultiCrateLarge() throws IOException, SQLException, InterruptedException { - File file = new File("../extra/current_pipeline.sql"); - compileToMultiCrate(file.getAbsolutePath(), true); - } - @Test public void testJoin() throws IOException, SQLException, InterruptedException { String sql = """ diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresIntervalTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresIntervalTests.java index acb34f46084..04974ec2df9 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresIntervalTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresIntervalTests.java @@ -217,9 +217,11 @@ public void testIntervalOperations() { public void testLimits() { this.qf("SELECT -(INTERVAL -2147483648 months)", "Overflow during LongInterval negation"); - this.qf("SELECT -(INTERVAL -9223372036854775.808 SECONDS)", ""); - this.qf("SELECT -(INTERVAL -2147483648 days)", ""); - this.qf("SELECT INTERVAL 2147483647 years", "Out of range"); + this.qf("SELECT -(INTERVAL -9223372036854775.808 SECONDS)", + "overflow in short interval multiplication: result too large: OutOfRange"); + this.qf("SELECT -(INTERVAL -2147483648 days)", "attempt to multiply with overflow"); + // Last one is simplified to -1 by Calcite + this.qf("SELECT INTERVAL 2147483647 years", "attempt to multiply with overflow"); } @Test diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresLimitTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresLimitTests.java index 001d89850dc..71cef4b6600 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresLimitTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresLimitTests.java @@ -76,7 +76,7 @@ public void testLimit() { (2 rows)"""); } - @Test @Ignore("OFFSET not yet implemented") + @Test public void testOffset() { this.qs(""" SELECT unique1, unique2, stringu1 diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresStringTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresStringTests.java index 5e9a92e9847..650882dd671 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresStringTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/postgres/PostgresStringTests.java @@ -373,7 +373,7 @@ public void testOverlay() { bubba"""); } - @Test @Ignore("Not yet implemented") + @Test @Ignore("regexp_replace with 4 arguments not implemented") public void testRegexpReplace() { this.qs(""" SELECT regexp_replace('1112223333', E'(\\\\d{3})(\\\\d{3})(\\\\d{4})', E'(\\\\1) \\\\2-\\\\3'); @@ -776,61 +776,6 @@ public void testLike3() { f"""); } - @Test @Ignore("We do not allow escape characters that are % or _") - public void testLike3Pattern() { - // -- escape character same as pattern character\n" - this.q(""" - SELECT 'maca' LIKE 'm%aca' ESCAPE '%' AS "true"; - true - ------ - t"""); - this.q(""" - SELECT 'maca' NOT LIKE 'm%aca' ESCAPE '%' AS "false"; - false - ------- - f"""); - this.q(""" - SELECT 'ma%a' LIKE 'm%a%%a' ESCAPE '%' AS "true"; - true - ------ - t"""); - this.q(""" - SELECT 'ma%a' NOT LIKE 'm%a%%a' ESCAPE '%' AS "false"; - false - ------- - f"""); - this.q(""" - SELECT 'bear' LIKE 'b_ear' ESCAPE '_' AS "true"; - true - ------ - t"""); - this.q(""" - SELECT 'bear' NOT LIKE 'b_ear' ESCAPE '_' AS "false"; - false - ------- - f"""); - this.q(""" - SELECT 'be_r' LIKE 'b_e__r' ESCAPE '_' AS "true"; - true - ------ - t"""); - this.q(""" - SELECT 'be_r' NOT LIKE 'b_e__r' ESCAPE '_' AS "false"; - false - ------- - f"""); - this.q(""" - SELECT 'be_r' LIKE '__e__r' ESCAPE '_' AS "false"; - false - ------- - f"""); - this.q(""" - SELECT 'be_r' NOT LIKE '__e__r' ESCAPE '_' AS "true"; - true - ------ - t"""); - } - @Test public void testILike2() { this.q(""" diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/quidem/WinAggPostTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/quidem/WinAggPostTests.java index dd7890a7ea4..37164b33fe8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/quidem/WinAggPostTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/quidem/WinAggPostTests.java @@ -600,7 +600,7 @@ public void testWindows2() { (6 rows)"""); } - @Test @Ignore("MAP not yet supported") + @Test public void testWindows3() { this.qs(""" -- [CALCITE-2271] Two windows under a JOIN 2 @@ -608,19 +608,23 @@ public void testWindows3() { t1.l, t1.key as key1, t2.key as key2 from ( - select - dense_rank() over (order by key) l, - key - from - unnest(map[1,1,2,2]) k + select * FROM ( + select + dense_rank() over (order by key) AS l, + key + from + unnest(map[1,1,2,2]) k(key, value)) + WHERE l <= 10 ) t1 join ( - select - dense_rank() over(order by key) l, - key - from - unnest(map[2,2]) k + select * FROM ( + select + dense_rank() over(order by key) AS l, + key + from + unnest(map[2,2]) k(key, value)) + WHERE l <= 10 ) t2 on (t1.l = t2.l and t1.key + 1 = t2.key); +---+------+------+ | L | KEY1 | KEY2 | diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generateDF.json b/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generateDF.json index a496a3e9eb5..5e503c3f83e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generateDF.json +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generateDF.json @@ -30,12 +30,6 @@ "fields": [ "col1" ], - "exprs": [ - { - "input": 0, - "name": "$0" - } - ], "inputs": [ 1 ] @@ -44,24 +38,6 @@ "id": 3, "relOp": "LogicalAggregate", "group": [], - "aggs": [ - { - "agg": { - "name": "SUM", - "kind": "SUM", - "syntax": "FUNCTION" - }, - "type": { - "type": "INTEGER", - "nullable": true - }, - "distinct": false, - "operands": [ - 0 - ], - "name": "EXPR$0" - } - ], "inputs": [ 2 ] diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generatePlan.json b/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generatePlan.json index 1a41274df0f..b3b110f7fff 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generatePlan.json +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/resources/metadataTests-generatePlan.json @@ -29,12 +29,6 @@ "fields": [ "col1" ], - "exprs": [ - { - "input": 0, - "name": "$0" - } - ], "inputs": [ 1 ] @@ -58,12 +52,6 @@ "fields": [ "col1" ], - "exprs": [ - { - "input": 0, - "name": "$0" - } - ], "inputs": [ 3 ] @@ -72,24 +60,6 @@ "id": 5, "relOp": "LogicalAggregate", "group": [], - "aggs": [ - { - "agg": { - "name": "SUM", - "kind": "SUM", - "syntax": "FUNCTION" - }, - "type": { - "type": "INTEGER", - "nullable": true - }, - "distinct": false, - "operands": [ - 0 - ], - "name": "EXPR$0" - } - ], "inputs": [ 4 ] diff --git a/sql-to-dbsp-compiler/slt/out17544834234273802340.tmp b/sql-to-dbsp-compiler/slt/out17544834234273802340.tmp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sql-to-dbsp-compiler/slt/src/test/java/org/dbsp/sqllogictest/DBTests.java b/sql-to-dbsp-compiler/slt/src/test/java/org/dbsp/sqllogictest/DBTests.java index 5c27a19c15f..47a37fa479a 100644 --- a/sql-to-dbsp-compiler/slt/src/test/java/org/dbsp/sqllogictest/DBTests.java +++ b/sql-to-dbsp-compiler/slt/src/test/java/org/dbsp/sqllogictest/DBTests.java @@ -68,16 +68,4 @@ strcol varchar(25)) Assert.assertEquals(direct.toString(), throughCalcite.toString()); } } - - @Test @Ignore("Fails due to a bug in HSQLDB") - public void HSQLDBDoubleNegTest() throws SQLException { - // Reproduction for https://sourceforge.net/p/hsqldb/bugs/1680/ - // and https://sourceforge.net/p/hsqldb/bugs/1681/ - String jdbcUrl = "jdbc:hsqldb:mem:db"; - Connection connection = DriverManager.getConnection(jdbcUrl, "", ""); - try (Statement s = connection.createStatement()) { - s.execute("SELECT +2;"); - s.execute("SELECT - -2;"); - } - } }