From 87e1d640f9ee91324dfe8c7978a80de6afcfe8b1 Mon Sep 17 00:00:00 2001 From: Adrian Todt Date: Thu, 26 Mar 2020 17:13:37 -0300 Subject: [PATCH 1/4] Support for kotlin-main-kts --- .gitignore | 24 ++- build.gradle.kts | 166 ++++++++++---------- gradle.properties | 5 + scripts.properties | 16 ++ scripts/run-kts | 21 +++ scripts/task-downloadProtoAndTests.main.kts | 36 +++++ 6 files changed, 174 insertions(+), 94 deletions(-) create mode 100644 scripts.properties create mode 100755 scripts/run-kts create mode 100644 scripts/task-downloadProtoAndTests.main.kts diff --git a/.gitignore b/.gitignore index dafefef7..1e16e2fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,25 @@ -build/ -out/ -.gradle/ +# ## RethinkDB .gitignore + +# General .#* + +# IDE-related *.iml .idea/ +.vscode/ +out/ + +# Java-related +*.class + +# Gradle-related +build/ +.gradle/ confidential.properties +# Kotlin scripts +scripts/kotlinc/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -27,7 +41,3 @@ virtualenv/ # RethinkDB scripts/*.proto - -# Editors -.vscode/ -.idea/ diff --git a/build.gradle.kts b/build.gradle.kts index cc54ac80..db817ebf 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,8 @@ -import java.util.Properties -import java.io.File import com.jfrog.bintray.gradle.BintrayExtension import com.jfrog.bintray.gradle.tasks.BintrayUploadTask import com.jfrog.bintray.gradle.tasks.RecordingCopyTask - +import java.io.File +import java.util.* plugins { java @@ -32,27 +31,23 @@ dependencies { compile("com.fasterxml.jackson.core:jackson-databind:2.10.2") } -file("confidential.properties").takeIf(File::exists)?.let { - val properties = Properties() - it.inputStream().use(properties::load) - allprojects { properties.forEach { name, value -> extra.set(name.toString(), value) } } -} +file("confidential.properties").takeIf(File::exists) + ?.let { Properties().apply { it.inputStream().use(this::load) } } + ?.let { allprojects { it.forEach { name, value -> extra.set(name.toString(), value) } } } + +fun String.propertyValue() = project.findProperty(this) as String? +fun File.child(child: String) = File(this, child) +fun download(src: String, dest: File) = ant.invokeMethod("get", mapOf("src" to src, "dest" to dest)) +fun unzip(src: File, dest: File) = ant.invokeMethod("unzip", mapOf("src" to src, "dest" to dest)) gradle.taskGraph.whenReady { - val hasUploadArchives = hasTask(":uploadArchives") - val hasBintrayUpload = hasTask(":bintrayUpload") - val hasDoSigning = hasTask(":doSigning") - signing.isRequired = hasBintrayUpload || hasUploadArchives || hasDoSigning + signing.isRequired = listOf(":bintrayUpload", ":uploadArchives", ":doSigning").any(this::hasTask) } -fun findProperty(s: String) = project.findProperty(s) as String? tasks { - val doSigning by creating { - dependsOn("signArchives") - } - withType { - options.encoding = "UTF-8" - } + create("doSigning") { dependsOn("signArchives") } + withType { options.encoding = "UTF-8" } + val sourcesJar by creating(Jar::class) { group = "build" description = "Generates a jar with the sources" @@ -74,49 +69,49 @@ tasks { add("archives", javadocJar) } - val downloadProtoAndTests by creating { - group = "build setup" - description = "Downloads contents from rethinkdb main repository." + create("setupKotlinScriptsEnv") { + val ktVersion = "build.scripts.kotlin_version".propertyValue() ?: "1.3.71" - //properties - val rethinkdb_repo = findProperty("build.rethinkdb_repo")!! - val rethinkdb_branch = findProperty("build.rethinkdb_branch")!! - val checkout_dir = findProperty("build.rethinkdb_checkout_dir")!! - val proto_location = findProperty("build.proto.src_location")!! - val proto_target = findProperty("build.proto.target_folder")!! - val tests_location = findProperty("build.tests.src_location")!! - val tests_target = findProperty("build.tests.target_folder")!! - - val proto_folder = File(buildDir, "rethinkdb_gen/$proto_target") - val tests_folder = File(buildDir, "rethinkdb_gen/$tests_target") + val downloaded = buildDir.child("downloaded") + val kotlinc = downloaded.child("kotlinc.zip") + val mainKts = downloaded.child("kotlin-main-kts.jar") + val kotlincDir = projectDir.child("scripts").child("kotlinc") doLast { - File(buildDir, "rethinkdb_gen").mkdirs() - delete(checkout_dir, proto_folder, tests_folder) - exec { - commandLine("git", "clone", "--progress", "-b", rethinkdb_branch, "--single-branch", rethinkdb_repo, checkout_dir) - } - exec { - commandLine("cp", "-a", "-R", "$checkout_dir/$proto_location/.", proto_folder.absolutePath) - } - exec { - commandLine("cp", "-a", "-R", "$checkout_dir/$tests_location/.", tests_folder.absolutePath) - } + downloaded.deleteRecursively() + downloaded.mkdirs() + logger.lifecycle("Downloading Kotlin compiler and kotlin-main-kts, may take a while...") + download("https://github.com/JetBrains/kotlin/releases/download/v$ktVersion/kotlin-compiler-$ktVersion.zip", kotlinc) + download("https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-main-kts/$ktVersion/kotlin-main-kts-$ktVersion.jar", mainKts) + logger.lifecycle("Finished downloading, setting up environiment...") + kotlincDir.deleteRecursively() + unzip(kotlinc, projectDir.child("scripts")) + mainKts.renameTo(kotlincDir.child(mainKts.name)) } } + projectDir.child("scripts").listFiles()!! + .filter { it.name.startsWith("task-") && it.name.endsWith(".main.kts") } + .forEach { + create(it.name.removeSurrounding("task-", ".main.kts")) { + group = "kotlin scripts" + description = "Runs ${it.name}" + doLast { exec { commandLine("./scripts/run-kts", it.path) } } + } + } + val generateJsonFiles by creating { group = "code generation" description = "Generates json files for the java file generation." - val convert_proto = findProperty("build.gen.py.convert_proto")!! - val metajava = findProperty("build.gen.py.metajava")!! - val json_target = findProperty("build.gen.json.target_folder")!! - val proto_folder = findProperty("build.proto.target_folder")!! - val proto_name = findProperty("build.proto.file_name")!! - val proto_basic_name = findProperty("build.gen.json.proto_basic")!! - val term_info_name = findProperty("build.gen.json.term_info")!! - val java_term_info_name = findProperty("build.gen.json.java_term_info")!! + val convert_proto = "build.gen.py.convert_proto".propertyValue()!! + val metajava = "build.gen.py.metajava".propertyValue()!! + val json_target = "build.gen.json.target_folder".propertyValue()!! + val proto_folder = "build.proto.target_folder".propertyValue()!! + val proto_name = "build.proto.file_name".propertyValue()!! + val proto_basic_name = "build.gen.json.proto_basic".propertyValue()!! + val term_info_name = "build.gen.json.term_info".propertyValue()!! + val java_term_info_name = "build.gen.json.java_term_info".propertyValue()!! val json_folder = File(buildDir, "rethinkdb_gen/$json_target") val proto_file = File(buildDir, "rethinkdb_gen/$proto_folder/$proto_name") @@ -156,18 +151,18 @@ tasks { group = "code generation" description = "Generates java files for the driver." - val localFiles = findProperty("build.gen.use_local_files")!!.toBoolean() + val localFiles = "build.gen.use_local_files".propertyValue()!!.toBoolean() enabled = localFiles // TODO enable this once we fix update-terminfo - val metajava = findProperty("build.gen.py.metajava")!! - val json_target = if (localFiles) "../../scripts" else findProperty("build.gen.json.target_folder")!! - val proto_basic_name = findProperty("build.gen.json.proto_basic")!! - val global_info = findProperty("build.json.global_info")!! - val java_term_info_name = findProperty("build.gen.json.java_term_info")!! - val src_main = findProperty("build.gen.src.main")!! - val templates = findProperty("build.gen.src.templates")!! - val folders = findProperty("build.gen.src.main.packages")!!.split(',') + val metajava = "build.gen.py.metajava".propertyValue()!! + val json_target = if (localFiles) "../../scripts" else "build.gen.json.target_folder".propertyValue()!! + val proto_basic_name = "build.gen.json.proto_basic".propertyValue()!! + val global_info = "build.json.global_info".propertyValue()!! + val java_term_info_name = "build.gen.json.java_term_info".propertyValue()!! + val src_main = "build.gen.src.main".propertyValue()!! + val templates = "build.gen.src.templates".propertyValue()!! + val folders = "build.gen.src.main.packages".propertyValue()!!.split(',') val proto_basic = File(buildDir, "rethinkdb_gen/$json_target/$proto_basic_name") val java_term_info = File(buildDir, "rethinkdb_gen/$json_target/$java_term_info_name") @@ -195,10 +190,10 @@ tasks { description = "Generates test files for the driver." //properties - val convert_tests = findProperty("build.gen.py.convert_tests")!! - val tests_target = findProperty("build.tests.target_folder")!! - val src_test = findProperty("build.gen.src.test")!! - val templates = findProperty("build.gen.src.templates")!! + val convert_tests = "build.gen.py.convert_tests".propertyValue()!! + val tests_target = "build.tests.target_folder".propertyValue()!! + val src_test = "build.gen.src.test".propertyValue()!! + val templates = "build.gen.src.templates".propertyValue()!! val tests_folder = File(buildDir, "rethinkdb_gen/$tests_target") val src_test_gen = File("$src_test/gen") @@ -217,7 +212,6 @@ tasks { } } - getByName("uploadArchives") { repositories { withConvention(MavenRepositoryHandlerConvention::class) { @@ -227,14 +221,14 @@ tasks { withGroovyBuilder { "repository"("url" to uri("https://oss.sonatype.org/service/local/staging/deploy/maven2/")) { "authentication"( - "userName" to findProperty("ossrhUsername"), - "password" to findProperty("ossrhPassword") + "userName" to "ossrhUsername".propertyValue(), + "password" to "ossrhPassword".propertyValue() ) } "snapshotRepository"("url" to uri("https://oss.sonatype.org/content/repositories/snapshots/")) { "authentication"( - "userName" to findProperty("ossrhUsername"), - "password" to findProperty("ossrhPassword") + "userName" to "ossrhUsername".propertyValue(), + "password" to "ossrhPassword".propertyValue() ) } } @@ -296,32 +290,30 @@ tasks { root.appendNode("url", "http://rethinkdb.com") val scm = root.appendNode("scm") - scm.appendNode("connection","scm:git:https://github.com/rethinkdb/rethinkdb-java") - scm.appendNode("developerConnection","scm:git:https://github.com/rethinkdb/rethinkdb-java") + scm.appendNode("connection", "scm:git:https://github.com/rethinkdb/rethinkdb-java") + scm.appendNode("developerConnection", "scm:git:https://github.com/rethinkdb/rethinkdb-java") scm.appendNode("url", "https://github.com/rethinkdb/rethinkdb-java") val license = root.appendNode("licenses").appendNode("license") - license.appendNode("name","The Apache License, Version 2.0") - license.appendNode("url","http://www.apache.org/licenses/LICENSE-2.0.txt") + license.appendNode("name", "The Apache License, Version 2.0") + license.appendNode("url", "http://www.apache.org/licenses/LICENSE-2.0.txt") val developers = root.appendNode("developers") val dev1 = developers.appendNode("developer") - dev1.appendNode("id","adriantodt") - dev1.appendNode("name","Adrian Todt") - dev1.appendNode("email","adriantodt.ms@gmail.com") + dev1.appendNode("id", "adriantodt") + dev1.appendNode("name", "Adrian Todt") + dev1.appendNode("email", "adriantodt.ms@gmail.com") val dev2 = developers.appendNode("developer") - dev2.appendNode("id","gabor-boros") - dev2.appendNode("name","Gábor Boros") - dev2.appendNode("email","gabor@rethinkdb.com") + dev2.appendNode("id", "gabor-boros") + dev2.appendNode("name", "Gábor Boros") + dev2.appendNode("email", "gabor@rethinkdb.com") } } } - withType { - dependsOn("assemble", "publishToMavenLocal") - } + withType { dependsOn("assemble", "publishToMavenLocal") } } signing { @@ -331,8 +323,8 @@ signing { } bintray { - user = findProperty("bintray.user") - key = findProperty("bintray.key") + user = "bintray.user".propertyValue() + key = "bintray.key".propertyValue() publish = true setPublications("mavenJava") @@ -357,8 +349,8 @@ bintray { vcsUrl = "https://github.com/rethinkdb/rethinkdb-java.git" version(delegateClosureOf { mavenCentralSync(delegateClosureOf { - user = findProperty("ossrhUsername") - password = findProperty("ossrhPassword") + user = "ossrhUsername".propertyValue() + password = "ossrhPassword".propertyValue() sync = !user.isNullOrBlank() && !password.isNullOrBlank() }) }) diff --git a/gradle.properties b/gradle.properties index 2be1fb64..35ab4889 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,11 @@ # ## RethinkDB gradle.properties # This file is used by tasks `downloadProtoAndTests`, `generateJsonFiles`, `genMainJava` and `genTestJava`. +# Kotlin version for running scripts +build.scripts.kotlin_version=1.3.71 + +build.scripts.always_download=false + # Checkout repo for git (change here to ssh checkout if you want) build.rethinkdb_repo=https://github.com/rethinkdb/rethinkdb.git diff --git a/scripts.properties b/scripts.properties new file mode 100644 index 00000000..45e37cf4 --- /dev/null +++ b/scripts.properties @@ -0,0 +1,16 @@ +# ## RethinkDB Scripts Properties + +sources.rethinkdbRepo=https://github.com/rethinkdb/rethinkdb.git +sources.rethinkdbBranch=next +sources.checkoutDir=/tmp/rethinkdb + +sources.protofile.sourceFile=src/rdb_protocol/ql2.proto +sources.protofile.targetFolder=build/rethinkdb_gen/rdb_protocol + +sources.tests.sourceFolder=test/rql_test/src +sources.tests.targetFolder=build/rethinkdb_gen/rql_test_src + +generation.jsonFiles.targetFolder=build/rethinkdb_gen/rdb_json + +generation.javaFiles.mainSrc=src/main/java/com/rethinkdb +generation.javaFiles.testSrc=src/test/java/com/rethinkdb \ No newline at end of file diff --git a/scripts/run-kts b/scripts/run-kts new file mode 100755 index 00000000..cad361a4 --- /dev/null +++ b/scripts/run-kts @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +dir="`dirname "$0"`" +if [[ ! -d $dir/kotlinc/ ]] +then + echo "Script environiment not set up." + exit 1 +fi +if [[ ! -f $dir/kotlinc/bin/kotlinc ]] +then + echo "Missing kotlinc binary." + exit 1 +fi +if [[ ! -f $dir/kotlinc/kotlin-main-kts.jar ]] +then + echo "Missing kotlin-main-kts.jar binary." + exit 1 +fi +chmod +x $dir/kotlinc/bin/kotlinc +src=$1 +shift 1 +$dir/kotlinc/bin/kotlinc -cp $dir/kotlinc/kotlin-main-kts.jar -script $src -- $@ \ No newline at end of file diff --git a/scripts/task-downloadProtoAndTests.main.kts b/scripts/task-downloadProtoAndTests.main.kts new file mode 100644 index 00000000..cef1c1b2 --- /dev/null +++ b/scripts/task-downloadProtoAndTests.main.kts @@ -0,0 +1,36 @@ +@file:Repository(url = "https://jcenter.bintray.com") +@file:DependsOn("pw.aru.libs:properties:1.2") + +import pw.aru.libs.properties.Properties +import java.io.File +import kotlin.system.exitProcess + +fun system(vararg command: String) = ProcessBuilder().inheritIO().command(*command).start().waitFor() + +val properties = Properties.fromFile("scripts.properties") + +// Delete all target folders +listOfNotNull( + properties["sources.checkoutDir"], + properties["sources.protofile.targetFolder"], + properties["sources.tests.targetFolder"] +).forEach { File(it).deleteRecursively() } + +// Git clone +val checkoutDir = properties["sources.checkoutDir"] ?: "/tmp/rethinkdb" +system( + "git", "clone", "--progress", "-b", properties["sources.rethinkdbBranch"] ?: "next", "--single-branch", + properties["sources.rethinkdbRepo"]!!, checkoutDir +) + +// Move protofile +File(checkoutDir, properties["sources.protofile.sourceFile"]!!).let { + it.renameTo(File(properties["sources.protofile.targetFolder"]!!, it.name).apply { parentFile.mkdirs() }) +} + +// Move test source folder +File(checkoutDir, properties["sources.tests.sourceFolder"]!!).renameTo( + File(properties["sources.tests.targetFolder"]!!).apply { parentFile.mkdirs() } +) + +exitProcess(0) \ No newline at end of file From 738ac2857970dd558ffd945e9682e7e513904a5a Mon Sep 17 00:00:00 2001 From: Adrian Todt Date: Mon, 30 Mar 2020 23:40:12 -0300 Subject: [PATCH 2/4] Protofile parsing --- scripts.properties | 2 +- scripts/script-parseProtoFile.main.kts | 163 ++++++++++++++++++++ scripts/task-downloadProtoAndTests.main.kts | 8 +- 3 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 scripts/script-parseProtoFile.main.kts diff --git a/scripts.properties b/scripts.properties index 45e37cf4..291ba9ed 100644 --- a/scripts.properties +++ b/scripts.properties @@ -5,7 +5,7 @@ sources.rethinkdbBranch=next sources.checkoutDir=/tmp/rethinkdb sources.protofile.sourceFile=src/rdb_protocol/ql2.proto -sources.protofile.targetFolder=build/rethinkdb_gen/rdb_protocol +sources.protofile.targetFile=build/rethinkdb_gen/rdb_protocol/ql2.proto sources.tests.sourceFolder=test/rql_test/src sources.tests.targetFolder=build/rethinkdb_gen/rql_test_src diff --git a/scripts/script-parseProtoFile.main.kts b/scripts/script-parseProtoFile.main.kts new file mode 100644 index 00000000..6465836d --- /dev/null +++ b/scripts/script-parseProtoFile.main.kts @@ -0,0 +1,163 @@ +@file:Repository(url = "https://jcenter.bintray.com") +@file:Repository(url = "https://dl.bintray.com/adriantodt/maven") +@file:DependsOn("pw.aru.libs:properties:1.2") +@file:DependsOn("com.github.adriantodt:tartar:1.1.1") +@file:DependsOn("com.fasterxml.jackson.core:jackson-databind:2.10.2") + +import com.fasterxml.jackson.databind.ObjectMapper +import com.github.adriantodt.tartar.api.lexer.Source +import com.github.adriantodt.tartar.api.parser.SyntaxException +import com.github.adriantodt.tartar.api.parser.Token +import com.github.adriantodt.tartar.createGrammar +import com.github.adriantodt.tartar.createLexer +import com.github.adriantodt.tartar.createParser +import com.github.adriantodt.tartar.extensions.* +import pw.aru.libs.properties.Properties +import java.io.File + +enum class TokenType { + ASSIGN, + SEMICOLON, + LBRACKET, + RBRACKET, + LBRACE, + RBRACE, + + STRING, + INTEGER, + DECIMAL, + BOOLEAN, + + IDENTIFIER, + MESSAGE, + ENUM, + SYNTAX, + OPTIONAL, + REPEATED, + + INVALID +} + +sealed class ProtoExpr { + object Ignore : ProtoExpr() + data class ModuleNode(val moduleName: String, val child: List) : ProtoExpr() + data class EnumNode(val enumName: String, val child: List) : ProtoExpr() + data class EnumValueExpr(val name: String, val value: Int) : ProtoExpr() + data class ModuleValueExpr( + val type: String, val name: String, val value: Int, + val modifier: Modifier? = null, val options: Map? = null + ) : ProtoExpr() { + enum class Modifier { + OPTIONAL, REPEATED + } + } +} + +val properties = Properties.fromFile("scripts.properties") +val source = Source(File(properties["sources.protofile.targetFile"]!!)) + +val lexer = createLexer> { + ' '() + '\n'() + '=' { process(makeToken(TokenType.ASSIGN)) } + ';' { process(makeToken(TokenType.SEMICOLON)) } + '{' { process(makeToken(TokenType.LBRACKET)) } + '}' { process(makeToken(TokenType.RBRACKET)) } + '[' { process(makeToken(TokenType.LBRACE)) } + ']' { process(makeToken(TokenType.RBRACE)) } + "//" { while (hasNext()) if (next() == '\n') break } + "/*" { while (hasNext()) if (next() == '*' && match('/')) break } + '"' { process(makeToken(TokenType.STRING, readString(it))) } + matching { it.isDigit() }.configure { + process(when (val n = readNumber(it)) { + is LexicalNumber.Invalid -> makeToken(TokenType.INVALID, n.string) + is LexicalNumber.Decimal -> makeToken(TokenType.DECIMAL, n.value.toString()) + is LexicalNumber.Integer -> makeToken(TokenType.INTEGER, n.value.toString()) + }) + } + matching { it.isLetter() || it == '_' }.configure { + process(when (val s = readIdentifier(it)) { + "syntax" -> makeToken(TokenType.SYNTAX, s) + "message" -> makeToken(TokenType.MESSAGE, s) + "enum" -> makeToken(TokenType.ENUM, s) + "repeated" -> makeToken(TokenType.REPEATED, s) + "optional" -> makeToken(TokenType.OPTIONAL, s) + "true", "false" -> makeToken(TokenType.BOOLEAN, s) + else -> makeToken(TokenType.IDENTIFIER, s) + }) + } +} + +val grammar = createGrammar { + prefix(TokenType.SYNTAX) { + eat(TokenType.ASSIGN) + eat(TokenType.STRING) + eat(TokenType.SEMICOLON) + ProtoExpr.Ignore + } + prefix(TokenType.MESSAGE) { + val name = eat(TokenType.IDENTIFIER).value + eat(TokenType.LBRACKET) + val stmt = ArrayList() + while (!match(TokenType.RBRACKET)) stmt += parseExpression() + stmt.removeIf(ProtoExpr.Ignore::equals) + ProtoExpr.ModuleNode(name, stmt) + } + prefix(TokenType.ENUM) { + val name = eat(TokenType.IDENTIFIER).value + eat(TokenType.LBRACKET) + val stmt = ArrayList() + while (!match(TokenType.RBRACKET)) stmt += parseExpression() + stmt.removeIf(ProtoExpr.Ignore::equals) + ProtoExpr.ModuleNode(name, stmt) + } + prefix(TokenType.OPTIONAL) { + val m = parseExpression() as? ProtoExpr.ModuleValueExpr + ?: throw SyntaxException("Expected a module value", it.section) + m.copy(modifier = ProtoExpr.ModuleValueExpr.Modifier.OPTIONAL) + } + prefix(TokenType.REPEATED) { + val m = parseExpression() as? ProtoExpr.ModuleValueExpr + ?: throw SyntaxException("Expected a module value", it.section) + m.copy(modifier = ProtoExpr.ModuleValueExpr.Modifier.REPEATED) + } + prefix(TokenType.IDENTIFIER) { + if (nextIs(TokenType.IDENTIFIER)) { + val type = it.value + val name = eat().value + eat(TokenType.ASSIGN) + val value = eat(TokenType.INTEGER).value.toInt() + val options = if (match(TokenType.LBRACE)) { + val optionName = eat().value + eat(TokenType.ASSIGN) + val optionValue: Any = eat().let { optVal -> + when (optVal.type) { + TokenType.STRING -> optVal.value.removeSurrounding("\"") + TokenType.INTEGER -> optVal.value.toInt() + TokenType.BOOLEAN -> optVal.value.toBoolean() + else -> throw SyntaxException("Unexpected $optVal", optVal.section) + } + } + eat(TokenType.RBRACE) + mapOf(optionName to optionValue) + } else null + eat(TokenType.SEMICOLON) + ProtoExpr.ModuleValueExpr(type, name, value, options = options) + } else { + val name = it.value + eat(TokenType.ASSIGN) + val value = eat(TokenType.INTEGER).value.toInt() + eat(TokenType.SEMICOLON) + ProtoExpr.EnumValueExpr(name, value) + } + } +} + +val parser = createParser(grammar) { + val stmt = ArrayList() + while (!eof) stmt += parseExpression() + stmt.removeIf(ProtoExpr.Ignore::equals) + stmt +} + +ObjectMapper().writeValueAsString(parser.parse(source, lexer)) \ No newline at end of file diff --git a/scripts/task-downloadProtoAndTests.main.kts b/scripts/task-downloadProtoAndTests.main.kts index cef1c1b2..53d2066d 100644 --- a/scripts/task-downloadProtoAndTests.main.kts +++ b/scripts/task-downloadProtoAndTests.main.kts @@ -24,12 +24,12 @@ system( ) // Move protofile -File(checkoutDir, properties["sources.protofile.sourceFile"]!!).let { - it.renameTo(File(properties["sources.protofile.targetFolder"]!!, it.name).apply { parentFile.mkdirs() }) -} +File(checkoutDir, properties["sources.protofile.sourceFile"]!!).copyTo( + File(properties["sources.protofile.targetFile"]!!).apply { parentFile.mkdirs() } +) // Move test source folder -File(checkoutDir, properties["sources.tests.sourceFolder"]!!).renameTo( +File(checkoutDir, properties["sources.tests.sourceFolder"]!!).copyRecursively( File(properties["sources.tests.targetFolder"]!!).apply { parentFile.mkdirs() } ) From cc9dd67b813f435caa034abcb52df2584a49d0fb Mon Sep 17 00:00:00 2001 From: Adrian Todt Date: Tue, 31 Mar 2020 13:34:21 -0300 Subject: [PATCH 3/4] Update script to use smaller implementation --- scripts.properties | 2 +- scripts/script-parseProtoFile.main.kts | 80 +++++++++----------------- 2 files changed, 27 insertions(+), 55 deletions(-) diff --git a/scripts.properties b/scripts.properties index 291ba9ed..66d27c9d 100644 --- a/scripts.properties +++ b/scripts.properties @@ -10,7 +10,7 @@ sources.protofile.targetFile=build/rethinkdb_gen/rdb_protocol/ql2.proto sources.tests.sourceFolder=test/rql_test/src sources.tests.targetFolder=build/rethinkdb_gen/rql_test_src -generation.jsonFiles.targetFolder=build/rethinkdb_gen/rdb_json +convert.protofile.targetFile=build/rethinkdb_gen/rdb_json/ql2.proto.json generation.javaFiles.mainSrc=src/main/java/com/rethinkdb generation.javaFiles.testSrc=src/test/java/com/rethinkdb \ No newline at end of file diff --git a/scripts/script-parseProtoFile.main.kts b/scripts/script-parseProtoFile.main.kts index 6465836d..68a897dc 100644 --- a/scripts/script-parseProtoFile.main.kts +++ b/scripts/script-parseProtoFile.main.kts @@ -1,7 +1,7 @@ @file:Repository(url = "https://jcenter.bintray.com") @file:Repository(url = "https://dl.bintray.com/adriantodt/maven") @file:DependsOn("pw.aru.libs:properties:1.2") -@file:DependsOn("com.github.adriantodt:tartar:1.1.1") +@file:DependsOn("com.github.adriantodt:tartar:1.1.2") @file:DependsOn("com.fasterxml.jackson.core:jackson-databind:2.10.2") import com.fasterxml.jackson.databind.ObjectMapper @@ -16,26 +16,9 @@ import pw.aru.libs.properties.Properties import java.io.File enum class TokenType { - ASSIGN, - SEMICOLON, - LBRACKET, - RBRACKET, - LBRACE, - RBRACE, - - STRING, - INTEGER, - DECIMAL, - BOOLEAN, - - IDENTIFIER, - MESSAGE, - ENUM, - SYNTAX, - OPTIONAL, - REPEATED, - - INVALID + LBRACKET, RBRACKET, LBRACE, RBRACE, ASSIGN, SEMICOLON, + STRING, INTEGER, DECIMAL, BOOLEAN, + IDENTIFIER, MESSAGE, ENUM, SYNTAX, OPTIONAL, REPEATED } sealed class ProtoExpr { @@ -44,18 +27,12 @@ sealed class ProtoExpr { data class EnumNode(val enumName: String, val child: List) : ProtoExpr() data class EnumValueExpr(val name: String, val value: Int) : ProtoExpr() data class ModuleValueExpr( - val type: String, val name: String, val value: Int, - val modifier: Modifier? = null, val options: Map? = null + val type: String, val name: String, val value: Int, val options: Map?, val modifier: Modifier? ) : ProtoExpr() { - enum class Modifier { - OPTIONAL, REPEATED - } + enum class Modifier { OPTIONAL, REPEATED } } } -val properties = Properties.fromFile("scripts.properties") -val source = Source(File(properties["sources.protofile.targetFile"]!!)) - val lexer = createLexer> { ' '() '\n'() @@ -70,7 +47,7 @@ val lexer = createLexer> { '"' { process(makeToken(TokenType.STRING, readString(it))) } matching { it.isDigit() }.configure { process(when (val n = readNumber(it)) { - is LexicalNumber.Invalid -> makeToken(TokenType.INVALID, n.string) + is LexicalNumber.Invalid -> throw SyntaxException("Invalid number '${n.string}'", section(n.string.length)) is LexicalNumber.Decimal -> makeToken(TokenType.DECIMAL, n.value.toString()) is LexicalNumber.Integer -> makeToken(TokenType.INTEGER, n.value.toString()) }) @@ -90,26 +67,22 @@ val lexer = createLexer> { val grammar = createGrammar { prefix(TokenType.SYNTAX) { - eat(TokenType.ASSIGN) - eat(TokenType.STRING) - eat(TokenType.SEMICOLON) + eatMulti(TokenType.ASSIGN, TokenType.STRING, TokenType.SEMICOLON) ProtoExpr.Ignore } prefix(TokenType.MESSAGE) { - val name = eat(TokenType.IDENTIFIER).value - eat(TokenType.LBRACKET) + val (name) = eatMulti(TokenType.IDENTIFIER, TokenType.LBRACKET) val stmt = ArrayList() while (!match(TokenType.RBRACKET)) stmt += parseExpression() stmt.removeIf(ProtoExpr.Ignore::equals) - ProtoExpr.ModuleNode(name, stmt) + ProtoExpr.ModuleNode(name.value, stmt) } prefix(TokenType.ENUM) { - val name = eat(TokenType.IDENTIFIER).value - eat(TokenType.LBRACKET) + val (name) = eatMulti(TokenType.IDENTIFIER, TokenType.LBRACKET) val stmt = ArrayList() while (!match(TokenType.RBRACKET)) stmt += parseExpression() stmt.removeIf(ProtoExpr.Ignore::equals) - ProtoExpr.ModuleNode(name, stmt) + ProtoExpr.EnumNode(name.value, stmt) } prefix(TokenType.OPTIONAL) { val m = parseExpression() as? ProtoExpr.ModuleValueExpr @@ -123,32 +96,25 @@ val grammar = createGrammar { } prefix(TokenType.IDENTIFIER) { if (nextIs(TokenType.IDENTIFIER)) { - val type = it.value - val name = eat().value - eat(TokenType.ASSIGN) - val value = eat(TokenType.INTEGER).value.toInt() + val (name, _, value) = eatMulti(TokenType.IDENTIFIER, TokenType.ASSIGN, TokenType.INTEGER) val options = if (match(TokenType.LBRACE)) { - val optionName = eat().value - eat(TokenType.ASSIGN) + val (optionKey) = eatMulti(TokenType.IDENTIFIER, TokenType.ASSIGN) val optionValue: Any = eat().let { optVal -> when (optVal.type) { - TokenType.STRING -> optVal.value.removeSurrounding("\"") + TokenType.STRING -> optVal.value TokenType.INTEGER -> optVal.value.toInt() TokenType.BOOLEAN -> optVal.value.toBoolean() else -> throw SyntaxException("Unexpected $optVal", optVal.section) } } eat(TokenType.RBRACE) - mapOf(optionName to optionValue) + mapOf(optionKey.value to optionValue) } else null eat(TokenType.SEMICOLON) - ProtoExpr.ModuleValueExpr(type, name, value, options = options) + ProtoExpr.ModuleValueExpr(it.value, name.value, value.value.toInt(), options, null) } else { - val name = it.value - eat(TokenType.ASSIGN) - val value = eat(TokenType.INTEGER).value.toInt() - eat(TokenType.SEMICOLON) - ProtoExpr.EnumValueExpr(name, value) + val (_, value) = eatMulti(TokenType.ASSIGN, TokenType.INTEGER, TokenType.SEMICOLON) + ProtoExpr.EnumValueExpr(it.value, value.value.toInt()) } } } @@ -160,4 +126,10 @@ val parser = createParser(grammar) { stmt } -ObjectMapper().writeValueAsString(parser.parse(source, lexer)) \ No newline at end of file +val properties = Properties.fromFile("scripts.properties") +val source = Source(File(properties["sources.protofile.targetFile"]!!)) +val target = File(properties["convert.protofile.targetFile"]!!) +target.parentFile.mkdirs() +target.delete() + +ObjectMapper().writerWithDefaultPrettyPrinter().writeValue(target, parser.parse(source, lexer)) \ No newline at end of file From 1cbbbf0c4677512a3eb19a13a1767738646ed74a Mon Sep 17 00:00:00 2001 From: Adrian Todt Date: Mon, 20 Apr 2020 00:51:04 -0300 Subject: [PATCH 4/4] Script used to generate raw reql terms. --- scripts/script-createTermsJson.main.kts | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 scripts/script-createTermsJson.main.kts diff --git a/scripts/script-createTermsJson.main.kts b/scripts/script-createTermsJson.main.kts new file mode 100644 index 00000000..73732a8f --- /dev/null +++ b/scripts/script-createTermsJson.main.kts @@ -0,0 +1,28 @@ +@file:Repository(url = "https://jcenter.bintray.com") +@file:Repository(url = "https://dl.bintray.com/adriantodt/maven") +@file:DependsOn("pw.aru.libs:properties:1.2") +@file:DependsOn("com.fasterxml.jackson.core:jackson-databind:2.10.2") +@file:Suppress("UNCHECKED_CAST") + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import pw.aru.libs.properties.Properties +import java.io.File + +val properties = Properties.fromFile("scripts.properties") +val target = File(properties["convert.protofile.targetFile"]!!) + +val mapper = ObjectMapper() + +val ql2 = mapper.readValue(target, object : TypeReference>>() {}) +val term = ql2.first { it["moduleName"] == "Term" }["child"] as List> +val termType = term.first { it["enumName"] == "TermType" }["child"] as List> + +val folder = File("scripts/terms") +folder.mkdirs() + +termType.forEach { + val name = it["name"].toString().toLowerCase() + ".json" + val map = mapOf("name" to it["name"], "id" to it["value"]) + mapper.writerWithDefaultPrettyPrinter().writeValue(File(folder, name), map) +} \ No newline at end of file