diff --git a/.github/workflows/java-wasm-bindings.yml b/.github/workflows/java-wasm-bindings.yml index 3a0a066334..679fb09a8b 100644 --- a/.github/workflows/java-wasm-bindings.yml +++ b/.github/workflows/java-wasm-bindings.yml @@ -38,7 +38,7 @@ jobs: uses: actions/setup-java@v5 with: distribution: 'temurin' - java-version: '17' + java-version: '21' cache: maven - name: Run the tests diff --git a/.gitignore b/.gitignore index 6706482b64..218a76cd00 100644 --- a/.gitignore +++ b/.gitignore @@ -39,9 +39,9 @@ out.svg /javascript/src/visitor.js /javascript/src/prism.wasm /javascript/src/*.d.ts -/java/org/prism/AbstractNodeVisitor.java -/java/org/prism/Loader.java -/java/org/prism/Nodes.java +/java/org/ruby_lang/prism/AbstractNodeVisitor.java +/java/org/ruby_lang/prism/Loader.java +/java/org/ruby_lang/prism/Nodes.java /lib/prism/compiler.rb /lib/prism/dispatcher.rb /lib/prism/dot_visitor.rb @@ -74,3 +74,5 @@ compile_commands.json .vscode/ tags + +wasi-sdk* diff --git a/Gemfile.lock b/Gemfile.lock index 7e23a63cd5..957a9ec6ba 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -65,6 +65,7 @@ GEM PLATFORMS java ruby + universal-java DEPENDENCIES benchmark-ips diff --git a/Makefile b/Makefile index 265d478734..d531e13774 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ SOEXT ?= $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]') CPPFLAGS := -Iinclude $(CPPFLAGS) CFLAGS := -g -O2 -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -Wno-missing-braces -fPIC -fvisibility=hidden -Wimplicit-fallthrough $(CFLAGS) -JAVA_WASM_CFLAGS := -g -Oz -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -Wno-missing-braces -fPIC -fvisibility=hidden -Wimplicit-fallthrough $(JAVA_WASM_CFLAGS) +JAVA_WASM_CFLAGS := -g0 -O2 -Oz -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -Wno-missing-braces -fno-sanitize=all -fno-stack-protector -fPIC -fvisibility=hidden -Wimplicit-fallthrough $(JAVA_WASM_CFLAGS) CC ?= cc AR ?= ar ARFLAGS ?= -r$(V0:1=v) @@ -53,6 +53,7 @@ javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS) java-wasm/src/test/resources/prism.wasm: Makefile $(SOURCES) $(HEADERS) $(ECHO) "building $@" + $(Q) $(MAKEDIRS) $(@D) $(Q) $(WASI_SDK_PATH)/bin/clang $(DEBUG_FLAGS) -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(JAVA_WASM_CFLAGS) -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -lc++ -lc++abi -o $@ $(SOURCES) build/shared/%.o: src/%.c Makefile $(HEADERS) diff --git a/Rakefile b/Rakefile index b06877107e..6e79e08b7f 100644 --- a/Rakefile +++ b/Rakefile @@ -55,6 +55,7 @@ end CLOBBER.concat(Prism::Template::TEMPLATES) CLOBBER.concat(["build"]) CLOBBER << "lib/prism/prism.#{RbConfig::CONFIG["DLEXT"]}" +CLOBBER << "java-wasm/src/main/resources/prism.wasm" Prism::Template::TEMPLATES.each do |filepath| desc "Generate #{filepath}" diff --git a/docs/configuration.md b/docs/configuration.md index 04887d6d9c..97e01c8d82 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,9 +7,9 @@ A lot of code in prism's repository is templated from a single configuration fil * `include/prism/diagnostic.h` - for defining the diagnostics * `javascript/src/deserialize.js` - for defining how to deserialize the nodes in JavaScript * `javascript/src/nodes.js` - for defining the nodes in JavaScript -* `java/org/prism/AbstractNodeVisitor.java` - for defining the visitor interface for the nodes in Java -* `java/org/prism/Loader.java` - for defining how to deserialize the nodes in Java -* `java/org/prism/Nodes.java` - for defining the nodes in Java +* `java/org/ruby_lang/prism/AbstractNodeVisitor.java` - for defining the visitor interface for the nodes in Java +* `java/org/ruby_lang/prism/Loader.java` - for defining how to deserialize the nodes in Java +* `java/org/ruby_lang/prism/Nodes.java` - for defining the nodes in Java * `lib/prism/compiler.rb` - for defining the compiler for the nodes in Ruby * `lib/prism/dispatcher.rb` - for defining the dispatch visitors for the nodes in Ruby * `lib/prism/dot_visitor.rb` - for defining the dot visitor for the nodes in Ruby diff --git a/docs/releasing.md b/docs/releasing.md index a7d3cf5eb3..6d0f892926 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -23,9 +23,9 @@ ruby -pi -e 'gsub(/PRISM_VERSION_PATCH \d+/, %Q{PRISM_VERSION_PATCH #{ENV["PRISM ruby -pi -e 'gsub(/PRISM_VERSION ".+?"/, %Q{PRISM_VERSION "#{ENV["PRISM_VERSION"]}"})' include/prism/version.h ruby -pi -e 'gsub(/"version": ".+?"/, %Q{"version": "#{ENV["PRISM_VERSION"]}"})' javascript/package.json ruby -pi -e 'gsub(/lossy\(\), ".+?"/, %Q{lossy(), "#{ENV["PRISM_VERSION"]}"})' rust/ruby-prism-sys/tests/utils_tests.rs -ruby -pi -e 'gsub(/\d+, "prism major/, %Q{#{ENV["PRISM_MAJOR"]}, "prism major})' templates/java/org/prism/Loader.java.erb -ruby -pi -e 'gsub(/\d+, "prism minor/, %Q{#{ENV["PRISM_MINOR"]}, "prism minor})' templates/java/org/prism/Loader.java.erb -ruby -pi -e 'gsub(/\d+, "prism patch/, %Q{#{ENV["PRISM_PATCH"]}, "prism patch})' templates/java/org/prism/Loader.java.erb +ruby -pi -e 'gsub(/\d+, "prism major/, %Q{#{ENV["PRISM_MAJOR"]}, "prism major})' templates/java/org/ruby_lang/prism/Loader.java.erb +ruby -pi -e 'gsub(/\d+, "prism minor/, %Q{#{ENV["PRISM_MINOR"]}, "prism minor})' templates/java/org/ruby_lang/prism/Loader.java.erb +ruby -pi -e 'gsub(/\d+, "prism patch/, %Q{#{ENV["PRISM_PATCH"]}, "prism patch})' templates/java/org/ruby_lang/prism/Loader.java.erb ruby -pi -e 'gsub(/MAJOR_VERSION = \d+/, %Q{MAJOR_VERSION = #{ENV["PRISM_MAJOR"]}})' templates/javascript/src/deserialize.js.erb ruby -pi -e 'gsub(/MINOR_VERSION = \d+/, %Q{MINOR_VERSION = #{ENV["PRISM_MINOR"]}})' templates/javascript/src/deserialize.js.erb ruby -pi -e 'gsub(/PATCH_VERSION = \d+/, %Q{PATCH_VERSION = #{ENV["PRISM_PATCH"]}})' templates/javascript/src/deserialize.js.erb diff --git a/java-wasm/README.md b/java-wasm/README.md new file mode 100644 index 0000000000..2ff93a66d7 --- /dev/null +++ b/java-wasm/README.md @@ -0,0 +1,23 @@ +This dir contains the chicory-prism artifact, a version of prism compiled to WASM and then AOT compiled to JVM bytecode by the Chicory project. + +Generate the templated sources: + +``` +PRISM_EXCLUDE_PRETTYPRINT=1 PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS=1 PRISM_JAVA_BACKEND=jruby bundle exec rake templates +``` + +Compile to WASM using WASI SDK version 25: + +``` +make java-wasm WASI_SDK_PATH=.../wasi-sdk-25.0-arm64-macos +``` + +Build the AOT-compiled machine and wrapper library: + +``` +mvn -f java-wasm/pom.xml clean package +``` + +This should build the chicory-wasm jar file and pass some basic tests. + +The jar will be under `java-wasm/target/chicory-prism-#####-SNAPSHOT.jar` or can be installed by using `install` instead of `pacakge` in the `mvn` command line above. diff --git a/java-wasm/perf-test/.gitignore b/java-wasm/perf-test/.gitignore new file mode 100644 index 0000000000..60286da241 --- /dev/null +++ b/java-wasm/perf-test/.gitignore @@ -0,0 +1 @@ +profile.html diff --git a/java-wasm/perf-test/bench.sh b/java-wasm/perf-test/bench.sh new file mode 100755 index 0000000000..1ad69239b1 --- /dev/null +++ b/java-wasm/perf-test/bench.sh @@ -0,0 +1,6 @@ +#! /bin/bash +set -euxo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +jbang --fresh --deps org.openjdk.jmh:jmh-generator-annprocess:1.36 --javaagent=ap-loader@jvm-profiling-tools/ap-loader=start,event=cpu,file=${SCRIPT_DIR}/profile.html ${SCRIPT_DIR}/test \ No newline at end of file diff --git a/java-wasm/perf-test/test b/java-wasm/perf-test/test new file mode 100755 index 0000000000..f2d5492bc4 --- /dev/null +++ b/java-wasm/perf-test/test @@ -0,0 +1,97 @@ +///usr/bin/env jbang "$0" "$@" ; exit $? + +//DEPS org.jruby:chicory-prism:0.0.1-SNAPSHOT +//DEPS org.jruby:jruby-complete:10.0.2.0 + +import static java.lang.System.*; + +import java.io.DataInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.EnumSet; + +import org.jruby.Ruby; + +import org.jruby.parser.prism.wasm.Prism; +import org.ruby_lang.prism.ParsingOptions; + +public class test { + + final static String[] JRUBY_BOOT_FILES = { + "jruby/java.rb", + "jruby/java/core_ext.rb", + "jruby/java/core_ext/object.rb", + "jruby/java/java_ext.rb", + "jruby/kernel.rb", + "jruby/kernel/signal.rb", + "jruby/kernel/kernel.rb", + "jruby/kernel/proc.rb", + "jruby/kernel/process.rb", + "jruby/kernel/enumerator.rb", + "jruby/kernel/enumerable.rb", + "jruby/kernel/io.rb", + "jruby/kernel/gc.rb", + "jruby/kernel/range.rb", + "jruby/kernel/file.rb", + "jruby/kernel/method.rb", + "jruby/kernel/thread.rb", + "jruby/kernel/integer.rb", + "jruby/kernel/time.rb", + "jruby/preludes.rb", + "jruby/kernel/prelude.rb", + "jruby/kernel/enc_prelude.rb", + "META-INF/jruby.home/lib/ruby/stdlib/unicode_normalize.rb", + "jruby/kernel/gem_prelude.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rbconfig.rb", + "jruby/kernel/rbconfig.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/compatibility.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/defaults.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/deprecate.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/errors.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/target_rbconfig.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/exceptions.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/unknown_command_spell_checker.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/specification.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/basic_specification.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/stub_specification.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/platform.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/specification_record.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/util/list.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/requirement.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/version.rb", + }; + + public static void main(String... args) throws Exception { + out.println("Starting"); + var count = 100; + + var prism = new Prism(); + byte[] src = new byte[1024 * 1024]; + + for (int i = 0; i < count; i++) { + for (var file : JRUBY_BOOT_FILES) { + byte[] options = ParsingOptions.serialize( + file.getBytes(StandardCharsets.UTF_8), + 1, + "UTF-8".getBytes(StandardCharsets.UTF_8), + false, + EnumSet.noneOf(ParsingOptions.CommandLine.class), + ParsingOptions.SyntaxVersion.LATEST, + false, + false, + false, + new byte[][][]{} + ); + + try (InputStream fileIn = Ruby.getClassLoader().getResourceAsStream(file)) { + DataInputStream dis = new DataInputStream(fileIn); + int read = dis.read(src); + prism.serialize(options, src, read); + } + } + } + + out.println("End"); + } +} diff --git a/java-wasm/pom.xml b/java-wasm/pom.xml index f69028bfd8..a4bc82343b 100644 --- a/java-wasm/pom.xml +++ b/java-wasm/pom.xml @@ -2,18 +2,18 @@ 4.0.0 - com.prism - java-prism - 999-SNAPSHOT + org.jruby + chicory-prism + 0.0.1-SNAPSHOT Java Prism - Pure Java Prism interpreting WASM + Pure Java Prism using Chicory WASM runtime UTF-8 UTF-8 - 11 - 11 + 21 + 21 1.6.1 6.0.2 @@ -59,6 +59,11 @@ annotations provided + + org.jruby + jruby-complete + 10.0.2.0 + @@ -81,7 +86,7 @@ maven-compiler-plugin 3.14.1 - 11 + 21 com.dylibso.chicory @@ -125,7 +130,7 @@ compile - org.prism.PrismModule + org.jruby.parser.prism.wasm.PrismParser src/test/resources/prism.wasm diff --git a/java-wasm/src/main/java-templates/org/prism/WasmResource.java b/java-wasm/src/main/java-templates/org/jruby/parser/prism/wasm/WasmResource.java similarity index 82% rename from java-wasm/src/main/java-templates/org/prism/WasmResource.java rename to java-wasm/src/main/java-templates/org/jruby/parser/prism/wasm/WasmResource.java index 48f2671714..2b95406d13 100644 --- a/java-wasm/src/main/java-templates/org/prism/WasmResource.java +++ b/java-wasm/src/main/java-templates/org/jruby/parser/prism/wasm/WasmResource.java @@ -1,4 +1,4 @@ -package org.prism; +package org.jruby.parser.prism.wasm; public final class WasmResource { public static final String absoluteFile = "file://${project.basedir}/src/test/resources/prism.wasm"; diff --git a/java-wasm/src/main/java/org/jruby/parser/prism/wasm/Prism.java b/java-wasm/src/main/java/org/jruby/parser/prism/wasm/Prism.java new file mode 100644 index 0000000000..6dd89c70cb --- /dev/null +++ b/java-wasm/src/main/java/org/jruby/parser/prism/wasm/Prism.java @@ -0,0 +1,95 @@ +package org.jruby.parser.prism.wasm; + +import com.dylibso.chicory.annotations.WasmModuleInterface; +import com.dylibso.chicory.runtime.ByteArrayMemory; +import com.dylibso.chicory.runtime.ImportValues; +import com.dylibso.chicory.runtime.Instance; +import com.dylibso.chicory.wasi.WasiOptions; +import com.dylibso.chicory.wasi.WasiPreview1; +import org.ruby_lang.prism.Loader; +import org.ruby_lang.prism.ParseResult; + +import java.nio.charset.StandardCharsets; + +@WasmModuleInterface(WasmResource.absoluteFile) +public class Prism implements AutoCloseable { + private final WasiPreview1 wasi; + protected final Prism_ModuleExports exports; + private final Instance instance; + + private int bufferPointer; + private int preSourcePointer; + private int preOptionsPointer; + + private final int SOURCE_SIZE = 2 * 1024 * 1024; // 2 MiB + private final int PACKED_OPTIONS_BUFFER_SIZE = 1024; + + public Prism() { + this(WasiOptions.builder().build()); + } + + public Prism(WasiOptions wasiOpts) { + wasi = WasiPreview1.builder().withOptions(wasiOpts).build(); + instance = Instance.builder(PrismParser.load()) + .withMemoryFactory(ByteArrayMemory::new) + .withMachineFactory(PrismParser::create) + .withImportValues(ImportValues.builder().addFunction(wasi.toHostFunctions()).build()) + .build(); + exports = new Prism_ModuleExports(instance); + + preOptionsPointer = exports.calloc(1, PACKED_OPTIONS_BUFFER_SIZE); + preSourcePointer = exports.calloc(1, SOURCE_SIZE); + + bufferPointer = exports.calloc(exports.pmBufferSizeof(), 1); + exports.pmBufferInit(bufferPointer); + } + + public byte[] serialize(byte[] packedOptions, byte[] sourceBytes, int sourceLength) { + int sourcePointer = 0; + boolean useDefaultSourcePointer = sourceLength + 1 > SOURCE_SIZE; + int optionsPointer = 0; + boolean useDefaultOptionsPointer = packedOptions.length > PACKED_OPTIONS_BUFFER_SIZE; + byte[] result; + try { + sourcePointer = (!useDefaultSourcePointer) ? + exports.calloc(1, sourceLength + 1) : preSourcePointer; + instance.memory().write(sourcePointer, sourceBytes, 0, sourceLength); + instance.memory().writeByte(sourcePointer + sourceLength, (byte) 0); + + optionsPointer = (!useDefaultOptionsPointer) ? + exports.calloc(1, packedOptions.length) : preOptionsPointer; + instance.memory().write(optionsPointer, packedOptions); + + exports.pmBufferClear(bufferPointer); + + exports.pmSerializeParse( + bufferPointer, sourcePointer, sourceLength, optionsPointer); + + result = instance.memory().readBytes( + exports.pmBufferValue(bufferPointer), + exports.pmBufferLength(bufferPointer)); + } finally { + if (!useDefaultSourcePointer) { + exports.free(sourcePointer); + } + if (!useDefaultOptionsPointer) { + exports.free(optionsPointer); + } + } + + return result; + } + + public ParseResult serializeParse(byte[] packedOptions, String source) { + var sourceBytes = source.getBytes(StandardCharsets.ISO_8859_1); + byte[] result = serialize(packedOptions, sourceBytes, sourceBytes.length); + return Loader.load(result, sourceBytes); + } + + @Override + public void close() { + if (wasi != null) { + wasi.close(); + } + } +} diff --git a/java-wasm/src/main/java/org/prism/Prism.java b/java-wasm/src/main/java/org/prism/Prism.java deleted file mode 100644 index 9578a441a1..0000000000 --- a/java-wasm/src/main/java/org/prism/Prism.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.prism; - -import com.dylibso.chicory.runtime.ByteArrayMemory; -import com.dylibso.chicory.annotations.WasmModuleInterface; -import com.dylibso.chicory.runtime.ImportValues; -import com.dylibso.chicory.runtime.Instance; -import com.dylibso.chicory.wasi.WasiOptions; -import com.dylibso.chicory.wasi.WasiPreview1; -import com.dylibso.chicory.wasm.types.MemoryLimits; - -import java.nio.charset.StandardCharsets; - -@WasmModuleInterface(WasmResource.absoluteFile) -public class Prism implements AutoCloseable { - private final WasiPreview1 wasi; - private final Instance instance; - private final Prism_ModuleExports exports; - - public Prism() { - this(WasiOptions.builder().build()); - } - public Prism(WasiOptions wasiOpts) { - wasi = WasiPreview1.builder().withOptions(wasiOpts).build(); - instance = Instance.builder(PrismModule.load()) - .withMemoryFactory(limits -> new ByteArrayMemory(new MemoryLimits(10, MemoryLimits.MAX_PAGES))) - .withMachineFactory(PrismModule::create) - .withImportValues(ImportValues.builder().addFunction(wasi.toHostFunctions()).build()) - .build(); - exports = new Prism_ModuleExports(instance); - } - - public Prism_ModuleExports exports() { - return exports; - } - - public byte[] serialize(byte[] packedOptions, byte[] source, int sourceLength) { - int sourcePointer = 0; - int optionsPointer = 0; - int bufferPointer = 0; - int resultPointer = 0; - byte[] result; - try { - sourcePointer = exports.calloc(1, sourceLength); - exports.memory().write(sourcePointer, source); - - optionsPointer = exports.calloc(1, packedOptions.length); - exports.memory().write(optionsPointer, packedOptions); - - bufferPointer = exports.calloc(exports.pmBufferSizeof(), 1); - exports.pmBufferInit(bufferPointer); - - exports.pmSerializeParse(bufferPointer, sourcePointer, sourceLength, optionsPointer); - - resultPointer = exports.pmBufferValue(bufferPointer); - - result = instance.memory().readBytes( - resultPointer, - exports.pmBufferLength(bufferPointer)); - } finally { - if (sourcePointer != 0) { - exports.free(sourcePointer); - } - if (optionsPointer != 0) { - exports.free(optionsPointer); - } - if (bufferPointer != 0) { - exports.free(bufferPointer); - } - if (resultPointer != 0) { - exports.free(resultPointer); - } - } - - return result; - } - - public ParseResult serializeParse(byte[] packedOptions, String source) { - var sourceBytes = source.getBytes(StandardCharsets.US_ASCII); - - byte[] result = serialize(packedOptions, sourceBytes, sourceBytes.length); - - return Loader.load(result, sourceBytes); - } - - @Override - public void close() { - if (wasi != null) { - wasi.close(); - } - } -} diff --git a/java-wasm/src/test/java/org/jruby/parser/prism/DummyTest.java b/java-wasm/src/test/java/org/jruby/parser/prism/DummyTest.java new file mode 100644 index 0000000000..4e89f6f6e3 --- /dev/null +++ b/java-wasm/src/test/java/org/jruby/parser/prism/DummyTest.java @@ -0,0 +1,91 @@ +package org.jruby.parser.prism; + +import org.jruby.parser.prism.wasm.Prism; +import org.junit.jupiter.api.Test; +import org.ruby_lang.prism.ParseResult; +import org.ruby_lang.prism.ParsingOptions; + +import java.util.EnumSet; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DummyTest { + + private static final byte[] packedOptions = ParsingOptions.serialize( + new byte[] {}, + 1, + new byte[] {}, + false, + EnumSet.noneOf(ParsingOptions.CommandLine.class), + ParsingOptions.SyntaxVersion.LATEST, + false, + false, + false, + new byte[][][] {} + ); + + @Test + public void test1() { + // The Ruby source code to be processed + var source = "1 + 1"; + + ParseResult pr = null; + try (Prism prism = new Prism()) { + pr = prism.serializeParse(packedOptions, source); + } + + assertEquals(1, pr.value.childNodes().length); + System.out.println("Nodes:"); + System.out.println(pr.value.childNodes()[0]); + assertTrue(pr.value.childNodes()[0].toString().contains("IntegerNode")); + } + + @Test + public void test1Aot() { + // The Ruby source code to be processed + var source = "1 + 1"; + + ParseResult pr = null; + try (Prism prism = new Prism()) { + pr = prism.serializeParse(packedOptions, source); + } + + assertEquals(1, pr.value.childNodes().length); + System.out.println("Nodes:"); + System.out.println(pr.value.childNodes()[0]); + assertTrue(pr.value.childNodes()[0].toString().contains("IntegerNode")); + } + + @Test + public void test2() { + // The Ruby source code to be processed + var source = "puts \"h\ne\nl\nl\no\n\""; + + ParseResult pr = null; + try (Prism prism = new Prism()) { + pr = prism.serializeParse(packedOptions, source); + } + + assertEquals(1, pr.value.childNodes().length); + System.out.println("Nodes:"); + System.out.println(pr.value.childNodes()[0]); + assertTrue(pr.value.childNodes()[0].toString().contains("CallNode")); + } + + @Test + public void test2Aot() { + // The Ruby source code to be processed + var source = "puts \"h\ne\nl\nl\no\n\""; + + ParseResult pr = null; + try (Prism prism = new Prism()) { + pr = prism.serializeParse(packedOptions, source); + } + + assertEquals(1, pr.value.childNodes().length); + System.out.println("Nodes:"); + System.out.println(pr.value.childNodes()[0]); + assertTrue(pr.value.childNodes()[0].toString().contains("CallNode")); + } +} diff --git a/java-wasm/src/test/java/org/jruby/parser/prism/JRubyTest.java b/java-wasm/src/test/java/org/jruby/parser/prism/JRubyTest.java new file mode 100644 index 0000000000..b8249f4c70 --- /dev/null +++ b/java-wasm/src/test/java/org/jruby/parser/prism/JRubyTest.java @@ -0,0 +1,115 @@ +package org.jruby.parser.prism; + +import org.jruby.Ruby; +import org.jruby.parser.prism.wasm.Prism; +import org.junit.jupiter.api.Test; +import org.ruby_lang.prism.ParsingOptions; + +import java.io.DataInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.EnumSet; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class JRubyTest { + final static String[] JRUBY_BOOT_FILES = { + "jruby/java.rb", + "jruby/java/core_ext.rb", + "jruby/java/core_ext/object.rb", + "jruby/java/java_ext.rb", + "jruby/kernel.rb", + "jruby/kernel/signal.rb", + "jruby/kernel/kernel.rb", + "jruby/kernel/proc.rb", + "jruby/kernel/process.rb", + "jruby/kernel/enumerator.rb", + "jruby/kernel/enumerable.rb", + "jruby/kernel/io.rb", + "jruby/kernel/gc.rb", + "jruby/kernel/range.rb", + "jruby/kernel/file.rb", + "jruby/kernel/method.rb", + "jruby/kernel/thread.rb", + "jruby/kernel/integer.rb", + "jruby/kernel/time.rb", + "jruby/preludes.rb", + "jruby/kernel/prelude.rb", + "jruby/kernel/enc_prelude.rb", + "META-INF/jruby.home/lib/ruby/stdlib/unicode_normalize.rb", + "jruby/kernel/gem_prelude.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rbconfig.rb", + "jruby/kernel/rbconfig.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/compatibility.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/defaults.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/deprecate.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/errors.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/target_rbconfig.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/exceptions.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/unknown_command_spell_checker.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/specification.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/basic_specification.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/stub_specification.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/platform.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/specification_record.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/util/list.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/requirement.rb", + "META-INF/jruby.home/lib/ruby/stdlib/rubygems/version.rb", + }; + + private class TestingPrism extends Prism { + TestingPrism() { + super(); + } + + public int memPages() { + return exports.memory().pages(); + } + } + + @Test + public void jrubyReproducer() throws Exception { + var prism = new TestingPrism(); + + for (int i = 0; i < 3; i++) { + basicJRubyTest(prism); + } + var memoryPagesBefore = prism.memPages(); + for (int i = 0; i < 100; i++) { + var before = System.currentTimeMillis(); + + basicJRubyTest(prism); + + var after = System.currentTimeMillis(); + System.out.println("Elapsed: " + (after - before)); + } + var memoryPagesAfter = prism.memPages(); + assertEquals(memoryPagesBefore, memoryPagesAfter); + } + + private static void basicJRubyTest(Prism prism) throws Exception { + byte[] src = new byte[1024 * 1024]; + + for (var file : JRUBY_BOOT_FILES) { + byte[] options = ParsingOptions.serialize( + file.getBytes(StandardCharsets.UTF_8), + 1, + "UTF-8".getBytes(StandardCharsets.UTF_8), + false, + EnumSet.noneOf(ParsingOptions.CommandLine.class), + ParsingOptions.SyntaxVersion.LATEST, + false, + false, + false, + new byte[][][]{} + ); + + try (InputStream fileIn = Ruby.getClassLoader().getResourceAsStream(file)) { + DataInputStream dis = new DataInputStream(fileIn); + int read = dis.read(src); + prism.serialize(options, src, read); + } + } + } +} diff --git a/java-wasm/src/test/java/org/prism/DummyTest.java b/java-wasm/src/test/java/org/prism/DummyTest.java deleted file mode 100644 index 9e49f1afdd..0000000000 --- a/java-wasm/src/test/java/org/prism/DummyTest.java +++ /dev/null @@ -1,154 +0,0 @@ -package org.prism; - -import com.dylibso.chicory.runtime.ImportValues; -import com.dylibso.chicory.runtime.Instance; -import com.dylibso.chicory.wasi.WasiOptions; -import com.dylibso.chicory.wasi.WasiPreview1; -import com.dylibso.chicory.wasm.Parser; -import com.dylibso.chicory.wasm.types.Value; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.EnumSet; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class DummyTest { - - private static final byte[] packedOptions = ParsingOptions.serialize( - new byte[] {}, - 1, - new byte[] {}, - false, - EnumSet.noneOf(ParsingOptions.CommandLine.class), - ParsingOptions.SyntaxVersion.LATEST, - false, - false, - false, - new byte[][][] {} - ); - - @Test - public void test1() { - WasiOptions wasiOpts = WasiOptions.builder().build(); - WasiPreview1 wasi = WasiPreview1.builder().withOptions(wasiOpts).build(); - var wasmPrism = Instance.builder(Parser.parse(DummyTest.class.getResourceAsStream("/prism.wasm"))) - .withImportValues(ImportValues.builder().addFunction(wasi.toHostFunctions()).build()) - .build(); - var memory = wasmPrism.memory(); - var calloc = wasmPrism.export("calloc"); - var pmSerializeParse = wasmPrism.export("pm_serialize_parse"); - var pmBufferInit = wasmPrism.export("pm_buffer_init"); - var pmBufferSizeof = wasmPrism.export("pm_buffer_sizeof"); - var pmBufferValue = wasmPrism.export("pm_buffer_value"); - var pmBufferLength = wasmPrism.export("pm_buffer_length"); - - // The Ruby source code to be processed - var source = "1 + 1"; - var sourceBytes = source.getBytes(StandardCharsets.US_ASCII); - - var sourcePointer = calloc.apply(1, source.length()); - memory.writeString((int) sourcePointer[0], source); - - var optionsPointer = calloc.apply(1, packedOptions.length); - memory.write((int) optionsPointer[0], packedOptions); - - var bufferPointer = calloc.apply(pmBufferSizeof.apply()[0], 1); - pmBufferInit.apply(bufferPointer); - - pmSerializeParse.apply( - bufferPointer[0], sourcePointer[0], source.length(), optionsPointer[0]); - - var result = memory.readBytes( - (int) pmBufferValue.apply(bufferPointer[0])[0], - (int) pmBufferLength.apply(bufferPointer[0])[0]); - - System.out.println("RESULT: " + new String(result)); - - ParseResult pr = Loader.load(result, sourceBytes); - - assertEquals(1, pr.value.childNodes().length); - System.out.println("Nodes:"); - System.out.println(pr.value.childNodes()[0]); - assertTrue(pr.value.childNodes()[0].toString().contains("IntegerNode")); - } - - @Test - public void test1Aot() { - // The Ruby source code to be processed - var source = "1 + 1"; - - ParseResult pr = null; - try (Prism prism = new Prism()) { - pr = prism.serializeParse(packedOptions, source); - } - - assertEquals(1, pr.value.childNodes().length); - System.out.println("Nodes:"); - System.out.println(pr.value.childNodes()[0]); - assertTrue(pr.value.childNodes()[0].toString().contains("IntegerNode")); - } - - @Test - public void test2() { - WasiOptions wasiOpts = WasiOptions.builder().build(); - WasiPreview1 wasi = WasiPreview1.builder().withOptions(wasiOpts).build(); - var wasmPrism = Instance.builder(Parser.parse(DummyTest.class.getResourceAsStream("/prism.wasm"))) - .withImportValues(ImportValues.builder().addFunction(wasi.toHostFunctions()).build()) - .build(); - var memory = wasmPrism.memory(); - var calloc = wasmPrism.export("calloc"); - var pmSerializeParse = wasmPrism.export("pm_serialize_parse"); - var pmBufferInit = wasmPrism.export("pm_buffer_init"); - var pmBufferSizeof = wasmPrism.export("pm_buffer_sizeof"); - var pmBufferValue = wasmPrism.export("pm_buffer_value"); - var pmBufferLength = wasmPrism.export("pm_buffer_length"); - - // The Ruby source code to be processed - var source = "puts \"h\ne\nl\nl\no\n\""; - var sourceBytes = source.getBytes(StandardCharsets.US_ASCII); - - var sourcePointer = calloc.apply(1, source.length()); - memory.writeString((int) sourcePointer[0], source); - - var optionsPointer = calloc.apply(1, packedOptions.length); - memory.write((int) optionsPointer[0], packedOptions); - - var bufferPointer = calloc.apply(pmBufferSizeof.apply()[0], 1); - pmBufferInit.apply(bufferPointer); - - pmSerializeParse.apply( - bufferPointer[0], sourcePointer[0], source.length(), optionsPointer[0]); - - var result = memory.readBytes( - (int) pmBufferValue.apply(bufferPointer[0])[0], - (int) pmBufferLength.apply(bufferPointer[0])[0]); - - System.out.println("RESULT: " + new String(result)); - - ParseResult pr = Loader.load(result, sourceBytes); - - assertEquals(1, pr.value.childNodes().length); - - System.out.println("Nodes:"); - System.out.println(pr.value.childNodes()[0]); - assertTrue(pr.value.childNodes()[0].toString().contains("CallNode")); - } - - @Test - public void test2Aot() { - // The Ruby source code to be processed - var source = "puts \"h\ne\nl\nl\no\n\""; - - ParseResult pr = null; - try (Prism prism = new Prism()) { - pr = prism.serializeParse(packedOptions, source); - } - - assertEquals(1, pr.value.childNodes().length); - System.out.println("Nodes:"); - System.out.println(pr.value.childNodes()[0]); - assertTrue(pr.value.childNodes()[0].toString().contains("CallNode")); - } -} diff --git a/java/org/prism/MarkNewlinesVisitor.java b/java/org/ruby_lang/prism/MarkNewlinesVisitor.java similarity index 98% rename from java/org/prism/MarkNewlinesVisitor.java rename to java/org/ruby_lang/prism/MarkNewlinesVisitor.java index 27ab8643e7..8d08424942 100644 --- a/java/org/prism/MarkNewlinesVisitor.java +++ b/java/org/ruby_lang/prism/MarkNewlinesVisitor.java @@ -1,4 +1,4 @@ -package org.prism; +package org.ruby_lang.prism; // Keep in sync with Ruby MarkNewlinesVisitor final class MarkNewlinesVisitor extends AbstractNodeVisitor { diff --git a/java/org/prism/ParseResult.java b/java/org/ruby_lang/prism/ParseResult.java similarity index 98% rename from java/org/prism/ParseResult.java rename to java/org/ruby_lang/prism/ParseResult.java index 144ea16e36..aad05a892d 100644 --- a/java/org/prism/ParseResult.java +++ b/java/org/ruby_lang/prism/ParseResult.java @@ -1,4 +1,4 @@ -package org.prism; +package org.ruby_lang.prism; // @formatter:off public final class ParseResult { diff --git a/java/org/prism/Parser.java b/java/org/ruby_lang/prism/Parser.java similarity index 92% rename from java/org/prism/Parser.java rename to java/org/ruby_lang/prism/Parser.java index 717c3e5179..5c60977d80 100644 --- a/java/org/prism/Parser.java +++ b/java/org/ruby_lang/prism/Parser.java @@ -1,4 +1,4 @@ -package org.prism; +package org.ruby_lang.prism; public abstract class Parser { diff --git a/java/org/prism/ParsingOptions.java b/java/org/ruby_lang/prism/ParsingOptions.java similarity index 99% rename from java/org/prism/ParsingOptions.java rename to java/org/ruby_lang/prism/ParsingOptions.java index be0a8a7dba..8990f72104 100644 --- a/java/org/prism/ParsingOptions.java +++ b/java/org/ruby_lang/prism/ParsingOptions.java @@ -1,4 +1,4 @@ -package org.prism; +package org.ruby_lang.prism; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; diff --git a/rakelib/serialization.rake b/rakelib/serialization.rake index a57a7ba700..499be443ce 100644 --- a/rakelib/serialization.rake +++ b/rakelib/serialization.rake @@ -26,7 +26,7 @@ task "test:java_loader:internal" => :compile do puts path serialized = Prism.dump_file(path) source_bytes = File.binread(path).unpack('c*') - parse_result = org.prism.Loader.load(serialized.unpack('c*'), source_bytes) + parse_result = org.ruby_lang.prism.Loader.load(serialized.unpack('c*'), source_bytes) puts parse_result.value end end diff --git a/templates/java/org/prism/AbstractNodeVisitor.java.erb b/templates/java/org/ruby_lang/prism/AbstractNodeVisitor.java.erb similarity index 94% rename from templates/java/org/prism/AbstractNodeVisitor.java.erb rename to templates/java/org/ruby_lang/prism/AbstractNodeVisitor.java.erb index a66daedf63..58eabebae8 100644 --- a/templates/java/org/prism/AbstractNodeVisitor.java.erb +++ b/templates/java/org/ruby_lang/prism/AbstractNodeVisitor.java.erb @@ -1,4 +1,4 @@ -package org.prism; +package org.ruby_lang.prism; // GENERATED BY <%= File.basename(__FILE__) %> // @formatter:off diff --git a/templates/java/org/prism/Loader.java.erb b/templates/java/org/ruby_lang/prism/Loader.java.erb similarity index 99% rename from templates/java/org/prism/Loader.java.erb rename to templates/java/org/ruby_lang/prism/Loader.java.erb index aa8e1e30c9..af47aeed73 100644 --- a/templates/java/org/prism/Loader.java.erb +++ b/templates/java/org/ruby_lang/prism/Loader.java.erb @@ -1,7 +1,7 @@ <%- string_type = Prism::Template::JAVA_STRING_TYPE -%> -package org.prism; +package org.ruby_lang.prism; -import org.prism.Nodes; +import org.ruby_lang.prism.Nodes; import java.lang.Short; import java.math.BigInteger; diff --git a/templates/java/org/prism/Nodes.java.erb b/templates/java/org/ruby_lang/prism/Nodes.java.erb similarity index 99% rename from templates/java/org/prism/Nodes.java.erb rename to templates/java/org/ruby_lang/prism/Nodes.java.erb index 8b0a9ce20d..47b2ea0a7e 100644 --- a/templates/java/org/prism/Nodes.java.erb +++ b/templates/java/org/ruby_lang/prism/Nodes.java.erb @@ -1,5 +1,5 @@ <%- string_type = Prism::Template::JAVA_STRING_TYPE -%> -package org.prism; +package org.ruby_lang.prism; import java.lang.Override; import java.lang.String; diff --git a/templates/template.rb b/templates/template.rb index aca626b5eb..d0287464f6 100755 --- a/templates/template.rb +++ b/templates/template.rb @@ -656,9 +656,9 @@ def locals "javascript/src/deserialize.js", "javascript/src/nodes.js", "javascript/src/visitor.js", - "java/org/prism/Loader.java", - "java/org/prism/Nodes.java", - "java/org/prism/AbstractNodeVisitor.java", + "java/org/ruby_lang/prism/Loader.java", + "java/org/ruby_lang/prism/Nodes.java", + "java/org/ruby_lang/prism/AbstractNodeVisitor.java", "lib/prism/compiler.rb", "lib/prism/dispatcher.rb", "lib/prism/dot_visitor.rb",