diff --git a/pom.xml b/pom.xml index c9b7c93..7a41fc0 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.jruby jruby-prism jar - 1.5.0 + 2.0.0-SNAPSHOT jruby-prism Java portion of JRuby Prism parser support. @@ -66,12 +66,12 @@ org.jruby jruby-base - 10.0.0.0-SNAPSHOT + 10.0.2.0 - com.prism - java-prism - 999-SNAPSHOT + org.jruby + chicory-prism + 0.0.1-SNAPSHOT @@ -116,45 +116,6 @@ - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - sign-artifacts - verify - - sign - - - - - - maven-source-plugin - - - attach-sources - - jar-no-fork - - - - - - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - - none - - @@ -210,5 +171,56 @@ + + release + + + + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + --pinentry-mode + loopback + + + + + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + none + + + + + diff --git a/src/main/java/org/jruby/prism/ParserProviderPrism.java b/src/main/java/org/jruby/prism/ParserProviderPrism.java index e81cd3d..07beb81 100644 --- a/src/main/java/org/jruby/prism/ParserProviderPrism.java +++ b/src/main/java/org/jruby/prism/ParserProviderPrism.java @@ -1,31 +1,43 @@ package org.jruby.prism; +import jnr.ffi.LibraryLoader; import org.jruby.Ruby; import org.jruby.ir.builder.IRBuilderFactory; import org.jruby.parser.Parser; +import org.jruby.parser.ParserManager; import org.jruby.parser.ParserProvider; -import org.jruby.prism.parser.ParserPrism; -import org.jruby.prism.parser.ParserBindingPrism; import org.jruby.prism.builder.IRBuilderFactoryPrism; +import org.jruby.prism.parser.ParserBindingPrism; +import org.jruby.prism.parser.ParserPrismNative; +import org.jruby.prism.parser.ParserPrismWasm; -import jnr.ffi.LibraryLoader; +import java.io.File; public class ParserProviderPrism implements ParserProvider { private static ParserBindingPrism prismLibrary; public void initialize(String path) { - if (prismLibrary != null) { - System.out.println("Prism already initialized"); - return; + if (new File(path).exists()) { + if (prismLibrary != null) { + System.out.println("Prism already initialized"); + return; + } + prismLibrary = LibraryLoader.create(ParserBindingPrism.class).load(path); + // We do something extra here as a side-effect which is how we get an UnsatisfiedLinkError + // If the library didn't in fact find the .so or has other loading problems. + ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary)); + } else { + prismLibrary = null; } - prismLibrary = LibraryLoader.create(ParserBindingPrism.class).load(path); - // We do something extra here as a side-effect which is how we get an UnsatisfiedLinkError - // If the library didn't in fact find the .so or has other loading problems. - ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary)); } public Parser getParser(Ruby runtime) { - return new ParserPrism(runtime, prismLibrary); + if (ParserManager.PARSER_WASM || prismLibrary == null) { + // uninitialized dynamic lib or wasm requested + return new ParserPrismWasm(runtime); + } + + return new ParserPrismNative(runtime, prismLibrary); } public IRBuilderFactory getBuilderFactory() { diff --git a/src/main/java/org/jruby/prism/parser/ParserPrism.java b/src/main/java/org/jruby/prism/parser/ParserPrismBase.java similarity index 83% rename from src/main/java/org/jruby/prism/parser/ParserPrism.java rename to src/main/java/org/jruby/prism/parser/ParserPrismBase.java index f11dee3..17ae433 100644 --- a/src/main/java/org/jruby/prism/parser/ParserPrism.java +++ b/src/main/java/org/jruby/prism/parser/ParserPrismBase.java @@ -11,7 +11,6 @@ import org.jruby.ext.coverage.CoverageData; import org.jruby.management.ParserStats; import org.jruby.parser.Parser; -import org.jruby.parser.ParserManager; import org.jruby.parser.ParserType; import org.jruby.parser.StaticScope; import org.jruby.runtime.DynamicScope; @@ -21,10 +20,16 @@ import org.jruby.util.ByteList; import org.jruby.util.CommonByteLists; import org.jruby.util.io.ChannelHelper; -import org.prism.Nodes; -import org.prism.Nodes.*; +import org.prism.Nodes.ArgumentsNode; +import org.prism.Nodes.CallNode; +import org.prism.Nodes.CallNodeFlags; +import org.prism.Nodes.GlobalVariableReadNode; +import org.prism.Nodes.GlobalVariableWriteNode; +import org.prism.Nodes.Node; +import org.prism.Nodes.ProgramNode; +import org.prism.Nodes.StatementsNode; +import org.prism.Nodes.WhileNode; import org.prism.ParsingOptions; -import org.prism.Prism; import java.io.ByteArrayInputStream; import java.io.DataInputStream; @@ -39,16 +44,15 @@ import static org.jruby.parser.ParserType.EVAL; import static org.jruby.parser.ParserType.MAIN; -public class ParserPrism extends Parser { - private boolean parserTiming = org.jruby.util.cli.Options.PARSER_SUMMARY.load(); +public abstract class ParserPrismBase extends Parser { + protected boolean parserTiming = org.jruby.util.cli.Options.PARSER_SUMMARY.load(); - private final ParserBindingPrism prismLibrary; - - public ParserPrism(Ruby runtime, ParserBindingPrism prismLibrary) { + public ParserPrismBase(Ruby runtime) { super(runtime); - this.prismLibrary = prismLibrary; } + protected abstract byte[] parse(byte[] source, int sourceLength, byte[] metadata); + @Override public ParseResult parse(String fileName, int lineNumber, ByteList content, DynamicScope existingScope, ParserType type) { int sourceLength = content.realSize(); @@ -113,10 +117,10 @@ private ParseResult parseInternal(String fileName, DynamicScope blockScope, byte coverageMode = runtime.getCoverageData().getMode(); } - ParseResultPrism result = new ParseResultPrism(fileName, source, (Nodes.ProgramNode) res.value, res.source, encoding, coverageMode); + ParseResultPrism result = new ParseResultPrism(fileName, source, (ProgramNode) res.value, res.source, encoding, coverageMode); if (blockScope != null) { if (type == MAIN) { // update TOPLEVEL_BINDNG - RubySymbol[] locals = ((Nodes.ProgramNode) result.getAST()).locals; + RubySymbol[] locals = ((ProgramNode) result.getAST()).locals; for (int i = 0; i < locals.length; i++) { blockScope.getStaticScope().addVariableThisScope(locals[i].idString()); } @@ -174,36 +178,6 @@ private byte[] loadFully(String fileName, InputStream in) { } } - - private byte[] parse(byte[] source, int sourceLength, byte[] metadata) { - if (ParserManager.PARSER_WASM) return parseChicory(source, sourceLength, metadata); - - long time = 0; - if (parserTiming) time = System.nanoTime(); - - ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary)); - prismLibrary.pm_buffer_init(buffer); - prismLibrary.pm_serialize_parse(buffer, source, sourceLength, metadata); - if (parserTiming) { - ParserStats stats = runtime.getParserManager().getParserStats(); - - stats.addPrismTimeCParseSerialize(System.nanoTime() - time); - } - - int length = buffer.length.intValue(); - byte[] src = new byte[length]; - buffer.value.get().get(0, src, 0, length); - - return src; - } - - - private byte[] parseChicory(byte[] source, int sourceLength, byte[] metadata) { - try (Prism prism = new Prism()) { - return prism.serialize(metadata, source, sourceLength); - } - } - // lineNumber (0-indexed) private byte[] generateMetadata(String fileName, int lineNumber, Encoding encoding, DynamicScope scope, ParserType type) { ByteList metadata = new ByteList(); @@ -238,7 +212,7 @@ private byte[] generateMetadata(String fileName, int lineNumber, Encoding encodi metadata.append(flags); // version - metadata.append(ParsingOptions.SyntaxVersion.V3_4.getValue()); + metadata.append(ParsingOptions.SyntaxVersion.V4_0.getValue()); // Do not lock encoding metadata.append(0); @@ -276,12 +250,17 @@ private void appendUnsignedInt(ByteList buf, int value) { buf.append(value >>> 24); } - private byte[] encodeEvalScopes(ByteList buf, StaticScope scope) { + private void encodeEvalScopes(ByteList buf, StaticScope scope) { int startIndex = buf.realSize(); + + // append uint 0 to reserve the space appendUnsignedInt(buf, 0); + + // write the scopes to the buffer int count = encodeEvalScopesInner(buf, scope, 1); + + // overwrite int 0 with scope count writeUnsignedInt(buf, startIndex, count); - return buf.bytes(); } private int encodeEvalScopesInner(ByteList buf, StaticScope scope, int count) { @@ -291,8 +270,13 @@ private int encodeEvalScopesInner(ByteList buf, StaticScope scope, int count) { // once more for method scope String names[] = scope.getVariables(); + + // number of variables appendUnsignedInt(buf, names.length); + + // forwarding flags buf.append(0); + for (String name : names) { // Get the bytes "raw" (which we use ISO8859_1 for) as this is how we record these in StaticScope. byte[] bytes = name.getBytes(ISO8859_1Encoding.INSTANCE.getCharset()); @@ -327,23 +311,23 @@ public IRubyObject getLineStub(ThreadContext context, ParseResult arg, int lineC @Override public ParseResult addGetsLoop(Ruby runtime, ParseResult result, boolean printing, boolean processLineEndings, boolean split) { var context = runtime.getCurrentContext(); - List newBody = new ArrayList<>(); + List newBody = new ArrayList<>(); if (processLineEndings) { - newBody.add(new Nodes.GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_BACKSLASH), + newBody.add(new GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_BACKSLASH), new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_SLASH)))); } - Nodes.GlobalVariableReadNode dollarUnderscore = new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, DOLLAR_UNDERSCORE)); + GlobalVariableReadNode dollarUnderscore = new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, DOLLAR_UNDERSCORE)); - List whileBody = new ArrayList<>(); + List whileBody = new ArrayList<>(); if (processLineEndings) { whileBody.add(new CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "chomp!"), null, null)); } if (split) { whileBody.add(new GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, "$F"), - new Nodes.CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "split"), null, null))); + new CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "split"), null, null))); } StatementsNode stmts = ((ProgramNode) result.getAST()).statements; @@ -362,7 +346,7 @@ public ParseResult addGetsLoop(Ruby runtime, ParseResult result, boolean printin nodes = new Node[newBody.size()]; newBody.toArray(nodes); - Nodes.ProgramNode newRoot = new Nodes.ProgramNode(-1, 0, 0, new RubySymbol[] {}, new StatementsNode(-1, 0, 0, nodes)); + ProgramNode newRoot = new ProgramNode(-1, 0, 0, new RubySymbol[] {}, new StatementsNode(-1, 0, 0, nodes)); ((ParseResultPrism) result).setRoot(newRoot); diff --git a/src/main/java/org/jruby/prism/parser/ParserPrismNative.java b/src/main/java/org/jruby/prism/parser/ParserPrismNative.java new file mode 100644 index 0000000..57bfa6c --- /dev/null +++ b/src/main/java/org/jruby/prism/parser/ParserPrismNative.java @@ -0,0 +1,33 @@ +package org.jruby.prism.parser; + +import org.jruby.Ruby; +import org.jruby.management.ParserStats; + +public class ParserPrismNative extends ParserPrismBase { + private final ParserBindingPrism prismLibrary; + + public ParserPrismNative(Ruby runtime, ParserBindingPrism prismLibrary) { + super(runtime); + this.prismLibrary = prismLibrary; + } + + protected byte[] parse(byte[] source, int sourceLength, byte[] metadata) { + long time = 0; + if (parserTiming) time = System.nanoTime(); + + ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary)); + prismLibrary.pm_buffer_init(buffer); + prismLibrary.pm_serialize_parse(buffer, source, sourceLength, metadata); + if (parserTiming) { + ParserStats stats = runtime.getParserManager().getParserStats(); + + stats.addPrismTimeCParseSerialize(System.nanoTime() - time); + } + + int length = buffer.length.intValue(); + byte[] src = new byte[length]; + buffer.value.get().get(0, src, 0, length); + + return src; + } +} diff --git a/src/main/java/org/jruby/prism/parser/ParserPrismWasm.java b/src/main/java/org/jruby/prism/parser/ParserPrismWasm.java new file mode 100644 index 0000000..c44b7e4 --- /dev/null +++ b/src/main/java/org/jruby/prism/parser/ParserPrismWasm.java @@ -0,0 +1,36 @@ +package org.jruby.prism.parser; + +import org.jcodings.Encoding; +import org.jcodings.specific.ISO8859_1Encoding; +import org.jruby.*; +import org.jruby.parser.Parser; +import org.jruby.parser.ParserType; +import org.jruby.parser.StaticScope; +import org.jruby.runtime.DynamicScope; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.util.ByteList; +import org.jruby.util.CommonByteLists; +import org.prism.Nodes.*; +import org.prism.ParsingOptions; +import org.prism.Prism; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.jruby.api.Convert.asSymbol; +import static org.jruby.lexer.LexingCommon.DOLLAR_UNDERSCORE; +import static org.jruby.parser.ParserType.EVAL; + +public class ParserPrismWasm extends ParserPrismBase { + private static final Prism prism = new Prism(); + + public ParserPrismWasm(Ruby runtime) { + super(runtime); + } + + protected byte[] parse(byte[] source, int sourceLength, byte[] metadata) { + return prism.serialize(metadata, source, sourceLength); + } +}