From cea1aa5dbe90f451b5961b14865c3927d2740cf6 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 18 Sep 2023 17:04:52 +0200 Subject: [PATCH 1/9] started a number of optimizations around AST building for module import --- .../rascalmpl/semantics/dynamic/Import.java | 21 ++++++++++-- .../values/parsetrees/ProductionAdapter.java | 3 +- .../values/parsetrees/SymbolAdapter.java | 1 + .../values/parsetrees/TreeAdapter.java | 33 +++++++++++++++++++ 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/semantics/dynamic/Import.java b/src/org/rascalmpl/semantics/dynamic/Import.java index a750d319177..2ae08dea575 100644 --- a/src/org/rascalmpl/semantics/dynamic/Import.java +++ b/src/org/rascalmpl/semantics/dynamic/Import.java @@ -538,7 +538,12 @@ public ITree visitTreeAppl(ITree tree) { IListWriter w = vf.listWriter(); IList args = TreeAdapter.getArgs(tree); for (IValue arg : args) { - w.append(arg.accept(this)); + if (TreeAdapter.isContextFree(tree)) { + w.append(arg.accept(this)); + } + else { + w.append(arg); + } } args = w.done(); @@ -686,7 +691,12 @@ public ITree visitTreeAppl(ITree tree) { IListWriter w = vf.listWriter(); IList args = TreeAdapter.getArgs(tree); for (IValue arg : args) { - w.append(arg.accept(this)); + if (!TreeAdapter.isLayout((ITree) arg)) { + w.append(arg.accept(this)); + } + else { + w.append(arg); + } } args = w.done(); @@ -772,7 +782,12 @@ public ITree visitTreeAppl(ITree tree) { IListWriter w = vf.listWriter(); IList args = TreeAdapter.getArgs(tree); for (IValue elem : args) { - w.append(elem.accept(this)); + if (!TreeAdapter.isLayout((ITree) elem)) { + w.append(elem.accept(this)); + } + else { + w.append(elem); + } } args = w.done(); diff --git a/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java b/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java index 591d781a674..faecbcc6073 100644 --- a/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java @@ -84,7 +84,8 @@ public static IList getASTSymbols(IConstructor tree) { } public static boolean isContextFree(IConstructor tree) { - return SymbolAdapter.isSort(getType(tree)); + IConstructor t = getType(tree); + return SymbolAdapter.isSort(t) || SymbolAdapter.isParameterizedSort(t); } public static boolean isLayout(IConstructor tree) { diff --git a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java index 991d6ed459c..5b29b107e73 100644 --- a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java @@ -194,6 +194,7 @@ public static String getLabel(IConstructor tree) { } public static boolean isParameterizedSort(IConstructor tree) { + tree = delabel(tree); return tree.getConstructorType() == Symbol_ParameterizedSort; } diff --git a/src/org/rascalmpl/values/parsetrees/TreeAdapter.java b/src/org/rascalmpl/values/parsetrees/TreeAdapter.java index 0ef3f77a04f..0657722f414 100644 --- a/src/org/rascalmpl/values/parsetrees/TreeAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/TreeAdapter.java @@ -18,7 +18,10 @@ import java.io.Writer; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.stream.IntStream; +import java.util.stream.Stream; import org.fusesource.jansi.Ansi; import org.fusesource.jansi.Ansi.Attribute; @@ -369,6 +372,21 @@ public static IList getListASTArgs(ITree tree) { } return writer.done(); } + + public static Stream streamListASTArgs(ITree tree) { + if (!isList(tree)) { + throw new ImplementationError("This is not a context-free list production: " + tree); + } + + IList children = getArgs(tree); + int jump = isSeparatedList(tree) ? getSeparatorCount(tree) - 1 : 2; + + return IntStream.range(0, children.size()) + .filter(i -> i % jump == 0) + .mapToObj(children::get) + .map(t -> (ITree) t) + ; + } public static int getSeparatorCount(ITree tree) { IConstructor nt = ProductionAdapter.getType(getProduction(tree)); @@ -412,6 +430,21 @@ public static IList getASTArgs(ITree tree) { return writer.done(); } + public static Stream streamASTArgs(ITree tree) { + if (SymbolAdapter.isStartSort(TreeAdapter.getType(tree))) { + return Stream.of((ITree) getArgs(tree).get(1)); + } + + if (isLexical(tree)) { + throw new ImplementationError("This is not a context-free production: " + tree); + } + + return getArgs(tree).stream() + .map(t -> (ITree) t) + .filter(t -> !isLiteral(t) && !isCILiteral(t) && !isLayout(t)) + ; + } + public static boolean isCILiteral(ITree tree) { return isAppl(tree) && ProductionAdapter.isCILiteral(getProduction(tree)); } From eb3a09e5379212367775ffc60d9e841f6a708aba Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 18 Sep 2023 17:10:46 +0200 Subject: [PATCH 2/9] made AST stream faster --- src/org/rascalmpl/values/parsetrees/TreeAdapter.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/values/parsetrees/TreeAdapter.java b/src/org/rascalmpl/values/parsetrees/TreeAdapter.java index 0657722f414..17935f11764 100644 --- a/src/org/rascalmpl/values/parsetrees/TreeAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/TreeAdapter.java @@ -439,9 +439,13 @@ public static Stream streamASTArgs(ITree tree) { throw new ImplementationError("This is not a context-free production: " + tree); } - return getArgs(tree).stream() + IList children = getArgs(tree); + + return IntStream.range(0, children.size()) + .filter(i -> i % 2 == 0) + .mapToObj(children::get) .map(t -> (ITree) t) - .filter(t -> !isLiteral(t) && !isCILiteral(t) && !isLayout(t)) + .filter(t -> !isLiteral(t) && !isCILiteral(t)) ; } From 18373eb0281fda1f2cdf4198923546fbbe1a80c9 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 18 Sep 2023 19:58:55 +0200 Subject: [PATCH 3/9] fixes --- src/org/rascalmpl/parser/ASTBuilder.java | 18 ++++++-- .../rascalmpl/semantics/dynamic/Import.java | 12 +++--- .../values/parsetrees/TreeAdapter.java | 43 +++---------------- 3 files changed, 25 insertions(+), 48 deletions(-) diff --git a/src/org/rascalmpl/parser/ASTBuilder.java b/src/org/rascalmpl/parser/ASTBuilder.java index 398c8ae0b42..26144b290c8 100644 --- a/src/org/rascalmpl/parser/ASTBuilder.java +++ b/src/org/rascalmpl/parser/ASTBuilder.java @@ -40,6 +40,7 @@ import io.usethesource.vallang.IValue; import io.usethesource.vallang.exceptions.FactTypeUseException; import org.rascalmpl.values.ValueFactoryFactory; +import org.rascalmpl.values.parsetrees.ITree; import org.rascalmpl.values.parsetrees.ProductionAdapter; import org.rascalmpl.values.parsetrees.SymbolAdapter; import org.rascalmpl.values.parsetrees.TreeAdapter; @@ -179,11 +180,20 @@ public AbstractAST buildValue(IValue arg) { } private List buildList(org.rascalmpl.values.parsetrees.ITree in) { - IList args = TreeAdapter.getListASTArgs(in); - List result = new ArrayList(args.length()); - for (IValue arg: args) { + List result = new ArrayList(TreeAdapter.getListLength(in)); + + IList args = in.getArgs(); + int seps = TreeAdapter.getSeparatorCount(in); + + for (int i = 0; i < args.length(); i++) { + ITree arg = (ITree) args.get(i); result.add(buildValue(arg)); + i += seps; } + // TreeAdapter.streamListASTArgs(in).forEach(arg -> { + // result.add(buildValue(arg)); + // }); + return result; } @@ -368,7 +378,7 @@ private IList getASTArgs(org.rascalmpl.values.parsetrees.ITree tree) { IList children = TreeAdapter.getArgs(tree); IListWriter writer = ValueFactoryFactory.getValueFactory().listWriter(); - for (int i = 0; i < children.length(); i++) { + for (int i = 0; i < children.length(); i++) { org.rascalmpl.values.parsetrees.ITree kid = (org.rascalmpl.values.parsetrees.ITree) children.get(i); if (!TreeAdapter.isLiteral(kid) && !TreeAdapter.isCILiteral(kid) && !TreeAdapter.isEmpty(kid)) { writer.append(kid); diff --git a/src/org/rascalmpl/semantics/dynamic/Import.java b/src/org/rascalmpl/semantics/dynamic/Import.java index 2ae08dea575..4304581a845 100644 --- a/src/org/rascalmpl/semantics/dynamic/Import.java +++ b/src/org/rascalmpl/semantics/dynamic/Import.java @@ -538,12 +538,12 @@ public ITree visitTreeAppl(ITree tree) { IListWriter w = vf.listWriter(); IList args = TreeAdapter.getArgs(tree); for (IValue arg : args) { - if (TreeAdapter.isContextFree(tree)) { - w.append(arg.accept(this)); - } - else { - w.append(arg); - } + if (!TreeAdapter.isLayout(tree) && !TreeAdapter.isLexical(tree)) { + w.append(arg.accept(this)); + } + else { + w.append(arg); + } } args = w.done(); diff --git a/src/org/rascalmpl/values/parsetrees/TreeAdapter.java b/src/org/rascalmpl/values/parsetrees/TreeAdapter.java index 17935f11764..0ac65a94b94 100644 --- a/src/org/rascalmpl/values/parsetrees/TreeAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/TreeAdapter.java @@ -18,11 +18,7 @@ import java.io.Writer; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.Map; -import java.util.stream.IntStream; -import java.util.stream.Stream; - import org.fusesource.jansi.Ansi; import org.fusesource.jansi.Ansi.Attribute; import org.fusesource.jansi.Ansi.Color; @@ -372,21 +368,6 @@ public static IList getListASTArgs(ITree tree) { } return writer.done(); } - - public static Stream streamListASTArgs(ITree tree) { - if (!isList(tree)) { - throw new ImplementationError("This is not a context-free list production: " + tree); - } - - IList children = getArgs(tree); - int jump = isSeparatedList(tree) ? getSeparatorCount(tree) - 1 : 2; - - return IntStream.range(0, children.size()) - .filter(i -> i % jump == 0) - .mapToObj(children::get) - .map(t -> (ITree) t) - ; - } public static int getSeparatorCount(ITree tree) { IConstructor nt = ProductionAdapter.getType(getProduction(tree)); @@ -430,25 +411,6 @@ public static IList getASTArgs(ITree tree) { return writer.done(); } - public static Stream streamASTArgs(ITree tree) { - if (SymbolAdapter.isStartSort(TreeAdapter.getType(tree))) { - return Stream.of((ITree) getArgs(tree).get(1)); - } - - if (isLexical(tree)) { - throw new ImplementationError("This is not a context-free production: " + tree); - } - - IList children = getArgs(tree); - - return IntStream.range(0, children.size()) - .filter(i -> i % 2 == 0) - .mapToObj(children::get) - .map(t -> (ITree) t) - .filter(t -> !isLiteral(t) && !isCILiteral(t)) - ; - } - public static boolean isCILiteral(ITree tree) { return isAppl(tree) && ProductionAdapter.isCILiteral(getProduction(tree)); } @@ -1074,4 +1036,9 @@ public static IList getNonLayoutArgs(ITree treeSubject) { return w.done(); } + public static int getListLength(ITree in) { + // [1,s1,s2,s3,2,s1,s2,s3,3] 9 / 3 + return (in.getArgs().length() / getSeparatorCount(in) + 1); + } + } From 4784f61074bf708654fa854e3e1319a97f80c4c2 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 19 Sep 2023 11:46:47 +0200 Subject: [PATCH 4/9] experimenting with a hand-written Java version of createHole --- src/org/rascalmpl/parser/ASTBuilder.java | 3 ++- .../values/RascalFunctionValueFactory.java | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/parser/ASTBuilder.java b/src/org/rascalmpl/parser/ASTBuilder.java index 26144b290c8..1a93474622a 100644 --- a/src/org/rascalmpl/parser/ASTBuilder.java +++ b/src/org/rascalmpl/parser/ASTBuilder.java @@ -28,6 +28,7 @@ import org.rascalmpl.ast.Expression; import org.rascalmpl.ast.Module; import org.rascalmpl.ast.Statement; +import org.rascalmpl.ast.Sym; import org.rascalmpl.exceptions.ImplementationError; import org.rascalmpl.interpreter.asserts.Ambiguous; import org.rascalmpl.parser.gtd.util.PointerKeyedHashMap; @@ -109,7 +110,7 @@ public Command buildCommand(org.rascalmpl.values.parsetrees.ITree parseTree) { return buildSort(parseTree, "Command"); } - public Command buildSym(org.rascalmpl.values.parsetrees.ITree parseTree) { + public Sym buildSym(org.rascalmpl.values.parsetrees.ITree parseTree) { return buildSort(parseTree, "Sym"); } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 9a8ce1bfbd8..5832ee3422a 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -24,6 +24,7 @@ import java.util.function.Supplier; import org.rascalmpl.ast.AbstractAST; +import org.rascalmpl.ast.Sym; import org.rascalmpl.exceptions.ImplementationError; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.exceptions.Throw; @@ -39,6 +40,7 @@ import org.rascalmpl.interpreter.result.ResultFactory; import org.rascalmpl.interpreter.staticErrors.UndeclaredNonTerminal; import org.rascalmpl.library.lang.rascal.syntax.RascalParser; +import org.rascalmpl.parser.ASTBuilder; import org.rascalmpl.parser.ParserGenerator; import org.rascalmpl.parser.gtd.IGTD; import org.rascalmpl.parser.gtd.exception.ParseError; @@ -58,6 +60,7 @@ import org.rascalmpl.values.functions.IFunction; import org.rascalmpl.values.parsetrees.ITree; import org.rascalmpl.values.parsetrees.SymbolAdapter; +import org.rascalmpl.values.parsetrees.SymbolFactory; import org.rascalmpl.values.parsetrees.TreeAdapter; import com.github.benmanes.caffeine.cache.Caffeine; @@ -369,7 +372,15 @@ public IFunction bootstrapParsers() { } public IString createHole(ITree part, IInteger index) { - return getParserGenerator().createHole(part, index); + ASTBuilder builder = new ASTBuilder(); + ITree hole = TreeAdapter.getArg(part, "hole"); + ITree sym = TreeAdapter.getArg(hole, "symbol"); + IConstructor symbol = SymbolFactory.typeToSymbol((Sym) builder.buildValue(sym) , false, "*default*"); + + return ctx.getValueFactory().string("\u0000" + symbol.toString() + index + ":\u0000"); + + // we replaced this: + // return getParserGenerator().createHole(part, index); } public IConstructor sym2symbol(ITree parsedSym) { From dc2e985d328cde00de8e38e1119b39a668a70fbc Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 19 Sep 2023 13:07:52 +0200 Subject: [PATCH 5/9] experimental re-implementation of createHole. --- .../values/RascalFunctionValueFactory.java | 7 ++--- .../values/parsetrees/SymbolFactory.java | 27 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 5832ee3422a..b216eb4457b 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -372,13 +372,14 @@ public IFunction bootstrapParsers() { } public IString createHole(ITree part, IInteger index) { - ASTBuilder builder = new ASTBuilder(); ITree hole = TreeAdapter.getArg(part, "hole"); ITree sym = TreeAdapter.getArg(hole, "symbol"); - IConstructor symbol = SymbolFactory.typeToSymbol((Sym) builder.buildValue(sym) , false, "*default*"); + IConstructor symbol = SymbolFactory.typeToSymbol(sym , false, null); - return ctx.getValueFactory().string("\u0000" + symbol.toString() + index + ":\u0000"); + IString result = ctx.getValueFactory().string("\u0000" + symbol.toString() + ":" + index + "\u0000"); + System.err.println("New hole: " + result); + return result; // we replaced this: // return getParserGenerator().createHole(part, index); } diff --git a/src/org/rascalmpl/values/parsetrees/SymbolFactory.java b/src/org/rascalmpl/values/parsetrees/SymbolFactory.java index 1a395c2289e..22e96b5b231 100644 --- a/src/org/rascalmpl/values/parsetrees/SymbolFactory.java +++ b/src/org/rascalmpl/values/parsetrees/SymbolFactory.java @@ -25,6 +25,7 @@ import org.rascalmpl.ast.Type; import org.rascalmpl.interpreter.asserts.NotYetImplemented; import org.rascalmpl.interpreter.utils.Names; +import org.rascalmpl.parser.ASTBuilder; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IInteger; @@ -39,10 +40,15 @@ public class SymbolFactory { private static IValueFactory factory = ValueFactoryFactory.getValueFactory(); + private static ASTBuilder builder = new ASTBuilder(); + public static IConstructor typeToSymbol(ITree parseTree, boolean lex, String layout) { + return SymbolFactory.typeToSymbol((Sym) builder.buildValue(parseTree), lex, layout); + } + public static IConstructor typeToSymbol(Sym type, boolean lex, String layout) { return (IConstructor) symbolAST2SymbolConstructor(type, lex, layout); - } + } public static IConstructor typeToSymbol(Type type, boolean lex, String layout) { if (type.isUser()) { @@ -63,7 +69,8 @@ public static IConstructor typeToSymbol(Type type, boolean lex, String layout) { // TODO: distribute this code over the dynamic.Sym classes in typeOf method private static IValue symbolAST2SymbolConstructor(Sym symbol, boolean lex, String layout) { - + boolean noExpand = lex || layout == null; + if (symbol.isCaseInsensitiveLiteral()) { return factory.constructor(RascalValueFactory.Symbol_Cilit, ciliteral2Symbol(symbol.getCistring())); } @@ -72,7 +79,7 @@ private static IValue symbolAST2SymbolConstructor(Sym symbol, boolean lex, Strin return charclass2Symbol(cc); } if (symbol.isIter()) { - if (lex) { + if (noExpand) { return factory.constructor(RascalValueFactory.Symbol_Iter, symbolAST2SymbolConstructor(symbol.getSymbol(), lex, layout)); } else { @@ -83,7 +90,7 @@ private static IValue symbolAST2SymbolConstructor(Sym symbol, boolean lex, Strin } } if (symbol.isIterStar()) { - if (lex) { + if (noExpand) { return factory.constructor(RascalValueFactory.Symbol_IterStar, symbolAST2SymbolConstructor(symbol.getSymbol(), lex, layout)); } else { @@ -99,7 +106,7 @@ private static IValue symbolAST2SymbolConstructor(Sym symbol, boolean lex, Strin IValue sepSym = symbolAST2SymbolConstructor(symbol.getSep(), lex, layout); IValue seps; - if (lex) { + if (noExpand) { seps = factory.list(sepSym); } else { @@ -114,7 +121,7 @@ private static IValue symbolAST2SymbolConstructor(Sym symbol, boolean lex, Strin IValue elementSym = symbolAST2SymbolConstructor(symbol.getSymbol(), lex, layout); IValue sepSym = symbolAST2SymbolConstructor(symbol.getSep(), lex, layout); IValue seps; - if (lex) { + if (noExpand) { seps = factory.list(sepSym); } else { @@ -148,10 +155,10 @@ private static IValue symbolAST2SymbolConstructor(Sym symbol, boolean lex, Strin if(symbol.isSequence()){ List symbols = symbol.getSequence(); IValue layoutSymbol = factory.constructor(RascalValueFactory.Symbol_Layouts, factory.string(layout)); - IValue[] symValues = new IValue[lex ? symbols.size() : symbols.size() * 2 - 1]; - for(int i = symbols.size() - 1; i >= 0; i -= lex ? 1 : 2) { - symValues[lex ? i : i * 2] = symbolAST2SymbolConstructor(symbols.get(i), lex, layout); - if (lex && i > 0) { + IValue[] symValues = new IValue[noExpand ? symbols.size() : symbols.size() * 2 - 1]; + for(int i = symbols.size() - 1; i >= 0; i -= noExpand ? 1 : 2) { + symValues[noExpand ? i : i * 2] = symbolAST2SymbolConstructor(symbols.get(i), lex, layout); + if (!noExpand && i > 0) { symValues[i - 1] = layoutSymbol; } } From 8e06272a4db1022dcaddd57697962fbaf18ec6db Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 19 Sep 2023 13:44:54 +0200 Subject: [PATCH 6/9] added class cache during AST building --- src/org/rascalmpl/parser/ASTBuilder.java | 34 ++++++++++++------- .../values/RascalFunctionValueFactory.java | 3 -- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/org/rascalmpl/parser/ASTBuilder.java b/src/org/rascalmpl/parser/ASTBuilder.java index 1a93474622a..381775644f5 100644 --- a/src/org/rascalmpl/parser/ASTBuilder.java +++ b/src/org/rascalmpl/parser/ASTBuilder.java @@ -54,8 +54,9 @@ public class ASTBuilder { private static final String MODULE_SORT = "Module"; private final PointerKeyedHashMap constructorCache = new PointerKeyedHashMap(); - private final static HashMap> astConstructors = new HashMap>(); + private final static HashMap> astClasses = new HashMap>(); + private final static ClassLoader classLoader = ASTBuilder.class.getClassLoader(); public static T make(String sort, ISourceLocation src, Object... args) { @@ -445,24 +446,31 @@ private AbstractAST newLift(org.rascalmpl.values.parsetrees.ITree tree, boolean return liftRec(fragment, false, getPatternLayout(tree)); } + private static Class loadClass(String name) throws ClassNotFoundException { + if (astClasses.containsKey(name)) { + return astClasses.get(name); + } + + Class result = null; + + try { + result = classLoader.loadClass("org.rascalmpl.semantics.dynamic." + name); + } + catch (ClassNotFoundException e) { + result = classLoader.loadClass("org.rascalmpl.ast." + name); + } + + astClasses.put(name, result); + return result; + } + private static AbstractAST callMakerMethod(String sort, String cons, Object actuals[], Object keywordActuals[]) { try { String name = sort + '$' + cons; Constructor constructor = astConstructors.get(name); if (constructor == null) { - Class clazz = null; - - try { - clazz = classLoader.loadClass("org.rascalmpl.semantics.dynamic." + name); - } - catch (ClassNotFoundException e) { - // it happens - } - - if (clazz == null) { - clazz = classLoader.loadClass("org.rascalmpl.ast." + name); - } + Class clazz = loadClass(name); constructor = clazz.getConstructors()[0]; constructor.setAccessible(true); diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index b216eb4457b..21cb4e834bc 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -378,10 +378,7 @@ public IString createHole(ITree part, IInteger index) { IString result = ctx.getValueFactory().string("\u0000" + symbol.toString() + ":" + index + "\u0000"); - System.err.println("New hole: " + result); return result; - // we replaced this: - // return getParserGenerator().createHole(part, index); } public IConstructor sym2symbol(ITree parsedSym) { From be4fed54cd2010c4d2b2373f307caf623b4d3ba2 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 19 Sep 2023 15:18:04 +0200 Subject: [PATCH 7/9] avoid superfluous array or list temporaries during AST building --- src/org/rascalmpl/parser/ASTBuilder.java | 94 +++++++++++-------- .../values/parsetrees/TreeAdapter.java | 6 +- 2 files changed, 54 insertions(+), 46 deletions(-) diff --git a/src/org/rascalmpl/parser/ASTBuilder.java b/src/org/rascalmpl/parser/ASTBuilder.java index 381775644f5..5b5de2bc78d 100644 --- a/src/org/rascalmpl/parser/ASTBuilder.java +++ b/src/org/rascalmpl/parser/ASTBuilder.java @@ -19,6 +19,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -192,9 +193,6 @@ private List buildList(org.rascalmpl.values.parsetrees.ITree in) { result.add(buildValue(arg)); i += seps; } - // TreeAdapter.streamListASTArgs(in).forEach(arg -> { - // result.add(buildValue(arg)); - // }); return result; } @@ -222,26 +220,33 @@ private AbstractAST buildContextFreeNode(org.rascalmpl.values.parsetrees.ITree t sort = "KeywordArguments_Expression"; break; } - IList args = getASTArgs(tree); - int arity = args.length(); - Object actuals[] = new Object[arity+2]; - actuals[0] = TreeAdapter.getLocation(tree); - actuals[1] = tree; + // Here we see how precisely the constructors of the generated AST hierarchy + // are connected to the shape of the grammar rules for the Rascal syntax. + // The hierarchy is generated from the same grammar that these parse trees + // come from, and that's why this works. + Constructor constructor = getConstructor(sort, cons); + int parameterCount = constructor.getParameterCount(); + Object[] actuals = new Object[parameterCount]; + int i = 0; + actuals[i++] = TreeAdapter.getLocation(tree); + actuals[i++] = tree; - int i = 2; - for (IValue arg : args) { - org.rascalmpl.values.parsetrees.ITree argTree = (org.rascalmpl.values.parsetrees.ITree) arg; + IList args = tree.getArgs(); - if (TreeAdapter.isList(argTree)) { - actuals[i] = buildList((org.rascalmpl.values.parsetrees.ITree) arg); - } - else { - actuals[i] = buildValue(arg); + for (int j = 0; j < args.length(); j += 2 /* skipping layout */) { + ITree argTree = (ITree) args.get(j); + + if (!TreeAdapter.isLiteral(argTree) && !TreeAdapter.isCILiteral(argTree) && !TreeAdapter.isEmpty(argTree)) { + if (TreeAdapter.isList(argTree)) { + actuals[i++] = buildList((org.rascalmpl.values.parsetrees.ITree) argTree); + } + else { + actuals[i++] = buildValue(argTree); + } } - i++; } - return callMakerMethod(sort, cons, actuals, null); + return callMakerMethod(constructor, actuals, null); } private AbstractAST buildLexicalNode(org.rascalmpl.values.parsetrees.ITree tree) { @@ -376,22 +381,6 @@ private String getLayoutName(IConstructor production) { return null; } - private IList getASTArgs(org.rascalmpl.values.parsetrees.ITree tree) { - IList children = TreeAdapter.getArgs(tree); - IListWriter writer = ValueFactoryFactory.getValueFactory().listWriter(); - - for (int i = 0; i < children.length(); i++) { - org.rascalmpl.values.parsetrees.ITree kid = (org.rascalmpl.values.parsetrees.ITree) children.get(i); - if (!TreeAdapter.isLiteral(kid) && !TreeAdapter.isCILiteral(kid) && !TreeAdapter.isEmpty(kid)) { - writer.append(kid); - } - // skip layout - i++; - } - - return writer.done(); - } - private String sortName(org.rascalmpl.values.parsetrees.ITree tree) { if (TreeAdapter.isAppl(tree)) { return TreeAdapter.getSortName(tree); @@ -434,10 +423,7 @@ private boolean isNewEmbedding(org.rascalmpl.values.parsetrees.ITree tree) { } private boolean isLexical(org.rascalmpl.values.parsetrees.ITree tree) { - if (TreeAdapter.isRascalLexical(tree)) { - return true; - } - return false; + return TreeAdapter.isRascalLexical(tree); } private AbstractAST newLift(org.rascalmpl.values.parsetrees.ITree tree, boolean match) { @@ -464,7 +450,7 @@ private static Class loadClass(String name) throws ClassNotFoundException { return result; } - private static AbstractAST callMakerMethod(String sort, String cons, Object actuals[], Object keywordActuals[]) { + private static Constructor getConstructor(String sort, String cons) { try { String name = sort + '$' + cons; Constructor constructor = astConstructors.get(name); @@ -477,7 +463,19 @@ private static AbstractAST callMakerMethod(String sort, String cons, Object actu astConstructors.put(name, constructor); } - return (AbstractAST) constructor.newInstance(actuals); + return constructor; + } catch (SecurityException e) { + throw unexpectedError(e); + } catch (IllegalArgumentException e) { + throw unexpectedError(e); + } catch (ClassNotFoundException e) { + throw unexpectedError(e); + } + } + + private static AbstractAST callMakerMethod(String sort, String cons, Object actuals[], Object keywordActuals[]) { + try { + return (AbstractAST) getConstructor(sort, cons).newInstance(actuals); } catch (SecurityException e) { throw unexpectedError(e); } catch (IllegalArgumentException e) { @@ -486,7 +484,21 @@ private static AbstractAST callMakerMethod(String sort, String cons, Object actu throw unexpectedError(e); } catch (InvocationTargetException e) { throw unexpectedError(e); - } catch (ClassNotFoundException e) { + } catch (InstantiationException e) { + throw unexpectedError(e); + } + } + + private static AbstractAST callMakerMethod(Constructor constructor, Object actuals[], Object keywordActuals[]) { + try { + return (AbstractAST) constructor.newInstance(actuals); + } catch (SecurityException e) { + throw unexpectedError(e); + } catch (IllegalArgumentException e) { + throw unexpectedError(e); + } catch (IllegalAccessException e) { + throw unexpectedError(e); + } catch (InvocationTargetException e) { throw unexpectedError(e); } catch (InstantiationException e) { throw unexpectedError(e); diff --git a/src/org/rascalmpl/values/parsetrees/TreeAdapter.java b/src/org/rascalmpl/values/parsetrees/TreeAdapter.java index 0ac65a94b94..68cb24672da 100644 --- a/src/org/rascalmpl/values/parsetrees/TreeAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/TreeAdapter.java @@ -324,11 +324,7 @@ public static boolean isOpt(ITree tree) { } public static IList getArgs(ITree tree) { - if (isAppl(tree)) { - return (IList) tree.get("args"); - } - - throw new ImplementationError(NO_ARGS_EXCEPTION_MESSAGE + tree.getName()); + return tree.getArgs(); } public static ITree setArgs(ITree tree, IList args) { From 921639d020fb096cedcf80ac5c11b6ab79a3c240 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 20 Sep 2023 09:30:12 +0200 Subject: [PATCH 8/9] some import cleanup and sharing more instances of ASTBuilder to avoid class reloading. Does not help much --- src/org/rascalmpl/interpreter/result/JavaMethod.java | 2 -- src/org/rascalmpl/interpreter/utils/Modules.java | 3 +-- src/org/rascalmpl/parser/ASTBuilder.java | 3 --- src/org/rascalmpl/semantics/dynamic/Import.java | 6 +++--- src/org/rascalmpl/values/RascalFunctionValueFactory.java | 2 -- .../test/functionality/ParallelEvaluatorsTests.java | 1 - 6 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/org/rascalmpl/interpreter/result/JavaMethod.java b/src/org/rascalmpl/interpreter/result/JavaMethod.java index 8a3d4532e96..10530bda752 100644 --- a/src/org/rascalmpl/interpreter/result/JavaMethod.java +++ b/src/org/rascalmpl/interpreter/result/JavaMethod.java @@ -37,8 +37,6 @@ import org.rascalmpl.interpreter.utils.JavaBridge; import org.rascalmpl.interpreter.utils.Names; import org.rascalmpl.uri.URIUtil; -import org.rascalmpl.uri.file.TempURIResolver; - import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IValue; import io.usethesource.vallang.type.Type; diff --git a/src/org/rascalmpl/interpreter/utils/Modules.java b/src/org/rascalmpl/interpreter/utils/Modules.java index e00b4ac59c3..595d0cc9f27 100644 --- a/src/org/rascalmpl/interpreter/utils/Modules.java +++ b/src/org/rascalmpl/interpreter/utils/Modules.java @@ -35,11 +35,10 @@ public static ISet getSyntax(ITree tree) { return get(tree, "syntax"); } - public static List getTypeDeclarations(ITree tree) { + public static List getTypeDeclarations(ITree tree, ASTBuilder builder) { ITree body = TreeAdapter.getArg(tree, "body"); ITree toplevels = TreeAdapter.getArg(body, "toplevels"); List result = new LinkedList<>(); - ASTBuilder builder = new ASTBuilder(); for (IValue toplevel : TreeAdapter.getListASTArgs(toplevels)) { ITree declaration = TreeAdapter.getArg((ITree) toplevel, "declaration"); diff --git a/src/org/rascalmpl/parser/ASTBuilder.java b/src/org/rascalmpl/parser/ASTBuilder.java index 5b5de2bc78d..67e2da2e3ff 100644 --- a/src/org/rascalmpl/parser/ASTBuilder.java +++ b/src/org/rascalmpl/parser/ASTBuilder.java @@ -19,7 +19,6 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -36,12 +35,10 @@ import org.rascalmpl.semantics.dynamic.Tree; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IList; -import io.usethesource.vallang.IListWriter; import io.usethesource.vallang.ISet; import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IValue; import io.usethesource.vallang.exceptions.FactTypeUseException; -import org.rascalmpl.values.ValueFactoryFactory; import org.rascalmpl.values.parsetrees.ITree; import org.rascalmpl.values.parsetrees.ProductionAdapter; import org.rascalmpl.values.parsetrees.SymbolAdapter; diff --git a/src/org/rascalmpl/semantics/dynamic/Import.java b/src/org/rascalmpl/semantics/dynamic/Import.java index 4304581a845..b5790fc8819 100644 --- a/src/org/rascalmpl/semantics/dynamic/Import.java +++ b/src/org/rascalmpl/semantics/dynamic/Import.java @@ -88,7 +88,7 @@ import io.usethesource.vallang.type.Type; public abstract class Import { - + private static final ThreadLocal builder = ThreadLocal.withInitial(() -> new ASTBuilder()); static public class External extends org.rascalmpl.ast.Import.External { public External(ISourceLocation src, IConstructor node, QualifiedName name, @@ -381,7 +381,7 @@ private static Module buildModule(ISourceLocation uri, ModuleEnvironment env, I } private static ASTBuilder getBuilder() { - return new ASTBuilder(); + return builder.get(); } private static void addImportToCurrentModule(ISourceLocation src, String name, IEvaluator> eval) { @@ -490,7 +490,7 @@ else if (reg.exists(parserCacheFile)) { } private static void declareTypesWhichDoNotNeedImportedModulesAlready(IEvaluator> eval, ModuleEnvironment env, ITree top) { - List decls = Modules.getTypeDeclarations(top); + List decls = Modules.getTypeDeclarations(top, getBuilder()); eval.__getTypeDeclarator().evaluateDeclarations(decls, eval.getCurrentEnvt(), true); } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 21cb4e834bc..40a89717b74 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -24,7 +24,6 @@ import java.util.function.Supplier; import org.rascalmpl.ast.AbstractAST; -import org.rascalmpl.ast.Sym; import org.rascalmpl.exceptions.ImplementationError; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.exceptions.Throw; @@ -40,7 +39,6 @@ import org.rascalmpl.interpreter.result.ResultFactory; import org.rascalmpl.interpreter.staticErrors.UndeclaredNonTerminal; import org.rascalmpl.library.lang.rascal.syntax.RascalParser; -import org.rascalmpl.parser.ASTBuilder; import org.rascalmpl.parser.ParserGenerator; import org.rascalmpl.parser.gtd.IGTD; import org.rascalmpl.parser.gtd.exception.ParseError; diff --git a/test/org/rascalmpl/test/functionality/ParallelEvaluatorsTests.java b/test/org/rascalmpl/test/functionality/ParallelEvaluatorsTests.java index 3730eaba9b0..89c01a6701c 100644 --- a/test/org/rascalmpl/test/functionality/ParallelEvaluatorsTests.java +++ b/test/org/rascalmpl/test/functionality/ParallelEvaluatorsTests.java @@ -16,7 +16,6 @@ import org.rascalmpl.interpreter.env.GlobalEnvironment; import org.rascalmpl.interpreter.env.ModuleEnvironment; import org.rascalmpl.interpreter.load.StandardLibraryContributor; -import org.rascalmpl.shell.ShellEvaluatorFactory; import org.rascalmpl.values.ValueFactoryFactory; import io.usethesource.vallang.ISourceLocation; From a443cca974ed5f45cae1b4e62c781bf80b2f18dd Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 20 Sep 2023 14:04:25 +0200 Subject: [PATCH 9/9] minor optimizations --- .../values/parsetrees/ProductionAdapter.java | 8 ++++---- .../values/parsetrees/SymbolAdapter.java | 15 +++++---------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java b/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java index faecbcc6073..907ae429a9f 100644 --- a/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java @@ -51,16 +51,16 @@ public static IConstructor getType(IConstructor tree) { } public static IConstructor getDefined(IConstructor tree) { - return (IConstructor) tree.get("def"); + return (IConstructor) tree.get(0); } public static IConstructor setDefined(IConstructor tree, IConstructor sym) { - return (IConstructor) tree.set("def", sym); + return (IConstructor) tree.set(0 /*def */, sym); } public static IList getSymbols(IConstructor tree) { if (isDefault(tree)) { - return (IList) tree.get("symbols"); + return (IList) tree.get(1 /*symbols */); } return null; } @@ -108,7 +108,7 @@ public static String getSortName(IConstructor tree) { public static ISet getAttributes(IConstructor tree) { if (isDefault(tree)) { - return (ISet) tree.get("attributes"); + return (ISet) tree.get(2 /* "attributes" */); } return ValueFactoryFactory.getValueFactory().set(); diff --git a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java index 5b29b107e73..5d147f453f6 100644 --- a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java @@ -83,7 +83,7 @@ private SymbolAdapter() { public static IConstructor delabel(IConstructor sym) { if (isLabel(sym)) { - return (IConstructor) sym.get("symbol"); // do not use getSymbol() here! + return (IConstructor) sym.get(1 /*symbol*/); // do not use getSymbol() here! } return sym; } @@ -147,7 +147,7 @@ public static boolean isStartSort(IConstructor tree) { public static IConstructor getStart(IConstructor tree) { if (isStartSort(tree)) { tree = delabel(tree); - return (IConstructor) tree.get("symbol"); + return (IConstructor) tree.get(0 /* symbol */); } throw new ImplementationError("Symbol does not have a child named symbol: " + tree); } @@ -160,7 +160,7 @@ public static IConstructor getLabeledSymbol(IConstructor tree) { public static IConstructor getSymbol(IConstructor tree) { tree = delabel(tree); if (isOpt(tree) || isIterPlus(tree) || isIterStar(tree) || isIterPlusSeps(tree) || isIterStarSeps(tree) || isMeta(tree) || isConditional(tree)) { - return ((IConstructor) tree.get("symbol")); + return ((IConstructor) tree.get(0 /*"symbol" */)); } throw new ImplementationError("Symbol does not have a child named symbol: " + tree); @@ -653,11 +653,6 @@ public static boolean isParameter(IConstructor symbol) { return symbol.getConstructorType() == Symbol_Parameter; } - public static IConstructor getRhs(IConstructor symbol) { - symbol = delabel(symbol); - return (IConstructor) symbol.get("rhs"); - } - public static boolean isIterStarSeps(IConstructor rhs) { rhs = delabel(rhs); return rhs.getConstructorType() == Symbol_IterStarSeps; @@ -1075,11 +1070,11 @@ private static IConstructor range(int begin, int end) { } private static int rangeBegin(IConstructor range) { - return ((IInteger) range.get("begin")).intValue(); + return ((IInteger) range.get(0 /*begin */)).intValue(); } private static int rangeEnd(IConstructor range) { - return ((IInteger) range.get("end")).intValue(); + return ((IInteger) range.get(1 /*end*/)).intValue(); } public static boolean isParametrizableType(IConstructor sort) {