From d860cf0bccfab41c75f70d2b691941d5b6dae416 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 Sep 2023 11:05:20 +0200 Subject: [PATCH 01/11] added optional storage of constructor names and datatype names while serializing to JSON. Wired this through the different webservers as well --- src/org/rascalmpl/library/Content.rsc | 2 +- src/org/rascalmpl/library/lang/json/IO.java | 8 +++-- src/org/rascalmpl/library/lang/json/IO.rsc | 36 ++++++++++++------- .../lang/json/internal/JsonValueWriter.java | 23 ++++++++++++ src/org/rascalmpl/library/util/TermREPL.java | 9 +++-- src/org/rascalmpl/library/util/Webserver.java | 7 +++- src/org/rascalmpl/repl/REPLContentServer.java | 7 +++- 7 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index 28be88bbe94..0075670e713 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -80,7 +80,7 @@ which involves a handy, automatic, encoding of Rascal values into json values. data Response = response(Status status, str mimeType, map[str,str] header, str content) | fileResponse(loc file, str mimeType, map[str,str] header) - | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") + | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", bool explicitConstructorNames=false, bool explicitDataTypes=false) ; diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 6cecd64debc..c21db87dbe8 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -137,7 +137,7 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool } } - public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) { + public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) { try (JsonWriter out = new JsonWriter(new OutputStreamWriter(URIResolverRegistry.getInstance().getOutputStream(loc, false), Charset.forName("UTF8")))) { if (indent.intValue() > 0) { out.setIndent(" ".substring(0, indent.intValue() % 9)); @@ -148,13 +148,15 @@ public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations .setDatesAsInt(dateTimeAsInt.getValue()) .setUnpackedLocations(unpackedLocations.getValue()) .setDropOrigins(dropOrigins.getValue()) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) .write(out, value); } catch (IOException e) { throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null); } } - public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) { + public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) { StringWriter string = new StringWriter(); try (JsonWriter out = new JsonWriter(string)) { @@ -166,6 +168,8 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor .setDatesAsInt(dateTimeAsInt.getValue()) .setUnpackedLocations(unpackedLocations.getValue()) .setDropOrigins(dropOrigins.getValue()) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) .write(out, value); return values.string(string.toString()); diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 3c8a9f3dede..661a062fa23 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -15,25 +15,20 @@ module lang::json::IO @javaClass{org.rascalmpl.library.lang.json.IO} -@deprecated{ -use writeJSON -} +@deprecated{use writeJSON} public java str toJSON(value v); @javaClass{org.rascalmpl.library.lang.json.IO} -@deprecated{ -use asJSON -} +@deprecated{use asJSON} public java str toJSON(value v, bool compact); @javaClass{org.rascalmpl.library.lang.json.IO} -@deprecated{ -use readJSON -} +@deprecated{use readJSON} public java &T fromJSON(type[&T] typ, str src); @javaClass{org.rascalmpl.library.lang.json.IO} -@synopsis{reads JSON values from a stream +@synopsis{reads JSON values from a stream} +@description{ In general the translation behaves as follows: * Objects translate to map[str,value] by default, unless a node is expected (properties are then translated to keyword fields) * Arrays translate to lists by default, or to a set if that is expected or a tuple if that is expected. Arrays may also be interpreted as constructors or nodes (see below) @@ -52,7 +47,24 @@ In general the translation behaves as the same as for ((readJSON)).} java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false); @javaClass{org.rascalmpl.library.lang.json.IO} -java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true); +@synopsis{Serializes a value as a JSON string and stream it} +@description{ +This function tries to map Rascal values to JSON values in a natural way. +In particular it tries to create a value that has the same number of recursive levels, +such that one constructor maps to one object. The serialization is typically _lossy_ since +JSON values by default do not explicitly encode the class or constructor while Rascal data types do. + +If you need the names of constructors or data-types in your result, then use the parameters: +* `explicitConstructorNames=true` will store the name of every constructor in a field `_constructor` +* `explicitDataTypes=true` will store the name of the ADT in a field called `_type` + +The `dateTimeFormat` parameter dictates how `datetime` values will be printed. + +The `unpackedLocations` parameter will produce an object with many fields for every property of a `loc` value, but +if set to false a `loc` will be printed as a string. +} +java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false); @javaClass{org.rascalmpl.library.lang.json.IO} -java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true); +@synopsis{Serializes a value as a JSON string and stores it as a string} +java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false); diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java index 8fc7452cd91..9ace3155adb 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java @@ -44,6 +44,8 @@ public class JsonValueWriter { private boolean datesAsInts = true; private boolean unpackedLocations = false; private boolean dropOrigins = true; + private boolean explicitConstructorNames = false; + private boolean explicitDataTypes; public JsonValueWriter() { setCalendarFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); @@ -78,6 +80,16 @@ public JsonValueWriter setDropOrigins(boolean setting) { return this; } + public JsonValueWriter setExplicitConstructorNames(boolean setting) { + this.explicitConstructorNames = setting; + return this; + } + + public JsonValueWriter setExplicitDataTypes(boolean setting) { + this.explicitDataTypes = setting; + return this; + } + public void write(JsonWriter out, IValue value) throws IOException { value.accept(new IValueVisitor() { @@ -229,6 +241,17 @@ public Void visitConstructor(IConstructor o) throws IOException { } out.beginObject(); + + if (explicitConstructorNames) { + out.name("_constructor"); + out.value(o.getName()); + } + + if (explicitDataTypes) { + out.name("_type"); + out.value(o.getType().getName()); + } + int i = 0; for (IValue arg : o) { out.name(o.getConstructorType().getFieldName(i)); diff --git a/src/org/rascalmpl/library/util/TermREPL.java b/src/org/rascalmpl/library/util/TermREPL.java index f149bb36340..8ed61b9d41e 100644 --- a/src/org/rascalmpl/library/util/TermREPL.java +++ b/src/org/rascalmpl/library/util/TermREPL.java @@ -238,10 +238,15 @@ private void handleJSONResponse(Map output, IConstructor re IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); - + IValue ecn = kws.getParameter("explicitConstructorNames"); + IValue edt = kws.getParameter("explicitDataTypes"); + JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") - .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); + .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) + .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) + .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) + ; final ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/src/org/rascalmpl/library/util/Webserver.java b/src/org/rascalmpl/library/util/Webserver.java index 87a2d14ee4a..6cff5ec98f1 100644 --- a/src/org/rascalmpl/library/util/Webserver.java +++ b/src/org/rascalmpl/library/util/Webserver.java @@ -238,10 +238,15 @@ private Response translateJsonResponse(Method method, IConstructor cons) { IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); + IValue ecn = kws.getParameter("explicitConstructorNames"); + IValue edt = kws.getParameter("explicitDataTypes"); JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") - .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); + .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) + .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) + .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) + ; try { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/src/org/rascalmpl/repl/REPLContentServer.java b/src/org/rascalmpl/repl/REPLContentServer.java index de811bcb6ef..becc0a80871 100644 --- a/src/org/rascalmpl/repl/REPLContentServer.java +++ b/src/org/rascalmpl/repl/REPLContentServer.java @@ -135,10 +135,15 @@ private static Response translateJsonResponse(Method method, IConstructor cons) IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); + IValue ecn = kws.getParameter("explicitConstructorNames"); + IValue edt = kws.getParameter("explicitDataTypes"); JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") - .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); + .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) + .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) + .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) + ; try { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); From 2ca8493ae27ff0606683b4ee0fb163ad0c563d2a Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 Sep 2023 11:08:45 +0200 Subject: [PATCH 02/11] added doc --- src/org/rascalmpl/library/lang/json/IO.rsc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 661a062fa23..95410e096e4 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -67,4 +67,7 @@ java void writeJSON(loc target, value val, bool unpackedLocations=false, str dat @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{Serializes a value as a JSON string and stores it as a string} +@description{ +This function uses `writeJSON` and stores the result in a string. +} java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false); From d86c683d7baac6573a9e2a8bcba297c73f396cd9 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 Sep 2023 11:23:54 +0200 Subject: [PATCH 03/11] explicitDatatypes requires explicitConstructors --- .../rascalmpl/library/lang/json/internal/JsonValueWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java index 9ace3155adb..c7430e883ec 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java @@ -242,7 +242,7 @@ public Void visitConstructor(IConstructor o) throws IOException { out.beginObject(); - if (explicitConstructorNames) { + if (explicitConstructorNames || explicitDataTypes) { out.name("_constructor"); out.value(o.getName()); } From 4cabf0c7d943eb8121b5909993228d492c8e749c Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 Sep 2023 11:24:18 +0200 Subject: [PATCH 04/11] Can use explicitConstructors and explicitTypes while reading JSON back as well to improve constructor and type selection accuracy --- .../lang/json/internal/JsonValueReader.java | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 23b64f81606..04b321b41e6 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.uri.URIUtil; @@ -59,6 +60,8 @@ public class JsonValueReader { private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; + private boolean explicitConstructorNames; + private boolean explicitDataTypes; /** * @param vf factory which will be used to construct values @@ -105,6 +108,17 @@ protected SimpleDateFormat initialValue() { return this; } + public JsonValueReader setExplicitConstructorNames(boolean value) { + this.explicitConstructorNames = value; + return this; + } + + public JsonValueReader setExplicitDataTypes(boolean value) { + this.explicitDataTypes = value; + return this; + } + + /** * Read and validate a Json stream as an IValue * @param in json stream @@ -495,7 +509,37 @@ public IValue visitAbstractData(Type type) throws IOException { assert in.peek() == JsonToken.BEGIN_OBJECT; - Set alternatives = store.lookupAlternatives(type); + Set alternatives = null; + + // use explicit information in the JSON to select and filter constructors from the TypeStore + // we expect always to have the field _constructor before _type. + if (explicitConstructorNames) { + String consLabel = in.nextName(); + if ("_constructor".equals(consLabel)) { + String consName = in.nextString(); + + alternatives = store.lookupConstructors(consName); + + if (explicitDataTypes) { + String dtLabel = in.nextName(); + + if ("_type".equals(dtLabel)) { + String dtValue = in.nextString(); + alternatives = alternatives.stream().filter(t -> t.isAbstractData() && t.getName().equals(dtValue)).collect(Collectors.toSet()); + } + else { + throw new IOException("Expected _type field but got " + dtLabel); + } + } + } + else { + throw new IOException("Expected _constructor field but got " + consLabel); + } + } + else { + alternatives = store.lookupAlternatives(type); + } + if (alternatives.size() > 1) { monitor.warning("selecting arbitrary constructor for " + type, vf.sourceLocation(in.getPath())); } From cb3675da61292b04792fb6ea1cb655703dd4b41e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 Sep 2023 11:40:40 +0200 Subject: [PATCH 05/11] documented behavior for overflow --- src/org/rascalmpl/library/lang/json/IO.rsc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 95410e096e4..e1b542bcd6c 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -62,6 +62,12 @@ The `dateTimeFormat` parameter dictates how `datetime` values will be printed. The `unpackedLocations` parameter will produce an object with many fields for every property of a `loc` value, but if set to false a `loc` will be printed as a string. + +:::warning +It is understood that Rascal's number types have arbitrary precision, but this is not supported by the JSON writer. +As such when an `int` is printed that does not fit into a JVM `long`, there will be truncation to the lower 64 bits. +For `real` numbers that are larger than JVM's double you get "negative infinity" or "positive infinity" as a result. +::: } java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false); From dbcc68b6f0efc96d5f9d346a921ea15b1df819a7 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 Sep 2023 11:42:49 +0200 Subject: [PATCH 06/11] new keyword parameters for readers --- src/org/rascalmpl/library/lang/json/IO.java | 8 ++++++-- src/org/rascalmpl/library/lang/json/IO.rsc | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index c21db87dbe8..6cf0a281074 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -100,7 +100,7 @@ public IValue fromJSON(IValue type, IString src) { } - public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins) { + public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); @@ -108,6 +108,8 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, in.setLenient(lenient.getValue()); return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? loc : null) .setCalendarFormat(dateTimeFormat.getValue()) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) .read(in, start); } catch (IOException e) { @@ -119,7 +121,7 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, } } - public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins) { + public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) { TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); @@ -127,6 +129,8 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool in.setLenient(lenient.getValue()); return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? URIUtil.rootLocation("unknown") : null) .setCalendarFormat(dateTimeFormat.getValue()) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) .read(in, start); } catch (IOException e) { diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index e1b542bcd6c..fa2c7e14cf1 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -39,12 +39,12 @@ In general the translation behaves as follows: * If num, int, real or rat are expected both strings and number values are mapped * If loc is expected than strings which look like URI are parsed (containing :/) or a file:/// URI is build, or if an object is found each separate field of a location object is read from the respective properties: { scheme : str, authority: str?, path: str?, fragment: str?, query: str?, offset: int, length: int, begin: [bl, bc], end: [el, ec]}} -java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false); +java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, bool explicitConstructorNames=false, bool explicitDataTypes=false); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{parses JSON values from a string In general the translation behaves as the same as for ((readJSON)).} -java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false); +java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, bool explicitConstructorNames=false, bool explicitDataTypes=false); @javaClass{org.rascalmpl.library.lang.json.IO} @synopsis{Serializes a value as a JSON string and stream it} From 4c57b8d2e03b5d239be8eefc90a402106c94e4b4 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 12 Sep 2023 11:51:57 +0200 Subject: [PATCH 07/11] enums are off in explicitConstructor mode --- src/org/rascalmpl/library/lang/json/IO.java | 3 --- .../rascalmpl/library/lang/json/internal/JsonValueWriter.java | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index 6cf0a281074..fcebd8fabbb 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -181,7 +181,4 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null); } } - - - } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java index c7430e883ec..90603550a83 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java @@ -234,7 +234,7 @@ public Void visitNode(INode o) throws IOException { @Override public Void visitConstructor(IConstructor o) throws IOException { - if (o.getConstructorType().getArity() == 0 && !o.asWithKeywordParameters().hasParameters()) { + if (!explicitConstructorNames && o.getConstructorType().getArity() == 0 && !o.asWithKeywordParameters().hasParameters()) { // enums! out.value(o.getName()); return null; From dc11d9d0c22cc200b7431276382faaa0cd069aa8 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Sat, 23 Nov 2024 12:38:14 +0100 Subject: [PATCH 08/11] changed :::warning to @pitfalls as it should be --- src/org/rascalmpl/library/lang/json/IO.rsc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index fa2c7e14cf1..171c1c0aca0 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -62,12 +62,11 @@ The `dateTimeFormat` parameter dictates how `datetime` values will be printed. The `unpackedLocations` parameter will produce an object with many fields for every property of a `loc` value, but if set to false a `loc` will be printed as a string. - -:::warning -It is understood that Rascal's number types have arbitrary precision, but this is not supported by the JSON writer. +} +@pitfalls{ +* It is understood that Rascal's number types have arbitrary precision, but this is not supported by the JSON writer. As such when an `int` is printed that does not fit into a JVM `long`, there will be truncation to the lower 64 bits. For `real` numbers that are larger than JVM's double you get "negative infinity" or "positive infinity" as a result. -::: } java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false); From c7ddc6c68a4bf6320675bbe0526b27eb31f43adb Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Sat, 23 Nov 2024 13:30:26 +0100 Subject: [PATCH 09/11] added tests and fixed the bugs that were triggered by them --- .../lang/json/internal/JsonValueReader.java | 86 ++++++++++++++----- .../lang/json/internal/JsonValueWriter.java | 2 +- .../tests/library/lang/json/JSONIOTests.rsc | 28 ++++++ 3 files changed, 92 insertions(+), 24 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 04b321b41e6..ffc9c00d512 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -115,6 +115,9 @@ public JsonValueReader setExplicitConstructorNames(boolean value) { public JsonValueReader setExplicitDataTypes(boolean value) { this.explicitDataTypes = value; + if (value) { + this.explicitConstructorNames = true; + } return this; } @@ -507,33 +510,61 @@ public IValue visitAbstractData(Type type) throws IOException { throw new IOException("no nullary constructor found for " + type); } - assert in.peek() == JsonToken.BEGIN_OBJECT; - - Set alternatives = null; + Set alternatives = null; + in.beginObject(); + int startPos = getPos(); + int startLine = getLine(); + int startCol = getCol(); + // use explicit information in the JSON to select and filter constructors from the TypeStore // we expect always to have the field _constructor before _type. - if (explicitConstructorNames) { - String consLabel = in.nextName(); - if ("_constructor".equals(consLabel)) { - String consName = in.nextString(); + if (explicitConstructorNames || explicitDataTypes) { + String consName = null; + String typeName = null; // this one is optional, and the order with cons is not defined. - alternatives = store.lookupConstructors(consName); + String consLabel = in.nextName(); - if (explicitDataTypes) { - String dtLabel = in.nextName(); + // first we read either a cons name or a type name + if (explicitConstructorNames && "_constructor".equals(consLabel)) { + consName = in.nextString(); + } + else if (explicitDataTypes && "_type".equals(consLabel)) { + typeName = in.nextString(); + } - if ("_type".equals(dtLabel)) { - String dtValue = in.nextString(); - alternatives = alternatives.stream().filter(t -> t.isAbstractData() && t.getName().equals(dtValue)).collect(Collectors.toSet()); - } - else { - throw new IOException("Expected _type field but got " + dtLabel); - } + // optionally read the second field + if (explicitDataTypes && typeName == null) { + // we've read a constructor name, but we still need a type name + consLabel = in.nextName(); + if (explicitDataTypes && "_type".equals(consLabel)) { + typeName = in.nextString(); + } + } + else if (explicitDataTypes && consName == null) { + // we've read type name, but we still need a constructor name + consLabel = in.nextName(); + if (explicitDataTypes && "_constructor".equals(consLabel)) { + consName = in.nextString(); } } + + if (explicitDataTypes && typeName == null) { + throw new IOException("Missing a _type field: " + in.getPath()); + } + else if (explicitConstructorNames && consName == null) { + throw new IOException("Missing a _constructor field: " + in.getPath()); + } + + if (typeName != null && consName != null) { + // first focus on the given type name + var dataType = TF.abstractDataType(store, typeName); + alternatives = store.lookupConstructor(dataType, consName); + } else { - throw new IOException("Expected _constructor field but got " + consLabel); + // we only have a constructor name + // lookup over all data types by constructor name + alternatives = store.lookupConstructors(consName); } } else { @@ -545,10 +576,6 @@ public IValue visitAbstractData(Type type) throws IOException { } Type cons = alternatives.iterator().next(); - in.beginObject(); - int startPos = getPos(); - int startLine = getLine(); - int startCol = getCol(); IValue[] args = new IValue[cons.getArity()]; Map kwParams = new HashMap<>(); @@ -576,7 +603,20 @@ else if (cons.hasKeywordField(label, store)) { } } else { // its a normal arg, pass its label to the child - throw new IOException("Unknown field " + label + ":" + in.getPath()); + if (!explicitConstructorNames && "_constructor".equals(label)) { + // ignore additional _constructor fields. + in.nextString(); // skip the constructor value + continue; + } + else if (!explicitDataTypes && "_type".equals(label)) { + // ignore additional _type fields. + in.nextString(); // skip the type value + continue; + } + else { + // field label does not match data type definition + throw new IOException("Unknown field " + label + ":" + in.getPath()); + } } } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java index 90603550a83..d51ac662c93 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java @@ -234,7 +234,7 @@ public Void visitNode(INode o) throws IOException { @Override public Void visitConstructor(IConstructor o) throws IOException { - if (!explicitConstructorNames && o.getConstructorType().getArity() == 0 && !o.asWithKeywordParameters().hasParameters()) { + if (!explicitConstructorNames && !explicitDataTypes && o.getConstructorType().getArity() == 0 && !o.asWithKeywordParameters().hasParameters()) { // enums! out.value(o.getName()); return null; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 4f99f35e3e0..de81374612a 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -69,4 +69,32 @@ test bool originTracking() { } return true; +} + +test bool explicitConstructorNames() { + example = data4(e=z()); + json = asJSON(example, explicitConstructorNames=true); + + assert json == "{\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\"}}"; + + assert parseJSON(#DATA4, json, explicitConstructorNames=true) == example; + + // here we can't be sure to get z() back, but we will get some Enum + assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitConstructorNames=false); + + return true; +} + +test bool explicitDataTypes() { + example = data4(e=z()); + json = asJSON(example, explicitDataTypes=true); + + assert json == "{\"_constructor\":\"data4\",\"_type\":\"DATA4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; + + assert parseJSON(#DATA4, json, explicitDataTypes=true) == example; + + // here we can't be sure to get z() back, but we will get some Enum + // assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); + + return true; } \ No newline at end of file From 6996898ffb03347619fff0dd100edbb7bf0e27ac Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Sat, 23 Nov 2024 13:53:26 +0100 Subject: [PATCH 10/11] added more tests and fixed more bugs --- .../library/lang/json/internal/JsonValueReader.java | 6 +++--- .../lang/rascal/tests/library/lang/json/JSONIOTests.rsc | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index ffc9c00d512..e43e658afa9 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -25,8 +25,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; - import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.uri.URIUtil; import io.usethesource.vallang.IInteger; @@ -494,7 +492,6 @@ private int getCol() { } } - @Override public IValue visitAbstractData(Type type) throws IOException { if (in.peek() == JsonToken.STRING) { @@ -574,6 +571,9 @@ else if (explicitConstructorNames && consName == null) { if (alternatives.size() > 1) { monitor.warning("selecting arbitrary constructor for " + type, vf.sourceLocation(in.getPath())); } + else if (alternatives.size() == 0) { + throw new IOException("No fitting constructor found for " + in.getPath()); + } Type cons = alternatives.iterator().next(); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index f94c3d06cff..0fd17e4119f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -91,10 +91,15 @@ test bool explicitDataTypes() { assert json == "{\"_constructor\":\"data4\",\"_type\":\"DATA4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; + // _constructor and _type must be the first fields assert parseJSON(#DATA4, json, explicitDataTypes=true) == example; + // _type and _constructor may appear in a different order + flippedJson = "{\"_type\":\"DATA4\",\"_constructor\":\"data4\",\"e\":{\"_constructor\":\"z\",\"_type\":\"Enum\"}}"; + assert parseJSON(#DATA4, flippedJson, explicitDataTypes=true) == example; + // here we can't be sure to get z() back, but we will get some Enum - // assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); + assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); return true; } \ No newline at end of file From 1c11d5e6caecbaf42ae63d6097b6909b4655aabc Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 25 Nov 2024 19:56:36 +0100 Subject: [PATCH 11/11] added synopsis to old JSON IO functions such that they are documented and their deprecation is too --- src/org/rascalmpl/library/lang/json/IO.rsc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 171c1c0aca0..bcf08efca99 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -15,15 +15,18 @@ module lang::json::IO @javaClass{org.rascalmpl.library.lang.json.IO} -@deprecated{use writeJSON} +@synopsis{Maps any Rascal value to a JSON string} +@deprecated{use ((writeJSON))} public java str toJSON(value v); @javaClass{org.rascalmpl.library.lang.json.IO} -@deprecated{use asJSON} +@synopsis{Maps any Rascal value to a JSON string, optionally in compact form.} +@deprecated{use ((asJSON))} public java str toJSON(value v, bool compact); @javaClass{org.rascalmpl.library.lang.json.IO} -@deprecated{use readJSON} +@deprecated{use ((readJSON))} +@synopsis{Parses a JSON string and maps it to the requested type of Rascal value.} public java &T fromJSON(type[&T] typ, str src); @javaClass{org.rascalmpl.library.lang.json.IO}