Skip to content

Commit

Permalink
Merge pull request #1862 from usethesource/json-serialization-improved
Browse files Browse the repository at this point in the history 
added optional storage of constructor names and datatype names while serializing to JSON. Wired this through the different webservers as well
  • Loading branch information
@jurgenvinju
jurgenvinju authored Nov 25, 2024
2 parents 5b9e319 + 1c11d5e commit 65610af
Show file tree
Hide file tree
Showing 9 changed files with 217 additions and 39 deletions.
2 changes: 1 addition & 1 deletion src/org/rascalmpl/library/Content.rsc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ which involves a handy, automatic, encoding of Rascal values into json values.
data Response
= response(Status status, str mimeType, map[str,str] header, str content)
| fileResponse(loc file, str mimeType, map[str,str] header)
| jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'")
| jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", bool explicitConstructorNames=false, bool explicitDataTypes=false)
;


Expand Down
19 changes: 12 additions & 7 deletions src/org/rascalmpl/library/lang/json/IO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,14 +100,16 @@ public IValue fromJSON(IValue type, IString src) {
}


public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins) {
public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) {
TypeStore store = new TypeStore();
Type start = new TypeReifier(values).valueToType((IConstructor) type, store);

try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) {
in.setLenient(lenient.getValue());
return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? loc : null)
.setCalendarFormat(dateTimeFormat.getValue())
.setExplicitConstructorNames(explicitConstructorNames.getValue())
.setExplicitDataTypes(explicitDataTypes.getValue())
.read(in, start);
}
catch (IOException e) {
Expand All @@ -119,14 +121,16 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat,
}
}

public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins) {
public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) {
TypeStore store = new TypeStore();
Type start = new TypeReifier(values).valueToType((IConstructor) type, store);

try (JsonReader in = new JsonReader(new StringReader(src.getValue()))) {
in.setLenient(lenient.getValue());
return new JsonValueReader(values, store, monitor, trackOrigins.getValue() ? URIUtil.rootLocation("unknown") : null)
.setCalendarFormat(dateTimeFormat.getValue())
.setExplicitConstructorNames(explicitConstructorNames.getValue())
.setExplicitDataTypes(explicitDataTypes.getValue())
.read(in, start);
}
catch (IOException e) {
Expand All @@ -137,7 +141,7 @@ public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool
}
}

public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) {
public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) {
try (JsonWriter out = new JsonWriter(new OutputStreamWriter(URIResolverRegistry.getInstance().getOutputStream(loc, false), Charset.forName("UTF8")))) {
if (indent.intValue() > 0) {
out.setIndent(" ".substring(0, indent.intValue() % 9));
Expand All @@ -148,13 +152,15 @@ public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations
.setDatesAsInt(dateTimeAsInt.getValue())
.setUnpackedLocations(unpackedLocations.getValue())
.setDropOrigins(dropOrigins.getValue())
.setExplicitConstructorNames(explicitConstructorNames.getValue())
.setExplicitDataTypes(explicitDataTypes.getValue())
.write(out, value);
} catch (IOException e) {
throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null);
}
}

public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins) {
public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFormat, IBool dateTimeAsInt, IInteger indent, IBool dropOrigins, IBool explicitConstructorNames, IBool explicitDataTypes) {
StringWriter string = new StringWriter();

try (JsonWriter out = new JsonWriter(string)) {
Expand All @@ -166,14 +172,13 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor
.setDatesAsInt(dateTimeAsInt.getValue())
.setUnpackedLocations(unpackedLocations.getValue())
.setDropOrigins(dropOrigins.getValue())
.setExplicitConstructorNames(explicitConstructorNames.getValue())
.setExplicitDataTypes(explicitDataTypes.getValue())
.write(out, value);

return values.string(string.toString());
} catch (IOException e) {
throw RuntimeExceptionFactory.io(values.string(e.getMessage()), null, null);
}
}



}
52 changes: 35 additions & 17 deletions src/org/rascalmpl/library/lang/json/IO.rsc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,23 @@
module lang::json::IO

@javaClass{org.rascalmpl.library.lang.json.IO}
@deprecated{
Use writeJSON
}
@synopsis{Maps any Rascal value to a JSON string}
@deprecated{use ((writeJSON))}
public java str toJSON(value v);

@javaClass{org.rascalmpl.library.lang.json.IO}
@deprecated{
Use asJSON
}
@synopsis{Maps any Rascal value to a JSON string, optionally in compact form.}
@deprecated{use ((asJSON))}
public java str toJSON(value v, bool compact);

@javaClass{org.rascalmpl.library.lang.json.IO}
@deprecated{
Use readJSON
}
@deprecated{use ((readJSON))}
@synopsis{Parses a JSON string and maps it to the requested type of Rascal value.}
public java &T fromJSON(type[&T] typ, str src);

@javaClass{org.rascalmpl.library.lang.json.IO}
@synopsis{reads JSON values from a stream
@synopsis{reads JSON values from a stream}
@description{
In general the translation behaves as follows:
* Objects translate to map[str,value] by default, unless a node is expected (properties are then translated to keyword fields)
* Arrays translate to lists by default, or to a set if that is expected or a tuple if that is expected. Arrays may also be interpreted as constructors or nodes (see below)
Expand All @@ -44,20 +42,40 @@ In general the translation behaves as follows:
* If num, int, real or rat are expected both strings and number values are mapped
* If loc is expected than strings which look like URI are parsed (containing :/) or a file:/// URI is build, or if an object is found each separate field of
a location object is read from the respective properties: { scheme : str, authority: str?, path: str?, fragment: str?, query: str?, offset: int, length: int, begin: [bl, bc], end: [el, ec]}}
java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false);
java &T readJSON(type[&T] expected, loc src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, bool explicitConstructorNames=false, bool explicitDataTypes=false);

@javaClass{org.rascalmpl.library.lang.json.IO}
@synopsis{parses JSON values from a string
In general the translation behaves as the same as for ((readJSON)).}
java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false);
java &T parseJSON(type[&T] expected, str src, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool lenient=false, bool trackOrigins=false, bool explicitConstructorNames=false, bool explicitDataTypes=false);

@javaClass{org.rascalmpl.library.lang.json.IO}
@synopsis{writes `val` to the location `target`}
@synopsis{Serializes a value as a JSON string and stream it}
@description{
If `dateTimeAsInt` is set to `true`, the dateTime values are converted to an int that represents the number of milliseconds from 1970-01-01T00:00Z.
If `indent` is set to a number greater than 0, the JSON file will be formatted with `indent` number of spaces as indentation.
This function tries to map Rascal values to JSON values in a natural way.
In particular it tries to create a value that has the same number of recursive levels,
such that one constructor maps to one object. The serialization is typically _lossy_ since
JSON values by default do not explicitly encode the class or constructor while Rascal data types do.
If you need the names of constructors or data-types in your result, then use the parameters:
* `explicitConstructorNames=true` will store the name of every constructor in a field `_constructor`
* `explicitDataTypes=true` will store the name of the ADT in a field called `_type`
The `dateTimeFormat` parameter dictates how `datetime` values will be printed.
The `unpackedLocations` parameter will produce an object with many fields for every property of a `loc` value, but
if set to false a `loc` will be printed as a string.
}
@pitfalls{
* It is understood that Rascal's number types have arbitrary precision, but this is not supported by the JSON writer.
As such when an `int` is printed that does not fit into a JVM `long`, there will be truncation to the lower 64 bits.
For `real` numbers that are larger than JVM's double you get "negative infinity" or "positive infinity" as a result.
}
java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true);
java void writeJSON(loc target, value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent=0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false);

@javaClass{org.rascalmpl.library.lang.json.IO}
java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true);
@synopsis{Serializes a value as a JSON string and stores it as a string}
@description{
This function uses `writeJSON` and stores the result in a string.
}
java str asJSON(value val, bool unpackedLocations=false, str dateTimeFormat="yyyy-MM-dd\'T\'HH:mm:ssZZZZZ", bool dateTimeAsInt=false, int indent = 0, bool dropOrigins=true, bool explicitConstructorNames=false, bool explicitDataTypes=false);
102 changes: 93 additions & 9 deletions src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.rascalmpl.debug.IRascalMonitor;
import org.rascalmpl.uri.URIUtil;
import io.usethesource.vallang.IInteger;
Expand Down Expand Up @@ -59,6 +58,8 @@ public class JsonValueReader {
private VarHandle posHandler;
private VarHandle lineHandler;
private VarHandle lineStartHandler;
private boolean explicitConstructorNames;
private boolean explicitDataTypes;

/**
* @param vf factory which will be used to construct values
Expand Down Expand Up @@ -105,6 +106,20 @@ protected SimpleDateFormat initialValue() {
return this;
}

public JsonValueReader setExplicitConstructorNames(boolean value) {
this.explicitConstructorNames = value;
return this;
}

public JsonValueReader setExplicitDataTypes(boolean value) {
this.explicitDataTypes = value;
if (value) {
this.explicitConstructorNames = true;
}
return this;
}


/**
* Read and validate a Json stream as an IValue
* @param in json stream
Expand Down Expand Up @@ -477,7 +492,6 @@ private int getCol() {
}
}


@Override
public IValue visitAbstractData(Type type) throws IOException {
if (in.peek() == JsonToken.STRING) {
Expand All @@ -493,18 +507,75 @@ public IValue visitAbstractData(Type type) throws IOException {
throw new IOException("no nullary constructor found for " + type);
}

assert in.peek() == JsonToken.BEGIN_OBJECT;
Set<Type> alternatives = null;

in.beginObject();
int startPos = getPos();
int startLine = getLine();
int startCol = getCol();

// use explicit information in the JSON to select and filter constructors from the TypeStore
// we expect always to have the field _constructor before _type.
if (explicitConstructorNames || explicitDataTypes) {
String consName = null;
String typeName = null; // this one is optional, and the order with cons is not defined.

String consLabel = in.nextName();

// first we read either a cons name or a type name
if (explicitConstructorNames && "_constructor".equals(consLabel)) {
consName = in.nextString();
}
else if (explicitDataTypes && "_type".equals(consLabel)) {
typeName = in.nextString();
}

// optionally read the second field
if (explicitDataTypes && typeName == null) {
// we've read a constructor name, but we still need a type name
consLabel = in.nextName();
if (explicitDataTypes && "_type".equals(consLabel)) {
typeName = in.nextString();
}
}
else if (explicitDataTypes && consName == null) {
// we've read type name, but we still need a constructor name
consLabel = in.nextName();
if (explicitDataTypes && "_constructor".equals(consLabel)) {
consName = in.nextString();
}
}

if (explicitDataTypes && typeName == null) {
throw new IOException("Missing a _type field: " + in.getPath());
}
else if (explicitConstructorNames && consName == null) {
throw new IOException("Missing a _constructor field: " + in.getPath());
}

if (typeName != null && consName != null) {
// first focus on the given type name
var dataType = TF.abstractDataType(store, typeName);
alternatives = store.lookupConstructor(dataType, consName);
}
else {
// we only have a constructor name
// lookup over all data types by constructor name
alternatives = store.lookupConstructors(consName);
}
}
else {
alternatives = store.lookupAlternatives(type);
}

Set<Type> alternatives = store.lookupAlternatives(type);
if (alternatives.size() > 1) {
monitor.warning("selecting arbitrary constructor for " + type, vf.sourceLocation(in.getPath()));
}
else if (alternatives.size() == 0) {
throw new IOException("No fitting constructor found for " + in.getPath());
}
Type cons = alternatives.iterator().next();

in.beginObject();
int startPos = getPos();
int startLine = getLine();
int startCol = getCol();

IValue[] args = new IValue[cons.getArity()];
Map<String,IValue> kwParams = new HashMap<>();
Expand Down Expand Up @@ -532,7 +603,20 @@ else if (cons.hasKeywordField(label, store)) {
}
}
else { // its a normal arg, pass its label to the child
throw new IOException("Unknown field " + label + ":" + in.getPath());
if (!explicitConstructorNames && "_constructor".equals(label)) {
// ignore additional _constructor fields.
in.nextString(); // skip the constructor value
continue;
}
else if (!explicitDataTypes && "_type".equals(label)) {
// ignore additional _type fields.
in.nextString(); // skip the type value
continue;
}
else {
// field label does not match data type definition
throw new IOException("Unknown field " + label + ":" + in.getPath());
}
}
}

Expand Down
25 changes: 24 additions & 1 deletion src/org/rascalmpl/library/lang/json/internal/JsonValueWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ public class JsonValueWriter {
private boolean datesAsInts = true;
private boolean unpackedLocations = false;
private boolean dropOrigins = true;
private boolean explicitConstructorNames = false;
private boolean explicitDataTypes;

public JsonValueWriter() {
setCalendarFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
Expand Down Expand Up @@ -78,6 +80,16 @@ public JsonValueWriter setDropOrigins(boolean setting) {
return this;
}

public JsonValueWriter setExplicitConstructorNames(boolean setting) {
this.explicitConstructorNames = setting;
return this;
}

public JsonValueWriter setExplicitDataTypes(boolean setting) {
this.explicitDataTypes = setting;
return this;
}

public void write(JsonWriter out, IValue value) throws IOException {
value.accept(new IValueVisitor<Void, IOException>() {

Expand Down Expand Up @@ -222,13 +234,24 @@ public Void visitNode(INode o) throws IOException {

@Override
public Void visitConstructor(IConstructor o) throws IOException {
if (o.getConstructorType().getArity() == 0 && !o.asWithKeywordParameters().hasParameters()) {
if (!explicitConstructorNames && !explicitDataTypes && o.getConstructorType().getArity() == 0 && !o.asWithKeywordParameters().hasParameters()) {
// enums!
out.value(o.getName());
return null;
}

out.beginObject();

if (explicitConstructorNames || explicitDataTypes) {
out.name("_constructor");
out.value(o.getName());
}

if (explicitDataTypes) {
out.name("_type");
out.value(o.getType().getName());
}

int i = 0;
for (IValue arg : o) {
out.name(o.getConstructorType().getFieldName(i));
Expand Down
Loading

0 comments on commit 65610af

Please sign in to comment.