Skip to content
This repository has been archived by the owner on Jul 17, 2024. It is now read-only.

Commit

Permalink
fix: Stack effect for Python 3.11+ with, ignore missing forward refer…
Browse files Browse the repository at this point in the history
…ences in type hints, copy extra and function attributes (#68)

- Use object.__getattribute__ instead of getattr when reading
  attributes when copying objects.

- Functions are restored by looking up the function with the same
  qualified name

- Extra attributes (that don't correspond to any instance attributes)
  are copied to the Python object

- Use copy to create instances to play nice with complex Python
  classes

- Change PythonIterator to an interface and move its implementation
  to DelegatePythonIterator
  • Loading branch information
Christopher-Chianelli authored Jun 12, 2024
1 parent a573b12 commit 03dd634
Show file tree
Hide file tree
Showing 31 changed files with 427 additions and 136 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1141,6 +1141,9 @@ public static void writeInstructionsForOpcodes(FunctionMetadata functionMetadata
Label targetLabel = exceptionTableTargetLabelMap.computeIfAbsent(exceptionBlock.targetInstruction,
offset -> new Label());

if (exceptionBlock.blockStartInstructionInclusive > exceptionBlock.targetInstruction) {
return;
}
functionMetadata.methodVisitor.visitTryCatchBlock(startLabel, endLabel, targetLabel,
Type.getInternalName(PythonBaseException.class));
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
import ai.timefold.jpyinterpreter.types.PythonSlice;
import ai.timefold.jpyinterpreter.types.PythonString;
import ai.timefold.jpyinterpreter.types.PythonSuperObject;
import ai.timefold.jpyinterpreter.types.collections.PythonIterator;
import ai.timefold.jpyinterpreter.types.collections.DelegatePythonIterator;
import ai.timefold.jpyinterpreter.types.collections.PythonLikeDict;
import ai.timefold.jpyinterpreter.types.collections.PythonLikeList;
import ai.timefold.jpyinterpreter.types.collections.PythonLikeTuple;
Expand Down Expand Up @@ -753,7 +753,7 @@ public static PythonLikeObject enumerate(List<PythonLikeObject> positionalArgs,
final AtomicReference<PythonLikeObject> currentIndex = new AtomicReference(start);
final AtomicBoolean shouldCallNext = new AtomicBoolean(true);

return new PythonIterator(new Iterator<PythonLikeObject>() {
return new DelegatePythonIterator(new Iterator<PythonLikeObject>() {
@Override
public boolean hasNext() {
if (shouldCallNext.get()) {
Expand Down Expand Up @@ -790,7 +790,7 @@ public PythonLikeObject next() {
});
}

public static PythonIterator filter(List<PythonLikeObject> positionalArgs,
public static DelegatePythonIterator filter(List<PythonLikeObject> positionalArgs,
Map<PythonString, PythonLikeObject> keywordArgs, PythonLikeObject instance) {
PythonLikeObject function;
PythonLikeObject iterable;
Expand Down Expand Up @@ -835,7 +835,7 @@ public static PythonIterator filter(List<PythonLikeObject> positionalArgs,
predicate = (PythonLikeFunction) function;
}

return new PythonIterator(StreamSupport.stream(
return new DelegatePythonIterator(StreamSupport.stream(
Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED),
false)
.filter(element -> PythonBoolean
Expand Down Expand Up @@ -1088,7 +1088,7 @@ public static PythonLikeDict locals(List<PythonLikeObject> positionalArgs,
throw new ValueError("builtin locals() is not supported when executed in Java bytecode");
}

public static PythonIterator map(List<PythonLikeObject> positionalArgs,
public static DelegatePythonIterator map(List<PythonLikeObject> positionalArgs,
Map<PythonString, PythonLikeObject> keywordArgs, PythonLikeObject instance) {
PythonLikeFunction function;
List<PythonLikeObject> iterableList = new ArrayList<>();
Expand Down Expand Up @@ -1138,7 +1138,7 @@ public List<PythonLikeObject> next() {
}
};

return new PythonIterator(StreamSupport.stream(
return new DelegatePythonIterator(StreamSupport.stream(
Spliterators.spliteratorUnknownSize(iteratorIterator, Spliterator.ORDERED),
false)
.map(element -> function.$call(element, Map.of(), null))
Expand Down Expand Up @@ -1395,7 +1395,7 @@ public PythonLikeObject next() {
}
};

return new PythonIterator(reversedIterator);
return new DelegatePythonIterator(reversedIterator);
}

throw new ValueError(sequenceType + " does not has a __reversed__ method and does not implement the Sequence protocol");
Expand Down Expand Up @@ -1569,7 +1569,7 @@ public static PythonLikeObject vars(List<PythonLikeObject> positionalArgs,
return positionalArgs.get(0).$getAttributeOrError("__dict__");
}

public static PythonIterator zip(List<PythonLikeObject> positionalArgs,
public static DelegatePythonIterator zip(List<PythonLikeObject> positionalArgs,
Map<PythonString, PythonLikeObject> keywordArgs, PythonLikeObject instance) {
List<PythonLikeObject> iterableList = positionalArgs;
boolean isStrict = false;
Expand All @@ -1596,7 +1596,7 @@ public static PythonIterator zip(List<PythonLikeObject> positionalArgs,

if (iteratorList.isEmpty()) {
// Return an empty iterator if there are no iterators
return new PythonIterator(iteratorList.iterator());
return new DelegatePythonIterator(iteratorList.iterator());
}

Iterator<List<PythonLikeObject>> iteratorIterator = new Iterator<List<PythonLikeObject>>() {
Expand Down Expand Up @@ -1633,7 +1633,7 @@ public List<PythonLikeObject> next() {
}
};

return new PythonIterator(iteratorIterator);
return new DelegatePythonIterator(iteratorIterator);
}

public static PythonLikeFunction importFunction(PythonInterpreter pythonInterpreter) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ public static void startExceptOrFinally(FunctionMetadata functionMetadata, Stack
}
}

public static void startWith(int jumpTarget, FunctionMetadata functionMetadata,
public static void setupWith(int jumpTarget, FunctionMetadata functionMetadata,
StackMetadata stackMetadata) {
MethodVisitor methodVisitor = functionMetadata.methodVisitor;

Expand Down Expand Up @@ -301,6 +301,50 @@ public static void startWith(int jumpTarget, FunctionMetadata functionMetadata,
// cannot free, since try block store stack in locals => freeing enterResult messes up indexing of locals
}

public static void beforeWith(FunctionMetadata functionMetadata,
StackMetadata stackMetadata) {
MethodVisitor methodVisitor = functionMetadata.methodVisitor;

methodVisitor.visitInsn(Opcodes.DUP); // duplicate context_manager twice; need one for __enter__, two for __exit__
methodVisitor.visitInsn(Opcodes.DUP);

// First load the method __exit__
methodVisitor.visitMethodInsn(Opcodes.INVOKEINTERFACE, Type.getInternalName(PythonLikeObject.class),
"$getType", Type.getMethodDescriptor(Type.getType(PythonLikeType.class)),
true);
methodVisitor.visitLdcInsn("__exit__");
methodVisitor.visitMethodInsn(Opcodes.INVOKEINTERFACE, Type.getInternalName(PythonLikeObject.class),
"$getAttributeOrError",
Type.getMethodDescriptor(Type.getType(PythonLikeObject.class), Type.getType(String.class)),
true);
methodVisitor.visitTypeInsn(Opcodes.CHECKCAST, Type.getInternalName(PythonLikeFunction.class));

// bind it to the context_manager
methodVisitor.visitTypeInsn(Opcodes.NEW, Type.getInternalName(BoundPythonLikeFunction.class));
methodVisitor.visitInsn(Opcodes.DUP_X2);
methodVisitor.visitInsn(Opcodes.DUP_X2);
methodVisitor.visitInsn(Opcodes.POP);
methodVisitor.visitMethodInsn(Opcodes.INVOKESPECIAL, Type.getInternalName(BoundPythonLikeFunction.class),
"<init>", Type.getMethodDescriptor(Type.VOID_TYPE, Type.getType(PythonLikeObject.class),
Type.getType(PythonLikeFunction.class)),
false);

// Swap __exit__ method with duplicated context_manager
methodVisitor.visitInsn(Opcodes.SWAP);

// Call __enter__
DunderOperatorImplementor.unaryOperator(methodVisitor, stackMetadata, PythonUnaryOperator.ENTER);

int enterResult = stackMetadata.localVariableHelper.newLocal();

// store enter result in temp, so it does not get saved in try block
stackMetadata.localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), enterResult);

// Push enter result back to the stack
stackMetadata.localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), enterResult);
// cannot free, since try block store stack in locals => freeing enterResult messes up indexing of locals
}

public static void startExceptBlock(FunctionMetadata functionMetadata, StackMetadata stackMetadata,
ExceptionBlock exceptionBlock) {
// In Python 3.11 and above, the stack here is
Expand Down Expand Up @@ -362,12 +406,18 @@ public static void handleExceptionInWith(FunctionMetadata functionMetadata, Stac
int instruction = localVariableHelper.newLocal();
int exitFunction = localVariableHelper.newLocal();

localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exception);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exceptionArgs);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), traceback);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), label);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), stackSize);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), instruction);
if (functionMetadata.pythonCompiledFunction.pythonVersion.isBefore(PythonVersion.PYTHON_3_11)) {
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exception);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exceptionArgs);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), traceback);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), label);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), stackSize);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), instruction);
} else {
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exception);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exceptionArgs);
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), traceback);
}
localVariableHelper.writeTemp(methodVisitor, Type.getType(PythonLikeObject.class), exitFunction);

// load exitFunction
Expand Down Expand Up @@ -424,14 +474,16 @@ public static void handleExceptionInWith(FunctionMetadata functionMetadata, Stac
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), exitFunction);
methodVisitor.visitInsn(Opcodes.SWAP);

localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), instruction);
methodVisitor.visitInsn(Opcodes.SWAP);
if (functionMetadata.pythonCompiledFunction.pythonVersion.isBefore(PythonVersion.PYTHON_3_11)) {
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), instruction);
methodVisitor.visitInsn(Opcodes.SWAP);

localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), stackSize);
methodVisitor.visitInsn(Opcodes.SWAP);
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), stackSize);
methodVisitor.visitInsn(Opcodes.SWAP);

localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), label);
methodVisitor.visitInsn(Opcodes.SWAP);
localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), label);
methodVisitor.visitInsn(Opcodes.SWAP);
}

localVariableHelper.readTemp(methodVisitor, Type.getType(PythonLikeObject.class), traceback);
methodVisitor.visitInsn(Opcodes.SWAP);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import ai.timefold.jpyinterpreter.types.PythonLikeType;
import ai.timefold.jpyinterpreter.types.PythonNone;
import ai.timefold.jpyinterpreter.types.PythonString;
import ai.timefold.jpyinterpreter.types.collections.DelegatePythonIterator;
import ai.timefold.jpyinterpreter.types.collections.PythonIterator;
import ai.timefold.jpyinterpreter.types.collections.PythonLikeDict;
import ai.timefold.jpyinterpreter.types.collections.PythonLikeFrozenSet;
Expand Down Expand Up @@ -94,7 +95,7 @@ public static PythonLikeObject wrapJavaObject(Object object, Map<Object, PythonL
}

if (object instanceof Iterator) {
return new PythonIterator((Iterator) object);
return new DelegatePythonIterator<>((Iterator) object);
}

if (object instanceof List) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import ai.timefold.jpyinterpreter.FunctionMetadata;
import ai.timefold.jpyinterpreter.LocalVariableHelper;
import ai.timefold.jpyinterpreter.PythonLikeObject;
import ai.timefold.jpyinterpreter.PythonVersion;
import ai.timefold.jpyinterpreter.StackMetadata;
import ai.timefold.jpyinterpreter.ValueSourceInfo;

Expand Down Expand Up @@ -296,12 +297,22 @@ public static void restoreExceptionTableStack(FunctionMetadata functionMetadata,
LocalVariableHelper localVariableHelper = stackMetadata.localVariableHelper;

localVariableHelper.readExceptionTableTargetStack(methodVisitor, exceptionBlock.getTargetInstruction());

for (int i = 0; i < exceptionBlock.getStackDepth(); i++) {
methodVisitor.visitInsn(Opcodes.DUP);
methodVisitor.visitLdcInsn(i);
methodVisitor.visitInsn(Opcodes.AALOAD);
methodVisitor.visitTypeInsn(Opcodes.CHECKCAST,
stackMetadata.getTypeAtStackIndex(exceptionBlock.getStackDepth() - i - 1).getJavaTypeInternalName());

if (functionMetadata.pythonCompiledFunction.pythonVersion.isBefore(PythonVersion.PYTHON_3_11) ||
functionMetadata.pythonCompiledFunction.pythonVersion.isAtLeast(PythonVersion.PYTHON_3_12)) {
// Not a 3.11 python version
methodVisitor.visitTypeInsn(Opcodes.CHECKCAST,
stackMetadata.getTypeAtStackIndex(exceptionBlock.getStackDepth() - i - 1).getJavaTypeInternalName());
} else {
// A 3.11 python version
methodVisitor.visitTypeInsn(Opcodes.CHECKCAST,
Type.getInternalName(PythonLikeObject.class));
}
methodVisitor.visitInsn(Opcodes.SWAP);
}
methodVisitor.visitInsn(Opcodes.POP);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import ai.timefold.jpyinterpreter.PythonBytecodeInstruction;
import ai.timefold.jpyinterpreter.PythonVersion;
import ai.timefold.jpyinterpreter.opcodes.Opcode;
import ai.timefold.jpyinterpreter.opcodes.exceptions.BeforeWithOpcode;
import ai.timefold.jpyinterpreter.opcodes.exceptions.CheckExcMatchOpcode;
import ai.timefold.jpyinterpreter.opcodes.exceptions.CleanupThrowOpcode;
import ai.timefold.jpyinterpreter.opcodes.exceptions.LoadAssertionErrorOpcode;
Expand Down Expand Up @@ -64,6 +65,7 @@ public enum ExceptionOpDescriptor implements OpcodeDescriptor {
WITH_EXCEPT_START(WithExceptStartOpcode::new),
SETUP_FINALLY(SetupFinallyOpcode::new, JumpUtils::getRelativeTarget),

BEFORE_WITH(BeforeWithOpcode::new),
SETUP_WITH(SetupWithOpcode::new, JumpUtils::getRelativeTarget),
CLEANUP_THROW(CleanupThrowOpcode::new);

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package ai.timefold.jpyinterpreter.opcodes.exceptions;

import ai.timefold.jpyinterpreter.FunctionMetadata;
import ai.timefold.jpyinterpreter.PythonBytecodeInstruction;
import ai.timefold.jpyinterpreter.StackMetadata;
import ai.timefold.jpyinterpreter.ValueSourceInfo;
import ai.timefold.jpyinterpreter.implementors.ExceptionImplementor;
import ai.timefold.jpyinterpreter.opcodes.AbstractOpcode;
import ai.timefold.jpyinterpreter.types.BuiltinTypes;
import ai.timefold.jpyinterpreter.types.PythonLikeFunction;

public class BeforeWithOpcode extends AbstractOpcode {

public BeforeWithOpcode(PythonBytecodeInstruction instruction) {
super(instruction);
}

@Override
protected StackMetadata getStackMetadataAfterInstruction(FunctionMetadata functionMetadata, StackMetadata stackMetadata) {
return stackMetadata
.pop()
.push(ValueSourceInfo.of(this, PythonLikeFunction.getFunctionType(), stackMetadata.getTOSValueSource()))
.push(ValueSourceInfo.of(this, BuiltinTypes.BASE_TYPE, stackMetadata.getTOSValueSource()));
}

@Override
public void implement(FunctionMetadata functionMetadata, StackMetadata stackMetadata) {
ExceptionImplementor.beforeWith(functionMetadata, stackMetadata);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,6 @@ public List<StackMetadata> getStackMetadataAfterInstructionForBranches(FunctionM

@Override
public void implement(FunctionMetadata functionMetadata, StackMetadata stackMetadata) {
ExceptionImplementor.startWith(jumpTarget, functionMetadata, stackMetadata);
ExceptionImplementor.setupWith(jumpTarget, functionMetadata, stackMetadata);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import ai.timefold.jpyinterpreter.FunctionMetadata;
import ai.timefold.jpyinterpreter.PythonBytecodeInstruction;
import ai.timefold.jpyinterpreter.PythonVersion;
import ai.timefold.jpyinterpreter.StackMetadata;
import ai.timefold.jpyinterpreter.ValueSourceInfo;
import ai.timefold.jpyinterpreter.implementors.ExceptionImplementor;
Expand All @@ -16,7 +17,10 @@ public WithExceptStartOpcode(PythonBytecodeInstruction instruction) {

@Override
protected StackMetadata getStackMetadataAfterInstruction(FunctionMetadata functionMetadata, StackMetadata stackMetadata) {
// TODO: this might need updating to handle Python 3.11
if (functionMetadata.pythonCompiledFunction.pythonVersion.isAtLeast(PythonVersion.PYTHON_3_11)) {
return stackMetadata
.push(ValueSourceInfo.of(this, BuiltinTypes.BASE_TYPE, stackMetadata.getValueSourceForStackIndex(1)));
}
return stackMetadata
.push(ValueSourceInfo.of(this, BuiltinTypes.BASE_TYPE, stackMetadata.getValueSourceForStackIndex(6)));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ public void setAttribute(String attributeName, PythonLikeObject value) {
__dir__.put(attributeName, value);
}

public Map<String, PythonLikeObject> getExtraAttributeMap() {
return __dir__;
}

@Override
public String toString() {
return $method$__str__().toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.util.List;
import java.util.Map;

import ai.timefold.jpyinterpreter.CPythonBackedPythonInterpreter;
import ai.timefold.jpyinterpreter.PythonInterpreter;
import ai.timefold.jpyinterpreter.PythonLikeObject;
import ai.timefold.jpyinterpreter.types.numeric.PythonInteger;
Expand Down Expand Up @@ -78,6 +79,10 @@ public CPythonBackedPythonLikeObject(PythonInterpreter interpreter,
}

public void $writeFieldsToCPythonReference(OpaquePythonReference cloneMap) {
for (var attributeEntry : getExtraAttributeMap().entrySet()) {
CPythonBackedPythonInterpreter.setAttributeOnPythonReference($cpythonReference, cloneMap, attributeEntry.getKey(),
attributeEntry.getValue());
}
}

@Override
Expand Down
Loading

0 comments on commit 03dd634

Please sign in to comment.