[Bug]: Agent crashing when loading large project #4893
Labels
Comments
|
This seems quite important if it's consistently reproducible. |
|
@harish-coeff can you share the docker command you used? |
|
It worked always with a small project, for a big project it didn't work 4 out of 5 times. The one time it worked it took around 4-5 mins to load. docker run -it --pull=always \
-e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:main-nikolaik \
-e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
-v $WORKSPACE_BASE:/opt/workspace_base \
-v /var/run/docker.sock:/var/run/docker.sock \
-p 3000:3000 \
--add-host host.docker.internal:host-gateway \
--name openhands-app \
docker.all-hands.dev/all-hands-ai/openhands:mainThe logs for the runtime container when it failed. I don't see any errors. 2024-11-11 21:14:30 INFO: Started server process [8]
2024-11-11 21:14:30 INFO: Waiting for application startup.
2024-11-11 21:16:36 INFO: Application startup complete.
2024-11-11 21:16:36 INFO: Uvicorn running on http://0.0.0.0:30099 (Press CTRL+C to quit)
2024-11-11 21:16:32 [I 2024-11-11 15:46:32.216 KernelGatewayApp] Writing Jupyter server cookie secret to /root/.local/share/jupyter/runtime/jupyter_cookie_secret
2024-11-11 21:16:32 [I 2024-11-11 15:46:32.217 KernelGatewayApp] Jupyter Kernel Gateway 3.0.1 is available at http://0.0.0.0:46750
2024-11-11 21:16:33 [I 2024-11-11 15:46:33.399 KernelGatewayApp] Kernel started: 00907365-2090-4213-8c4c-43de5336d123
2024-11-11 21:16:33 [I 241111 15:46:33 web:2348] 201 POST /api/kernels (127.0.0.1) 856.20ms
2024-11-11 21:16:33 [W 2024-11-11 15:46:33.404 KernelGatewayApp] No session ID specified
2024-11-11 21:16:34 [I 241111 15:46:34 web:2348] 101 GET /api/kernels/00907365-2090-4213-8c4c-43de5336d123/channels (127.0.0.1) 679.60ms
2024-11-11 21:16:34 [I 2024-11-11 15:46:34.083 KernelGatewayApp] Connecting to kernel 00907365-2090-4213-8c4c-43de5336d123. |
|
can you grab the output of I don't think this should matter but you might need to increase the size of the disk allocated for docker |
|
Docker inspect for the runtime container {
"Id": "<container_id>",
"Created": "2024-11-11T16:02:25.931769843Z",
"Path": "/openhands/micromamba/bin/micromamba",
"Args": [
"run",
"-n",
"openhands",
"poetry",
"run",
"python",
"-u",
"-m",
"openhands.runtime.action_execution_server",
"37059",
"--working-dir",
"/workspace",
"--plugins",
"agent_skills",
"jupyter",
"--username",
"root",
"--user-id",
"0"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 97859,
"ExitCode": 0,
"Error": "",
"StartedAt": "2024-11-11T16:02:26.353286093Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "<image_id>",
"ResolvConfPath": "/var/lib/docker/containers/<container_id>/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/<container_id>/hostname",
"HostsPath": "/var/lib/docker/containers/<container_id>/hosts",
"LogPath": "/var/lib/docker/containers/<container_id>/<container_id>-json.log",
"Name": "/openhands-runtime-<uuid>",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"<local_path>:/workspace:rw"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "bridge",
"PortBindings": {
"37059/tcp": [
{
"HostIp": "",
"HostPort": "37059"
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"ConsoleSize": [
0,
0
],
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "private",
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DeviceRequests": null,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": null,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware",
"/sys/devices/virtual/powercap"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "<removed>",
"MergedDir": "/var/lib/docker/overlay2/<id>/merged",
"UpperDir": "/var/lib/docker/overlay2/<id>/diff",
"WorkDir": "/var/lib/docker/overlay2/<id>/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "<local_path>",
"Destination": "/workspace",
"Mode": "rw",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "<container_id_short>",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"37059/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"port=37059",
"PYTHONUNBUFFERED=1",
"PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"LANG=C.UTF-8",
"GPG_KEY=<removed>",
"PYTHON_VERSION=3.12.7",
"PYTHON_SHA256=<removed>",
"POETRY_HOME=/usr/local",
"POETRY_VIRTUALENVS_PATH=/openhands/poetry",
"MAMBA_ROOT_PREFIX=/openhands/micromamba"
],
"Cmd": [
"/openhands/micromamba/bin/micromamba",
"run",
"-n",
"openhands",
"poetry",
"run",
"python",
"-u",
"-m",
"openhands.runtime.action_execution_server",
"37059",
"--working-dir",
"/workspace",
"--plugins",
"agent_skills",
"jupyter",
"--username",
"root",
"--user-id",
"0"
],
"Image": "<image_name>",
"Volumes": {
"/workspace": {}
},
"WorkingDir": "/openhands/code",
"Entrypoint": null,
"OnBuild": null,
"Labels": {
"org.opencontainers.image.authors": "<removed>"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "<sandbox_id>",
"SandboxKey": "/var/run/docker/netns/<sandbox_id_short>",
"Ports": {
"37059/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "37059"
}
]
},
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "<endpoint_id>",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "<mac_address>",
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"MacAddress": "<mac_address>",
"NetworkID": "<network_id>",
"EndpointID": "<endpoint_id>",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"DriverOpts": null,
"DNSNames": null
}
}
}
}docker inspect for the app container {
"Id": "<container_id>",
"Created": "2024-11-11T16:02:10.834182627Z",
"Path": "/app/entrypoint.sh",
"Args": [
"uvicorn",
"openhands.server.listen:app",
"--host",
"0.0.0.0",
"--port",
"3000"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 97770,
"ExitCode": 0,
"Error": "",
"StartedAt": "2024-11-11T16:02:11.395694919Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "<image_id>",
"ResolvConfPath": "/var/lib/docker/containers/<container_id>/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/<container_id>/hostname",
"HostsPath": "/var/lib/docker/containers/<container_id>/hosts",
"LogPath": "/var/lib/docker/containers/<container_id>/<container_id>-json.log",
"Name": "/openhands-app",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"<local_path>:/opt/workspace_base",
"/var/run/docker.sock:/var/run/docker.sock"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "bridge",
"PortBindings": {
"3000/tcp": [
{
"HostIp": "",
"HostPort": "3000"
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"ConsoleSize": [
64,
210
],
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "private",
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": [
"host.docker.internal:host-gateway"
],
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": [],
"BlkioDeviceWriteBps": [],
"BlkioDeviceReadIOps": [],
"BlkioDeviceWriteIOps": [],
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DeviceRequests": null,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": null,
"PidsLimit": null,
"Ulimits": [],
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware",
"/sys/devices/virtual/powercap"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "<removed>",
"MergedDir": "/var/lib/docker/overlay2/<id>/merged",
"UpperDir": "/var/lib/docker/overlay2/<id>/diff",
"WorkDir": "/var/lib/docker/overlay2/<id>/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "<local_path>",
"Destination": "/opt/workspace_base",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/var/run/docker.sock",
"Destination": "/var/run/docker.sock",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "<container_id_short>",
"Domainname": "",
"User": "root",
"AttachStdin": true,
"AttachStdout": true,
"AttachStderr": true,
"ExposedPorts": {
"3000/tcp": {}
},
"Tty": true,
"OpenStdin": true,
"StdinOnce": true,
"Env": [
"SANDBOX_RUNTIME_CONTAINER_IMAGE=<image_name>",
"WORKSPACE_MOUNT_PATH=<local_path>",
"PATH=/app/.venv/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"LANG=C.UTF-8",
"GPG_KEY=<removed>",
"PYTHON_VERSION=3.12.3",
"PYTHON_PIP_VERSION=24.0",
"PYTHON_GET_PIP_URL=<removed>",
"PYTHON_GET_PIP_SHA256=<removed>",
"RUN_AS_OPENHANDS=true",
"OPENHANDS_USER_ID=42420",
"SANDBOX_LOCAL_RUNTIME_URL=http://host.docker.internal",
"USE_HOST_NETWORK=false",
"WORKSPACE_BASE=/opt/workspace_base",
"OPENHANDS_BUILD_VERSION=main",
"SANDBOX_USER_ID=0",
"VIRTUAL_ENV=/app/.venv",
"PYTHONPATH=/app"
],
"Cmd": [
"uvicorn",
"openhands.server.listen:app",
"--host",
"0.0.0.0",
"--port",
"3000"
],
"Image": "<image_name>",
"Volumes": null,
"WorkingDir": "/app",
"Entrypoint": [
"/app/entrypoint.sh"
],
"OnBuild": null,
"Labels": {}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "<sandbox_id>",
"SandboxKey": "/var/run/docker/netns/<sandbox_id_short>",
"Ports": {
"3000/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "3000"
}
]
},
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "<endpoint_id>",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "<mac_address>",
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"MacAddress": "<mac_address>",
"NetworkID": "<network_id>",
"EndpointID": "<endpoint_id>",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"DriverOpts": null,
"DNSNames": null
}
}
}
} |
|
Hmm OK so they're both still running, you're just seeing a stack trace in the app saying Are you able to |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Is there an existing issue for the same bug?
Describe the bug and reproduction steps
I am just trying the normal setup with local workspace flag. It worked for a small folder with 10 scripts. But when I tried with a production project (just changed the WORKSPACE_BASE from the previous working folder) with more than 2k files and more than 200 MB (ignoring all files which are in gitignore). The agent is crashing. Also, I hope openhands ignores the gitignore files (all the node modules, packages etc).
OpenHands Installation
Docker command in README
OpenHands Version
main
Operating System
MacOS
Logs, Errors, Screenshots, and Additional Context
The text was updated successfully, but these errors were encountered: