Skip to content

Commit

Permalink
Added timeout detection on commands
Browse files Browse the repository at this point in the history 
Added timeout detection on commands
  • Loading branch information
@zhenghaven
zhenghaven authored Oct 19, 2023
2 parents b1816e2 + 56d1539 commit 4ce38d7
Show file tree
Hide file tree
Showing 4 changed files with 162 additions and 12 deletions.
67 changes: 57 additions & 10 deletions GradescopeGrader/Cmd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import os
import subprocess
import threading
import time

from typing import Union
Expand All @@ -22,32 +23,78 @@ def __init__(
cmd: Union[str, list],
workDir: str = os.getcwd(),
env: dict = os.environ,
timeout: Union[None, float] = None,
) -> None:
super(Cmd, self).__init__()

self.workDir = workDir
self.cmd = cmd
self.env = env
self.timeout = timeout

self.proc = None
self.stdout = None
self.stderr = None
self.returncode = None
self.returncode = None # None value by default
self.runtimeNS = None
self.hasTimedOut = False
self.timer = None

def Kill(self) -> None:
if self.proc:
self.proc.kill()
self.hasTimedOut = True

def StartKillTimer(self) -> None:
if self.timer is not None:
self.CancelKillTimer()

if self.timeout is not None:
self.hasTimedOut = False # reset
self.timer = threading.Timer(
interval=self.timeout,
function=self.Kill
)
self.timer.start()

def CancelKillTimer(self) -> None:
if self.timer:
self.timer.cancel()
self.timer = None

def Run(self) -> None:
startTime = time.time_ns()
with subprocess.Popen(
self.cmd,
cwd=self.workDir,
env=self.env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as proc:
self.stdout, self.stderr = proc.communicate()
self.returncode = proc.returncode

try:
with subprocess.Popen(
self.cmd,
cwd=self.workDir,
env=self.env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as proc:
self.proc = proc
self.StartKillTimer()
self.stdout, self.stderr = proc.communicate()
# update return code based on the process return code
# or -9 if the process is killed by timeout
self.returncode = proc.returncode if not self.hasTimedOut else -9
finally:
self.CancelKillTimer()
self.proc = None

endTime = time.time_ns()
self.runtimeNS = endTime - startTime

def GetRunTimeNS(self) -> int:
return self.runtimeNS

def GetRunTimeMS(self) -> float:
return self.GetRunTimeNS() / 1000 / 1000

def GetRunTime(self) -> float:
return self.GetRunTimeMS() / 1000

def __str__(self) -> str:
if isinstance(self.cmd, str):
return self.cmd
Expand Down
9 changes: 7 additions & 2 deletions GradescopeGrader/CmdAllOrNothingTest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def __init__(
self.preCmd = preCmd
self.postCmd = postCmd

# failed score and status by default
self.score = 0
self.status = 'failed'

Expand Down Expand Up @@ -66,6 +67,7 @@ def Run(self) -> None:
self.runtimeNS = self.cmd.runtimeNS

if self.cmd.returncode == 0:
# only update score and status when th return code is 0
self.score = self.maxScore
self.status = 'passed'

Expand All @@ -75,7 +77,7 @@ def Run(self) -> None:

def _GenOutput(self) -> Tuple[str, str]:
output = ''
output += 'Execution Time: {:.2f} ms\n'.format(self.runtimeNS / 1000000)
output += 'Execution Time: {:.2f} ms\n'.format(self.GetRunTimeMS())
output += 'Return Code: {}\n'.format(self.returncode)
output += '\n'

Expand All @@ -97,8 +99,11 @@ def _GenOutput(self) -> Tuple[str, str]:
def GetRunTimeNS(self) -> int:
return self.runtimeNS

def GetRunTimeMS(self) -> float:
return self.GetRunTimeNS() / 1000 / 1000

def GetRunTime(self) -> float:
return self.GetRunTimeNS() / 1000000
return self.GetRunTimeMS() / 1000

def GetResult(self) -> dict:
'''
Expand Down
62 changes: 62 additions & 0 deletions tests/unittest/Cmd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,65 @@ def test_fail_run(self):
self.assertEqual(inst.stderr, b'')
self.assertEqual(inst.returncode, 12)
self.assertGreater(inst.runtimeNS, 0)

def test_timeout_run_sleep(self):
cmd = [ sys.executable, '-c', 'import time; print("before sleep"); time.sleep(10); print("after sleep")' ]
inst = Cmd.Cmd(
workDir='/tmp',
cmd=cmd,
env={},
timeout=1.0,
)
inst.Run()
#self.assertEqual(inst.stdout, b'')
self.assertEqual(inst.stderr, b'')
self.assertEqual(inst.returncode, -9)
self.assertGreater(inst.GetRunTime(), 0.500) # 0.5s
self.assertLess(inst.GetRunTime(), 2.0) # 2s
self.assertGreater(inst.GetRunTimeMS(), 500.0) # 500ms
self.assertLess(inst.GetRunTimeMS(), 2000.0) # 2000ms
self.assertEqual(inst.hasTimedOut, True)

def test_timeout_run_loop(self):
cmd = [ sys.executable, '-c', 'i = 1\nwhile True:\n\ti += 1' ]
inst = Cmd.Cmd(
workDir='/tmp',
cmd=cmd,
env={},
timeout=1.0,
)
inst.Run()
self.assertEqual(inst.stdout, b'')
self.assertEqual(inst.stderr, b'')
self.assertEqual(inst.returncode, -9)
self.assertGreater(inst.GetRunTime(), 0.500) # 0.5s
self.assertLess(inst.GetRunTime(), 2.0) # 2s
self.assertGreater(inst.GetRunTimeMS(), 500.0) # 500ms
self.assertLess(inst.GetRunTimeMS(), 2000.0) # 2000ms
self.assertEqual(inst.hasTimedOut, True)

def test_runtime_500ms(self):
inst = Cmd.Cmd(
cmd=[sys.executable, '-c', 'import time; time.sleep(0.5); exit(123)']
)
inst.Run()
self.assertGreaterEqual(inst.returncode, 123)
self.assertGreater(inst.GetRunTimeNS(), 400000000)
self.assertLess( inst.GetRunTimeNS(), 600000000)
self.assertGreater(inst.GetRunTimeMS(), 400.0)
self.assertLess( inst.GetRunTimeMS(), 600.0)
self.assertGreater(inst.GetRunTime(), 0.4)
self.assertLess( inst.GetRunTime(), 0.6)

def test_runtime_2s(self):
inst = Cmd.Cmd(
cmd=[sys.executable, '-c', 'import time; time.sleep(2.0); exit(132)']
)
inst.Run()
self.assertGreaterEqual(inst.returncode, 132)
self.assertGreater(inst.GetRunTimeNS(), 1500000000)
self.assertLess( inst.GetRunTimeNS(), 2500000000)
self.assertGreater(inst.GetRunTimeMS(), 1500.0)
self.assertLess( inst.GetRunTimeMS(), 2500.0)
self.assertGreater(inst.GetRunTime(), 1.5)
self.assertLess( inst.GetRunTime(), 2.5)
36 changes: 36 additions & 0 deletions tests/unittest/CmdAllOrNothingTest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,42 @@ def test_serializable(self):
self.assertEqual(resJsonDict['output_format'], 'md')
self.assertEqual(res, resJsonDict)

def test_runtime_500ms(self):
okCmd = Cmd.Cmd(
cmd=[sys.executable, '-c', 'import time; time.sleep(0.5); exit(0)']
)
inst = CmdAllOrNothingTest(
testId='test_inst',
cmd=okCmd,
maxScore=123,
)
inst.Run()
self.assertGreaterEqual(inst.returncode, 0)
self.assertGreater(inst.GetRunTimeNS(), 400000000)
self.assertLess( inst.GetRunTimeNS(), 600000000)
self.assertGreater(inst.GetRunTimeMS(), 400.0)
self.assertLess( inst.GetRunTimeMS(), 600.0)
self.assertGreater(inst.GetRunTime(), 0.4)
self.assertLess( inst.GetRunTime(), 0.6)

def test_runtime_2s(self):
okCmd = Cmd.Cmd(
cmd=[sys.executable, '-c', 'import time; time.sleep(2.0); exit(0)']
)
inst = CmdAllOrNothingTest(
testId='test_inst',
cmd=okCmd,
maxScore=123,
)
inst.Run()
self.assertGreaterEqual(inst.returncode, 0)
self.assertGreater(inst.GetRunTimeNS(), 1500000000)
self.assertLess( inst.GetRunTimeNS(), 2500000000)
self.assertGreater(inst.GetRunTimeMS(), 1500.0)
self.assertLess( inst.GetRunTimeMS(), 2500.0)
self.assertGreater(inst.GetRunTime(), 1.5)
self.assertLess( inst.GetRunTime(), 2.5)

class TestGrader(unittest.TestCase):

def test_constructor(self):
Expand Down

0 comments on commit 4ce38d7

Please sign in to comment.