From 56d15392fc838e0ffd227038efa3def14f62bb24 Mon Sep 17 00:00:00 2001 From: Haofan Zheng Date: Thu, 19 Oct 2023 14:10:56 -0700 Subject: [PATCH] Added timeout detection on commands --- GradescopeGrader/Cmd.py | 67 +++++++++++++++++++++---- GradescopeGrader/CmdAllOrNothingTest.py | 9 +++- tests/unittest/Cmd.py | 62 +++++++++++++++++++++++ tests/unittest/CmdAllOrNothingTest.py | 36 +++++++++++++ 4 files changed, 162 insertions(+), 12 deletions(-) diff --git a/GradescopeGrader/Cmd.py b/GradescopeGrader/Cmd.py index f98a3ec..ccfced4 100644 --- a/GradescopeGrader/Cmd.py +++ b/GradescopeGrader/Cmd.py @@ -10,6 +10,7 @@ import os import subprocess +import threading import time from typing import Union @@ -22,32 +23,78 @@ def __init__( cmd: Union[str, list], workDir: str = os.getcwd(), env: dict = os.environ, + timeout: Union[None, float] = None, ) -> None: super(Cmd, self).__init__() self.workDir = workDir self.cmd = cmd self.env = env + self.timeout = timeout + self.proc = None self.stdout = None self.stderr = None - self.returncode = None + self.returncode = None # None value by default self.runtimeNS = None + self.hasTimedOut = False + self.timer = None + + def Kill(self) -> None: + if self.proc: + self.proc.kill() + self.hasTimedOut = True + + def StartKillTimer(self) -> None: + if self.timer is not None: + self.CancelKillTimer() + + if self.timeout is not None: + self.hasTimedOut = False # reset + self.timer = threading.Timer( + interval=self.timeout, + function=self.Kill + ) + self.timer.start() + + def CancelKillTimer(self) -> None: + if self.timer: + self.timer.cancel() + self.timer = None def Run(self) -> None: startTime = time.time_ns() - with subprocess.Popen( - self.cmd, - cwd=self.workDir, - env=self.env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as proc: - self.stdout, self.stderr = proc.communicate() - self.returncode = proc.returncode + + try: + with subprocess.Popen( + self.cmd, + cwd=self.workDir, + env=self.env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as proc: + self.proc = proc + self.StartKillTimer() + self.stdout, self.stderr = proc.communicate() + # update return code based on the process return code + # or -9 if the process is killed by timeout + self.returncode = proc.returncode if not self.hasTimedOut else -9 + finally: + self.CancelKillTimer() + self.proc = None + endTime = time.time_ns() self.runtimeNS = endTime - startTime + def GetRunTimeNS(self) -> int: + return self.runtimeNS + + def GetRunTimeMS(self) -> float: + return self.GetRunTimeNS() / 1000 / 1000 + + def GetRunTime(self) -> float: + return self.GetRunTimeMS() / 1000 + def __str__(self) -> str: if isinstance(self.cmd, str): return self.cmd diff --git a/GradescopeGrader/CmdAllOrNothingTest.py b/GradescopeGrader/CmdAllOrNothingTest.py index cfb8f63..6e33517 100644 --- a/GradescopeGrader/CmdAllOrNothingTest.py +++ b/GradescopeGrader/CmdAllOrNothingTest.py @@ -35,6 +35,7 @@ def __init__( self.preCmd = preCmd self.postCmd = postCmd + # failed score and status by default self.score = 0 self.status = 'failed' @@ -66,6 +67,7 @@ def Run(self) -> None: self.runtimeNS = self.cmd.runtimeNS if self.cmd.returncode == 0: + # only update score and status when th return code is 0 self.score = self.maxScore self.status = 'passed' @@ -75,7 +77,7 @@ def Run(self) -> None: def _GenOutput(self) -> Tuple[str, str]: output = '' - output += 'Execution Time: {:.2f} ms\n'.format(self.runtimeNS / 1000000) + output += 'Execution Time: {:.2f} ms\n'.format(self.GetRunTimeMS()) output += 'Return Code: {}\n'.format(self.returncode) output += '\n' @@ -97,8 +99,11 @@ def _GenOutput(self) -> Tuple[str, str]: def GetRunTimeNS(self) -> int: return self.runtimeNS + def GetRunTimeMS(self) -> float: + return self.GetRunTimeNS() / 1000 / 1000 + def GetRunTime(self) -> float: - return self.GetRunTimeNS() / 1000000 + return self.GetRunTimeMS() / 1000 def GetResult(self) -> dict: ''' diff --git a/tests/unittest/Cmd.py b/tests/unittest/Cmd.py index fe9460d..ec00828 100644 --- a/tests/unittest/Cmd.py +++ b/tests/unittest/Cmd.py @@ -61,3 +61,65 @@ def test_fail_run(self): self.assertEqual(inst.stderr, b'') self.assertEqual(inst.returncode, 12) self.assertGreater(inst.runtimeNS, 0) + + def test_timeout_run_sleep(self): + cmd = [ sys.executable, '-c', 'import time; print("before sleep"); time.sleep(10); print("after sleep")' ] + inst = Cmd.Cmd( + workDir='/tmp', + cmd=cmd, + env={}, + timeout=1.0, + ) + inst.Run() + #self.assertEqual(inst.stdout, b'') + self.assertEqual(inst.stderr, b'') + self.assertEqual(inst.returncode, -9) + self.assertGreater(inst.GetRunTime(), 0.500) # 0.5s + self.assertLess(inst.GetRunTime(), 2.0) # 2s + self.assertGreater(inst.GetRunTimeMS(), 500.0) # 500ms + self.assertLess(inst.GetRunTimeMS(), 2000.0) # 2000ms + self.assertEqual(inst.hasTimedOut, True) + + def test_timeout_run_loop(self): + cmd = [ sys.executable, '-c', 'i = 1\nwhile True:\n\ti += 1' ] + inst = Cmd.Cmd( + workDir='/tmp', + cmd=cmd, + env={}, + timeout=1.0, + ) + inst.Run() + self.assertEqual(inst.stdout, b'') + self.assertEqual(inst.stderr, b'') + self.assertEqual(inst.returncode, -9) + self.assertGreater(inst.GetRunTime(), 0.500) # 0.5s + self.assertLess(inst.GetRunTime(), 2.0) # 2s + self.assertGreater(inst.GetRunTimeMS(), 500.0) # 500ms + self.assertLess(inst.GetRunTimeMS(), 2000.0) # 2000ms + self.assertEqual(inst.hasTimedOut, True) + + def test_runtime_500ms(self): + inst = Cmd.Cmd( + cmd=[sys.executable, '-c', 'import time; time.sleep(0.5); exit(123)'] + ) + inst.Run() + self.assertGreaterEqual(inst.returncode, 123) + self.assertGreater(inst.GetRunTimeNS(), 400000000) + self.assertLess( inst.GetRunTimeNS(), 600000000) + self.assertGreater(inst.GetRunTimeMS(), 400.0) + self.assertLess( inst.GetRunTimeMS(), 600.0) + self.assertGreater(inst.GetRunTime(), 0.4) + self.assertLess( inst.GetRunTime(), 0.6) + + def test_runtime_2s(self): + inst = Cmd.Cmd( + cmd=[sys.executable, '-c', 'import time; time.sleep(2.0); exit(132)'] + ) + inst.Run() + self.assertGreaterEqual(inst.returncode, 132) + self.assertGreater(inst.GetRunTimeNS(), 1500000000) + self.assertLess( inst.GetRunTimeNS(), 2500000000) + self.assertGreater(inst.GetRunTimeMS(), 1500.0) + self.assertLess( inst.GetRunTimeMS(), 2500.0) + self.assertGreater(inst.GetRunTime(), 1.5) + self.assertLess( inst.GetRunTime(), 2.5) diff --git a/tests/unittest/CmdAllOrNothingTest.py b/tests/unittest/CmdAllOrNothingTest.py index 57249e7..9edb221 100644 --- a/tests/unittest/CmdAllOrNothingTest.py +++ b/tests/unittest/CmdAllOrNothingTest.py @@ -277,6 +277,42 @@ def test_serializable(self): self.assertEqual(resJsonDict['output_format'], 'md') self.assertEqual(res, resJsonDict) + def test_runtime_500ms(self): + okCmd = Cmd.Cmd( + cmd=[sys.executable, '-c', 'import time; time.sleep(0.5); exit(0)'] + ) + inst = CmdAllOrNothingTest( + testId='test_inst', + cmd=okCmd, + maxScore=123, + ) + inst.Run() + self.assertGreaterEqual(inst.returncode, 0) + self.assertGreater(inst.GetRunTimeNS(), 400000000) + self.assertLess( inst.GetRunTimeNS(), 600000000) + self.assertGreater(inst.GetRunTimeMS(), 400.0) + self.assertLess( inst.GetRunTimeMS(), 600.0) + self.assertGreater(inst.GetRunTime(), 0.4) + self.assertLess( inst.GetRunTime(), 0.6) + + def test_runtime_2s(self): + okCmd = Cmd.Cmd( + cmd=[sys.executable, '-c', 'import time; time.sleep(2.0); exit(0)'] + ) + inst = CmdAllOrNothingTest( + testId='test_inst', + cmd=okCmd, + maxScore=123, + ) + inst.Run() + self.assertGreaterEqual(inst.returncode, 0) + self.assertGreater(inst.GetRunTimeNS(), 1500000000) + self.assertLess( inst.GetRunTimeNS(), 2500000000) + self.assertGreater(inst.GetRunTimeMS(), 1500.0) + self.assertLess( inst.GetRunTimeMS(), 2500.0) + self.assertGreater(inst.GetRunTime(), 1.5) + self.assertLess( inst.GetRunTime(), 2.5) + class TestGrader(unittest.TestCase): def test_constructor(self):