Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Experiment: log error/success on teardown and look at private _outcome #59

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Experiment: log error/success on teardown and look at private _outcome #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 36 additions & 14 deletions framework/test_cases/base_testcase.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,33 @@
"""Base test case used for xds test suites."""
import inspect
import traceback
from typing import Optional, Union
import unittest
from typing import Optional, TypeVar, Union
import unittest.case

from absl import logging
from absl.testing import absltest
from typing_extensions import override

# Aliases
_OutcomeType: TypeVar = unittest.case._Outcome # noqa


class BaseTestCase(absltest.TestCase):
# @override
@override
def tearDown(self) -> None:
if not (outcome := self._get_outcome()):
return
if outcome.success:
logging.info(
"----- PSM Test Case PASSED: %s -----\n", self.test_name
)
return

test_errors = [error for test, error in outcome.errors if test is self]
logging.error("----- PSM Test Case FAILED: %s -----", self.test_name)
self._log_test_errors(test_errors)

@override
def run(self, result: Optional[unittest.TestResult] = None) -> None:
super().run(result)
# TODO(sergiitk): should this method be returning result? See
Expand All @@ -40,17 +58,18 @@ def run(self, result: Optional[unittest.TestResult] = None) -> None:
)
# Assume one test case will only have one status.
if test_errors or test_failures:
total_errors = len(test_errors) + len(test_failures)
logging.error(
"----- PSM Test Case FAILED: %s -----", self.test_name
)
self._log_test_errors(test_errors, is_unexpected=True)
self._log_test_errors(test_failures)
logging.info(
"----- PSM Test Case %s Error Count: %s -----",
self.test_name,
total_errors,
)
pass
# total_errors = len(test_errors) + len(test_failures)
# logging.error(
# "----- PSM Test Case FAILED: %s -----", self.test_name
# )
# self._log_test_errors(test_errors, is_unexpected=True)
# self._log_test_errors(test_failures)
# logging.info(
# "----- PSM Test Case %s Error Count: %s -----",
# self.test_name,
# total_errors,
# )
elif test_unexpected_successes:
logging.error(
"----- PSM Test Case UNEXPECTEDLY SUCCEEDED: %s -----\n",
Expand All @@ -69,6 +88,9 @@ def run(self, result: Optional[unittest.TestResult] = None) -> None:
self.test_name,
)

def _get_outcome(self) -> Optional[_OutcomeType]:
return getattr(self, "_outcome", None)

@property
def test_name(self) -> str:
"""Pretty test name (details in the description).
Expand Down
2 changes: 2 additions & 0 deletions framework/xds_k8s_testcase.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -752,6 +752,8 @@ def initKubernetesClientRunner(self, **kwargs) -> KubernetesClientRunner:
raise NotImplementedError

def tearDown(self):
super().tearDown()

logger.info("----- TestMethod %s teardown -----", self.test_name)
logger.debug("Getting pods restart times")
client_restarts: int = 0
Expand Down
4 changes: 2 additions & 2 deletions tests/bootstrap_generator_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ def tearDownClass(cls):
super().tearDownClass()

def tearDown(self):
super().tearDown()
logger.info("----- TestMethod %s teardown -----", self.test_name)
retryer = retryers.constant_retryer(
wait_fixed=_timedelta(seconds=10),
Expand All @@ -133,7 +134,6 @@ def tearDown(self):
retryer(self._cleanup)
except retryers.RetryError:
logger.exception("Got error during teardown")
super().tearDown()

def _cleanup(self):
self.client_runner.cleanup(force=self.force_cleanup)
Expand Down Expand Up @@ -168,6 +168,7 @@ class BootstrapGeneratorServerTest(
test_server: XdsTestServer

def tearDown(self):
super().tearDown()
logger.info("----- TestMethod %s teardown -----", self.test_name)
retryer = retryers.constant_retryer(
wait_fixed=_timedelta(seconds=10),
Expand All @@ -178,7 +179,6 @@ def tearDown(self):
retryer(self._cleanup)
except retryers.RetryError:
logger.exception("Got error during teardown")
super().tearDown()

def _cleanup(self):
self.client_runner.cleanup(force=self.force_cleanup)
Expand Down
Loading