improve timeout handling

pepper/cli.py

@@ -139,6 +139,14 @@ def parse(self):
 
         self.options, self.args = self.parser.parse_args()
 
+        # set up logging
+        rootLogger = logging.getLogger(name=None)
+        rootLogger.setLevel(max(logging.ERROR - (self.options.verbose * 10), 1))
+        formatter = logging.Formatter('%(levelname)s %(asctime)s %(module)s: %(message)s')
+        console = logging.StreamHandler()
+        console.setFormatter(formatter)
+        rootLogger.addHandler(console)
+
         option_names = ["fail_any", "fail_any_none", "fail_all", "fail_all_none"]
         toggled_options = [name for name in option_names if getattr(self.options, name)]
         if len(toggled_options) > 1:
@@ -498,13 +506,15 @@ def parse_cmd(self, api):
             if len(args) < 2:
                 self.parser.error("Command or target not specified")
 
+            low['full_return'] = True
             low['tgt_type'] = self.options.expr_form
             low['tgt'] = args.pop(0)
             low['fun'] = args.pop(0)
             low['batch'] = self.options.batch
             low['arg'] = args
         elif client.startswith('runner'):
             low['fun'] = args.pop(0)
+            low['full_return'] = True
             # post https://github.com/saltstack/salt/pull/50124, kwargs can be
             # passed as is in foo=bar form, splitting and deserializing will
             # happen in salt-api. additionally, the presence of salt-version header
@@ -529,11 +539,12 @@ def parse_cmd(self, api):
             else:
                 for arg in args:
                     if '=' in arg:
+                        low.setdefault('kwarg', {})
                         key, value = arg.split('=', 1)
                         try:
-                            low[key] = json.loads(value)
+                            low['kwarg'][key] = json.loads(value)
                         except JSONDecodeError:
-                            low[key] = value
+                            low['kwarg'][key] = value
                     else:
                         low.setdefault('arg', []).append(arg)
         elif client.startswith('ssh'):
@@ -655,24 +666,16 @@ def run(self):
         '''
         Parse all arguments and call salt-api
         '''
-        # set up logging
-        rootLogger = logging.getLogger(name=None)
-        rootLogger.addHandler(logging.StreamHandler())
-        rootLogger.setLevel(max(logging.ERROR - (self.options.verbose * 10), 1))
-
         api = pepper.Pepper(
             self.parse_url(),
             debug_http=self.options.debug_http,
-            ignore_ssl_errors=self.options.ignore_ssl_certificate_errors)
+            ignore_ssl_errors=self.options.ignore_ssl_certificate_errors,
+            timeout=self.options.timeout)
 
         self.login(api)
 
         load = self.parse_cmd(api)
 
-        for entry in load:
-            if not entry.get('client', '').startswith('wheel'):
-                entry['full_return'] = True
-
         if self.options.fail_if_minions_dont_respond:
             for exit_code, ret in self.poll_for_returns(api, load):  # pragma: no cover
                 yield exit_code, json.dumps(ret, sort_keys=True, indent=4)

pepper/libpepper.py

@@ -57,7 +57,7 @@ class Pepper(object):
               u'ms-4': True}]}
 
     '''
-    def __init__(self, api_url='https://localhost:8000', debug_http=False, ignore_ssl_errors=False):
+    def __init__(self, api_url='https://localhost:8000', debug_http=False, ignore_ssl_errors=False, timeout=None):
         '''
         Initialize the class with the URL of the API
 
@@ -81,6 +81,7 @@ def __init__(self, api_url='https://localhost:8000', debug_http=False, ignore_ss
         self._ssl_verify = not ignore_ssl_errors
         self.auth = {}
         self.salt_version = None
+        self.timeout = timeout
 
     def req_stream(self, path):
         '''
@@ -224,11 +225,13 @@ def req(self, path, data=None):
 
         # Send request
         try:
+            con_kwargs = {}
             if not (self._ssl_verify):
                 con = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-                f = urlopen(req, context=con)
-            else:
-                f = urlopen(req)
+                con_kwargs['context'] = con
+            #if self.timeout:
+            #    con_kwargs['timeout'] = self.timeout + 5 # throw a bit of buffer for upstream lag
+            f = urlopen(req, **con_kwargs)
             content = f.read().decode('utf-8')
             if (self.debug_http):
                 logger.debug('Response: %s', content)
@@ -283,6 +286,8 @@ def req_requests(self, path, data=None):
                   'auth': auth,
                   'data': json.dumps(data),
                   }
+        #if self.timeout:
+        #    params['timeout'] = self.timeout + 5
         logger.debug('postdata {0}'.format(params))
         resp = requests.post(**params)
         if resp.status_code == 401:

pepper/script.py

@@ -87,9 +87,11 @@ def __call__(self):
                                         )
                             elif 'data' in ret:
                                 # unfold runners
-                                outputter = ret.get('outputter', 'nested')
+                                outputter = ret.get('outputter') or ret.get('data', {}).get('outputter') or ret.get('data', {}).get('return', {}).get('outputter', 'nested')
+
                                 if isinstance(ret['data'], dict) and 'return' in ret['data']:
                                     ret = ret['data']['return']
+
                                 salt.output.display_output(
                                     ret,
                                     self.cli.options.output or outputter,