From be105ea0bb12e2a8b33d4b5be6e177cd6fadf09f Mon Sep 17 00:00:00 2001 From: Eric Paulson Date: Sat, 10 Nov 2018 12:00:56 -0800 Subject: [PATCH 1/4] Has Unicode errors. --- dragonfly/actions/action_text.py | 147 ++++++++++++++++++------------- dragonfly/actions/keyboard.py | 111 ++++++++++++++--------- dragonfly/actions/sendinput.py | 33 +++++-- 3 files changed, 182 insertions(+), 109 deletions(-) diff --git a/dragonfly/actions/action_text.py b/dragonfly/actions/action_text.py index a587fcb..312ee67 100644 --- a/dragonfly/actions/action_text.py +++ b/dragonfly/actions/action_text.py @@ -3,109 +3,134 @@ # (c) Copyright 2007, 2008 by Christo Butcher # Licensed under the LGPL. # -# Dragonfly is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or +# Dragonfly is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Dragonfly is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Dragonfly is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # -# You should have received a copy of the GNU Lesser General Public -# License along with Dragonfly. If not, see +# You should have received a copy of the GNU Lesser General Public +# License along with Dragonfly. If not, see # . # - """ Text action ============================================================================ -This section describes the :class:`Text` action object. This type of +This section describes the :class:`Text` action object. This type of action is used for typing text into the foreground application. -It differs from the :class:`Key` action in that :class:`Text` is used for -typing literal text, while :class:`dragonfly.actions.action_key.Key` -emulates pressing keys on the keyboard. An example of this is that the -arrow-keys are not part of a text and so cannot be typed using the -:class:`Text` action, but can be sent by the +It differs from the :class:`Key` action in that :class:`Text` is used for +typing literal text, while :class:`dragonfly.actions.action_key.Key` +emulates pressing keys on the keyboard. An example of this is that the +arrow-keys are not part of a text and so cannot be typed using the +:class:`Text` action, but can be sent by the :class:`dragonfly.actions.action_key.Key` action. """ +from ..engines import get_engine +from ..windows.clipboard import Clipboard +from .action_base import ActionError, DynStrActionBase +from .action_key import Key +from .keyboard import Keyboard +from .typeables import typeables +from six import text_type + +# --------------------------------------------------------------------------- -from .action_base import DynStrActionBase, ActionError -from .typeables import typeables -from .keyboard import Keyboard -from .action_key import Key -from ..windows.clipboard import Clipboard -from ..engines import get_engine +USE_UNICODE = True -#--------------------------------------------------------------------------- +def require_hardware_emulation(): + """Return `True` if the current context requires hardware emulation.""" + global USE_UNICODE + return not USE_UNICODE + class Text(DynStrActionBase): """ - Action that sends keyboard events to type text. - - Arguments: - - *spec* (*str*) -- the text to type - - *static* (boolean) -- - if *True*, do not dynamically interpret *spec* - when executing this action - - *pause* (*float*) -- - the time to pause between each keystroke, given - in seconds - - *autofmt* (boolean) -- - if *True*, attempt to format the text with correct - spacing and capitalization. This is done by first mimicking - a word recognition and then analyzing its spacing and - capitalization and applying the same formatting to the text. - + `Action` that sends keyboard events to type text. + + Arguments: + - *spec* (*str*) -- the text to type + - *static* (boolean) -- + if *True*, do not dynamically interpret *spec* + when executing this action + - *pause* (*float*) -- + the time to pause between each keystroke, given + in seconds + - *autofmt* (boolean) -- + if *True*, attempt to format the text with correct + spacing and capitalization. This is done by first mimicking + a word recognition and then analyzing its spacing and + capitalization and applying the same formatting to the text. """ _pause_default = 0.02 _keyboard = Keyboard() _specials = { - "\n": typeables["enter"], - "\t": typeables["tab"], + "\n": typeables["enter"], + "\t": typeables["tab"], } def __init__(self, spec=None, static=False, pause=_pause_default, - autofmt=False): + autofmt=False, use_hardware=False): self._pause = pause self._autofmt = autofmt + self._use_hardware = use_hardware DynStrActionBase.__init__(self, spec=spec, static=static) def _parse_spec(self, spec): - """ Convert the given *spec* to keyboard events. """ + """Convert the given *spec* to keyboard events.""" + from struct import unpack events = [] - for character in spec: - if character in self._specials: - typeable = self._specials[character] - else: - try: - typeable = Keyboard.get_typeable(character) - except ValueError as e: - raise ActionError("Keyboard interface cannot type this" - " character: %r (in %r)" - % (character, spec)) - events.extend(typeable.events(self._pause)) + if self._use_hardware or require_hardware_emulation(): + for character in spec: + if character in self._specials: + typeable = self._specials[character] + events.extend(typeable.events(self._pause)) + else: + try: + typeable = Keyboard.get_typeable(character) + events.extend(typeable.events(self._pause)) + except ValueError: + raise ActionError("Keyboard interface cannot type this" + " character: %r (in %r)" + % (character, spec)) + else: + for character in text_type(spec): + if character in self._specials: + typeable = self._specials[character] + events.extend(typeable.events(self._pause)) + else: + byte_stream = character.encode("utf-16-le") + for short in unpack("<" + str(len(byte_stream) // 2) + "H", + byte_stream): + try: + typeable = Keyboard.get_typeable(short, + is_text=True) + events.extend(typeable.events(self._pause * 0.5)) + except ValueError: + raise ActionError("Keyboard interface cannot type " + "this character: %r (in %r)" % + (character, spec)) return events def _execute_events(self, events): """ - Send keyboard events. - - If instance was initialized with *autofmt* True, - then this method will mimic a word recognition - and analyze its formatting so as to autoformat - the text's spacing and capitalization before - sending it as keyboard events. + Send keyboard events. + If instance was initialized with *autofmt* True, + then this method will mimic a word recognition + and analyze its formatting so as to autoformat + the text's spacing and capitalization before + sending it as keyboard events. """ - if self._autofmt: # Mimic a word, select and copy it to retrieve capitalization. get_engine().mimic("test") diff --git a/dragonfly/actions/keyboard.py b/dragonfly/actions/keyboard.py index 1adc622..faee554 100644 --- a/dragonfly/actions/keyboard.py +++ b/dragonfly/actions/keyboard.py @@ -3,25 +3,22 @@ # (c) Copyright 2007, 2008 by Christo Butcher # Licensed under the LGPL. # -# Dragonfly is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or +# Dragonfly is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Dragonfly is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Dragonfly is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # -# You should have received a copy of the GNU Lesser General Public -# License along with Dragonfly. If not, see +# You should have received a copy of the GNU Lesser General Public +# License along with Dragonfly. If not, see # . # -""" -This file implements a Win32 keyboard interface using sendinput. - -""" +"""This file implements a Win32 keyboard interface using sendinput.""" import time @@ -34,68 +31,94 @@ send_input_array) -#--------------------------------------------------------------------------- -# Typeable class. - class Typeable(object): + """Container for keypress events.""" - __slots__ = ("_code", "_modifiers", "_name") + __slots__ = ("_code", "_modifiers", "_name", "_is_text") - def __init__(self, code, modifiers=(), name=None): + def __init__(self, code, modifiers=(), name=None, is_text=False): + """Set keypress information.""" self._code = code self._modifiers = modifiers self._name = name + self._is_text = is_text def __str__(self): - return "%s(%s)" % (self.__class__.__name__, self._name) + repr(self.events()) + """Return information useful for debugging.""" + return ("%s(%s)" % (self.__class__.__name__, self._name) + + repr(self.events())) def on_events(self, timeout=0): - events = [(m, True, 0) for m in self._modifiers] - events.append((self._code, True, timeout)) + """Return events for pressing this key down.""" + if self._is_text: + events = [(self._code, True, timeout, True)] + else: + events = [(m, True, 0) for m in self._modifiers] + events.append((self._code, True, timeout)) return events def off_events(self, timeout=0): - events = [(m, False, 0) for m in self._modifiers] - events.append((self._code, False, timeout)) - events.reverse() + """Return events for releasing this key.""" + if self._is_text: + events = [(self._code, False, timeout, True)] + else: + events = [(m, False, 0) for m in self._modifiers] + events.append((self._code, False, timeout)) + events.reverse() return events def events(self, timeout=0): - events = [(self._code, True, 0), (self._code, False, timeout)] - for m in self._modifiers[-1::-1]: - events.insert(0, (m, True, 0)) - events.append((m, False , 0)) + """Return events for pressing and then releasing this key.""" + if self._is_text: + events = [(self._code, True, timeout, True), + (self._code, False, timeout, True)] + else: + events = [(self._code, True, 0), (self._code, False, timeout)] + for m in self._modifiers[-1::-1]: + events.insert(0, (m, True, 0)) + events.append((m, False, 0)) return events -#--------------------------------------------------------------------------- -# Keyboard access class. - class Keyboard(object): + """Static class wrapper around SendInput.""" - shift_code = win32con.VK_SHIFT - ctrl_code = win32con.VK_CONTROL - alt_code = win32con.VK_MENU + shift_code = win32con.VK_SHIFT + ctrl_code = win32con.VK_CONTROL + alt_code = win32con.VK_MENU @classmethod def send_keyboard_events(cls, events): """ - Send a sequence of keyboard events. + Send a sequence of keyboard events. - Positional arguments: - events -- a sequence of 3-tuples of the form - (keycode, down, timeout), where - keycode (int): virtual key code. + Positional arguments: + events -- a sequence of tuples of the form + (keycode, down, timeout), where + keycode (int): Win32 Virtual Key code. down (boolean): True means the key will be pressed down, False means the key will be released. timeout (int): number of seconds to sleep after the keyboard event. - + or + (character, down, timeout, is_text), where + character (str): Unicode character. + down (boolean): True means the key will be pressed down, + False means the key will be released. + timeout (int): number of seconds to sleep after + the keyboard event. + is_text (boolean): True means that the keypress is targeted + at a window or control that accepts Unicode text. """ items = [] - for keycode, down, timeout in events: - input = KeyboardInput(keycode, down) - items.append(input) + for event in events: + if len(event) == 3: + keycode, down, timeout = event + input_structure = KeyboardInput(keycode, down) + elif len(event) == 4 and event[3]: + character, down, timeout = event[:3] + input_structure = KeyboardInput(0, down, scancode=character) + items.append(input_structure) if timeout: array = make_input_array(items) items = [] @@ -143,7 +166,9 @@ def get_keycode_and_modifiers(cls, char): return code, modifiers @classmethod - def get_typeable(cls, char): + def get_typeable(cls, char, is_text=False): + if is_text: + return Typeable(char, is_text=True) code, modifiers = cls.get_keycode_and_modifiers(char) return Typeable(code, modifiers) diff --git a/dragonfly/actions/sendinput.py b/dragonfly/actions/sendinput.py index d610473..413c154 100644 --- a/dragonfly/actions/sendinput.py +++ b/dragonfly/actions/sendinput.py @@ -19,8 +19,10 @@ # """ - This file implements an interface to the Win32 SendInput function - for simulating keyboard and mouse events. +Win32 input wrapper functions. + +This file implements an interface to the Win32 SendInput function +for simulating keyboard and mouse events. """ @@ -29,14 +31,29 @@ import win32con import win32api +# These virtual keys don't have corresponding scancodes. +# The list was found experimentally and is open to improvement. +SOFT_KEYS = [x for x in range(0xc1, 0xdb)] +SOFT_KEYS += [x for x in range(0x15, 0x1b)] +SOFT_KEYS += [x for x in range(0x1c, 0x20)] +SOFT_KEYS += [x for x in range(0x3a, 0x41)] +SOFT_KEYS += [x for x in range(0x88, 0x90)] +SOFT_KEYS += [x for x in range(0xa6, 0xba)] +SOFT_KEYS += [ + 0xe0, 0xe5, 0xe7, 0xe8, 0xfc, 0x01, 0x02, 0x4, 0x5, 0x6, 0x7, 0x0a, 0x0b, + 0x0e, 0x0f, 0x5d, 0x5e, 0x5f +] + class KeyboardInput(Structure): + """Win32 KEYBDINPUT wrapper.""" + _fields_ = [("wVk", c_ushort), ("wScan", c_ushort), ("dwFlags", c_ulong), ("time", c_ulong), ("dwExtraInfo", POINTER(c_ulong))] - + soft_keys = tuple(SOFT_KEYS) # From https://docs.microsoft.com/en-us/windows/desktop/inputdev/about-keyboard-input#extended-key-flag # The extended keys consist of the ALT and CTRL keys # on the right-hand side of the keyboard; the INS, DEL, HOME, @@ -68,10 +85,16 @@ class KeyboardInput(Structure): win32con.VK_RWIN, ) - def __init__(self, virtual_keycode, down): - scancode = windll.user32.MapVirtualKeyA(virtual_keycode, 0) + def __init__(self, virtual_keycode, down, scancode=-1): + """Initialize structure based on key type.""" + if scancode == -1: + scancode = windll.user32.MapVirtualKeyW(virtual_keycode, 0) flags = 0 + if virtual_keycode is 0: + flags |= 4 # KEYEVENTF_UNICODE + elif virtual_keycode not in self.soft_keys: + flags |= 8 # KEYEVENTF_SCANCODE if not down: flags |= win32con.KEYEVENTF_KEYUP if virtual_keycode in self.extended_keys: From 9da9b1c5b0da9c27a648c8fd10148f290ef4a2f3 Mon Sep 17 00:00:00 2001 From: Eric Paulson Date: Sat, 10 Nov 2018 20:36:37 -0800 Subject: [PATCH 2/4] Fixed Unicode errors. --- dragonfly/engines/backend_natlink/engine.py | 29 ++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/dragonfly/engines/backend_natlink/engine.py b/dragonfly/engines/backend_natlink/engine.py index 8b2a55f..35f5c47 100644 --- a/dragonfly/engines/backend_natlink/engine.py +++ b/dragonfly/engines/backend_natlink/engine.py @@ -28,7 +28,7 @@ - http://blogs.msdn.com/b/tsfaware/archive/2010/03/22/detecting-sleep-mode-in-sapi.aspx """ -from six import text_type +from six import text_type, binary_type from ..base import EngineBase, EngineError, MimicFailure from ...error import GrammarError @@ -39,7 +39,21 @@ import dragonfly.grammar.state as state_ -#--------------------------------------------------------------------------- +# --------------------------------------------------------------------------- + +def map_word(word, encoding="windows-1252"): + """ + Wraps output from Dragon. + + This wrapper ensures text output from the engine is Unicode. It assumes the + encoding of byte streams is Windows-1252 by default. + """ + if isinstance(word, text_type): + return word + elif isinstance(word, binary_type): + return word.decode(encoding) + return word + class NatlinkEngine(EngineBase): """ Speech recognition engine back-end for Natlink and DNS. """ @@ -250,6 +264,7 @@ def _get_language(self): #--------------------------------------------------------------------------- + class GrammarWrapper(object): def __init__(self, grammar, grammar_object, engine): @@ -258,7 +273,8 @@ def __init__(self, grammar, grammar_object, engine): self.engine = engine def begin_callback(self, module_info): - executable, title, handle = module_info + executable, title, handle = tuple(map_word(word) + for word in module_info) self.grammar.process_begin(executable, title, handle) def results_callback(self, words, results): @@ -268,7 +284,7 @@ def results_callback(self, words, results): if words == "other": func = getattr(self.grammar, "process_recognition_other", None) if func: - words = tuple(text_type(w).encode("windows-1252") + words = tuple(map_word(w) for w in results.getWords(0)) func(words) return @@ -281,11 +297,6 @@ def results_callback(self, words, results): # If the words argument was not "other" or "reject", then # it is a sequence of (word, rule_id) 2-tuples. Convert this # into a tuple of unicode objects. - def map_word(w): - if isinstance(w, text_type): - return w - else: - return w.decode("windows-1252") words_rules = tuple((map_word(w), r) for w, r in words) words = tuple(w for w, r in words_rules) From ff25280ddeb62d2020821570c1e73c06a0d09ba1 Mon Sep 17 00:00:00 2001 From: Eric Paulson Date: Tue, 20 Nov 2018 07:13:12 -0800 Subject: [PATCH 3/4] Implemented basic configuration loading. --- dragonfly/actions/action_text.py | 52 +++++++++++++++++++-- dragonfly/engines/backend_natlink/engine.py | 9 ++-- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/dragonfly/actions/action_text.py b/dragonfly/actions/action_text.py index 312ee67..c511c3b 100644 --- a/dragonfly/actions/action_text.py +++ b/dragonfly/actions/action_text.py @@ -33,23 +33,67 @@ """ + +from six import text_type + from ..engines import get_engine from ..windows.clipboard import Clipboard +from ..windows.window import Window from .action_base import ActionError, DynStrActionBase from .action_key import Key from .keyboard import Keyboard from .typeables import typeables -from six import text_type # --------------------------------------------------------------------------- -USE_UNICODE = True +UNICODE_KEYBOARD = True +HARDWARE_APPS = [ + "tvnviewer.exe", "vncviewer.exe", "mstsc.exe", "virtualbox.exe" + ] + + +def load_configuration(): + """Locate and load configuration.""" + import io + import os + try: + import configparser + except ImportError: + import ConfigParser as configparser + + global UNICODE_KEYBOARD + global HARDWARE_APPS + + home = os.path.expanduser("~") + config_folder = os.path.join(home, ".dragonfly2-speech") + config_file = "settings.cfg" + config_path = os.path.join(config_folder, config_file) + + if not os.path.exists(config_folder): + os.mkdir(config_folder) + if not os.path.exists(config_path): + with io.open(config_path, "w") as f: + f.write(u'[Text]\nhardware_apps = ' + u'tvnviewer.exe|vncviewer.exe|mstsc.exe|virtualbox.exe\n' + u'unicode_keyboard = true\n') + + parser = configparser.ConfigParser() + parser.read(config_path) + if parser.has_option("Text", "hardware_apps"): + HARDWARE_APPS = parser.get("Text", "hardware_apps").lower().split("|") + if parser.has_option("Text", "unicode_keyboard"): + UNICODE_KEYBOARD = parser.getboolean("Text", "unicode_keyboard") + + +load_configuration() def require_hardware_emulation(): """Return `True` if the current context requires hardware emulation.""" - global USE_UNICODE - return not USE_UNICODE + from os.path import basename + foreground_executable = basename(Window.get_foreground() + .executable.lower()) + return (not UNICODE_KEYBOARD) or (foreground_executable in HARDWARE_APPS) class Text(DynStrActionBase): diff --git a/dragonfly/engines/backend_natlink/engine.py b/dragonfly/engines/backend_natlink/engine.py index 35f5c47..9260fdb 100644 --- a/dragonfly/engines/backend_natlink/engine.py +++ b/dragonfly/engines/backend_natlink/engine.py @@ -86,7 +86,7 @@ def disconnect(self): """ Disconnect from natlink. """ self.natlink.natDisconnect() - #----------------------------------------------------------------------- + # ----------------------------------------------------------------------- # Methods for working with grammars. def _load_grammar(self, grammar): @@ -111,11 +111,8 @@ def _load_grammar(self, grammar): (compiled_grammar, rule_names) = c.compile_grammar(grammar) grammar._rule_names = rule_names - if (hasattr(grammar, "process_recognition_other") - or hasattr(grammar, "process_recognition_failure")): - all_results = True - else: - all_results = False + all_results = (hasattr(grammar, "process_recognition_other") + or hasattr(grammar, "process_recognition_failure")) hypothesis = False attempt_connect = False From 5106ba47d2e8dca48bc074203b26832e0d31a6b4 Mon Sep 17 00:00:00 2001 From: Eric Paulson Date: Sat, 8 Dec 2018 10:00:55 -0800 Subject: [PATCH 4/4] `map_word` auto detects system text encoding. --- dragonfly/engines/backend_natlink/engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dragonfly/engines/backend_natlink/engine.py b/dragonfly/engines/backend_natlink/engine.py index 9260fdb..8424603 100644 --- a/dragonfly/engines/backend_natlink/engine.py +++ b/dragonfly/engines/backend_natlink/engine.py @@ -28,6 +28,7 @@ - http://blogs.msdn.com/b/tsfaware/archive/2010/03/22/detecting-sleep-mode-in-sapi.aspx """ +from locale import getpreferredencoding from six import text_type, binary_type from ..base import EngineBase, EngineError, MimicFailure @@ -41,12 +42,12 @@ # --------------------------------------------------------------------------- -def map_word(word, encoding="windows-1252"): +def map_word(word, encoding=getpreferredencoding(do_setlocale=False)): """ Wraps output from Dragon. This wrapper ensures text output from the engine is Unicode. It assumes the - encoding of byte streams is Windows-1252 by default. + encoding of byte streams is the current locale's preferred encoding by default. """ if isinstance(word, text_type): return word