diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b678f571..e91d4ddc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,16 +9,21 @@ jobs: linux: strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [ + '3.7', + '3.8', + '3.9', + '3.10', + ] include: - - python-version: 3.6 - os: ubuntu-18.04 # MySQL 5.7.32 - - python-version: 3.7 - os: ubuntu-18.04 # MySQL 5.7.32 - - python-version: 3.8 - os: ubuntu-18.04 # MySQL 5.7.32 - - python-version: 3.9 + - python-version: '3.7' + os: ubuntu-18.04 # MySQL 5.7.32 + - python-version: '3.8' + os: ubuntu-18.04 # MySQL 5.7.32 + - python-version: '3.9' os: ubuntu-20.04 # MySQL 8.0.22 + - python-version: '3.10' + os: ubuntu-22.04 # MySQL 8.0.28 runs-on: ${{ matrix.os }} steps: @@ -61,4 +66,3 @@ jobs: run: | coverage combine coverage report - codecov diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..bd0617b5 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: "12 18 * * 1" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" diff --git a/.gitignore b/.gitignore index b13429e4..970fcd4f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ .cache/ .coverage .coverage.* + +.venv/ +venv/ diff --git a/README.md b/README.md index 35b03092..124686d4 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ $ sudo apt-get install mycli # Only on debian or ubuntu --ssh-config-host TEXT Host to connect to ssh server reading from ssh configuration. + --ssl Enable SSL for connection (automatically + enabled with other flags). --ssl-ca PATH CA file in PEM format. --ssl-capath TEXT CA directory. --ssl-cert PATH X509 cert in PEM format. @@ -136,6 +138,7 @@ Features * Log every query and its results to a file (disabled by default). * Pretty prints tabular data (with colors!) * Support for SSL connections +* Some features are only exposed as [key bindings](doc/key_bindings.rst) Contributions: -------------- @@ -220,7 +223,7 @@ Thanks to [PyMysql](https://github.com/PyMySQL/PyMySQL) for a pure python adapte ### Compatibility -Mycli is tested on macOS and Linux. +Mycli is tested on macOS and Linux, and requires Python 3.7 or better. **Mycli is not tested on Windows**, but the libraries used in this app are Windows-compatible. This means it should work without any modifications. If you're unable to run it diff --git a/changelog.md b/changelog.md index 52d6f8f9..b4309b49 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,59 @@ +Upcoming +======== + + +Bug Fixes: +---------- + +* Don't install tests. +* Do not ignore the socket passed with the -S option, even when no port is passed +* Fix unexpected exception when using dsn without username & password (Thanks: [Will Wang]) +* Let the `--prompt` option act normally with its predefined default value + + + +Internal: +--------- +* paramiko is newer than 2.11.0 now, remove version pinning `cryptography`. + + +1.27.0 (2023/08/11) +=================== + +Features: +--------- + +* Detect TiDB instance, show in the prompt, and use additional keywords. +* Fix the completion order to show more commonly-used keywords at the top. + +Bug Fixes: +---------- + +* Better handle empty statements in un/prettify +* Remove vi-mode bindings for prettify/unprettify. +* Honor `\G` when executing from commandline with `-e`. +* Correctly report the version of TiDB. +* Revised `botton` spelling mistakes with `bottom` in `mycli/clitoolbar.py` + + +1.26.1 (2022/09/01) +=================== + +Bug Fixes: +---------- +* Require Python 3.7 in `setup.py` + + +1.26.0 (2022/09/01) +=================== + +Features: +--------- + +* Add `--ssl` flag to enable ssl/tls. +* Add `pager` option to `~/.myclirc`, for instance `pager = 'pspg --csv'` (Thanks: [BuonOmo]) +* Add prettify/unprettify keybindings to format the current statement using `sqlglot`. -TBD -=== Features: --------- @@ -894,6 +947,7 @@ Bug Fixes: [Amjith Ramanujam]: https://blog.amjith.com [Artem Bezsmertnyi]: https://github.com/mrdeathless +[BuonOmo]: https://github.com/BuonOmo [Carlos Afonso]: https://github.com/afonsocarlos [Casper Langemeijer]: https://github.com/langemeijer [Daniel West]: http://github.com/danieljwest @@ -920,3 +974,4 @@ Bug Fixes: [William GARCIA]: https://github.com/willgarcia [xeron]: https://github.com/xeron [Zach DeCook]: https://zachdecook.com +[Will Wang]: https://github.com/willww64 diff --git a/doc/key_bindings.rst b/doc/key_bindings.rst new file mode 100644 index 00000000..e3ebcd9b --- /dev/null +++ b/doc/key_bindings.rst @@ -0,0 +1,65 @@ +************* +Key Bindings: +************* + +Most key bindings are simply inherited from `prompt-toolkit `_ . + +The following key bindings are special to mycli: + +### +F2 +### + +Enable/Disable SmartCompletion Mode. + +### +F3 +### + +Enable/Disable Multiline Mode. + +### +F4 +### + +Toggle between Vi and Emacs mode. + +### +Tab +### + +Force autocompletion at cursor. + +####### +C-space +####### + +Initialize autocompletion at cursor. + +If the autocompletion menu is not showing, display it with the appropriate completions for the context. + +If the menu is showing, select the next completion. + +######### +ESC Enter +######### + +Introduce a line break in multi-line mode, or dispatch the command in single-line mode. + +The sequence ESC-Enter is often sent by Alt-Enter. + +################## +C-x p (Emacs-mode) +################## + +Prettify and indent current statement, usually into multiple lines. + +Only accepts buffers containing single SQL statements. + +################## +C-x u (Emacs-mode) +################## + +Unprettify and dedent current statement, usually into one line. + +Only accepts buffers containing single SQL statements. diff --git a/mycli/AUTHORS b/mycli/AUTHORS index dd276867..e9c73aa6 100644 --- a/mycli/AUTHORS +++ b/mycli/AUTHORS @@ -1,11 +1,7 @@ -Project Lead: -------------- - * Thomas Roten - - Core Developers: ---------------- + * Thomas Roten * Irina Truong * Matheus Rosa * Darik Gamble @@ -24,6 +20,7 @@ Contributors: * Artem Bezsmertnyi * bitkeen * bjarnagin + * BuonOmo * caitinggui * Carlos Afonso * Casper Langemeijer @@ -34,6 +31,7 @@ Contributors: * Daniel Black * Daniel West * Daniël van Eeden + * Fabrizio Gennari * François Pietka * Frederic Aoustin * Georgy Frolov @@ -93,6 +91,12 @@ Contributors: * Arvind Mishra * Kevin Schmeichel * Mel Dafert + * Thomas Copper + * Will Wang + * Alfred Wingate + * Zhanze Wang + * Houston Wong + Created by: ----------- diff --git a/mycli/__init__.py b/mycli/__init__.py index 8de33c0c..e2ba8ba3 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.25.0' +__version__ = '1.27.0' diff --git a/mycli/clitoolbar.py b/mycli/clitoolbar.py index eec2978f..52b6ee45 100644 --- a/mycli/clitoolbar.py +++ b/mycli/clitoolbar.py @@ -7,8 +7,7 @@ def create_toolbar_tokens_func(mycli, show_fish_help): """Return a function that generates the toolbar tokens.""" def get_toolbar_tokens(): - result = [] - result.append(('class:bottom-toolbar', ' ')) + result = [('class:bottom-toolbar', ' ')] if mycli.multi_line: delimiter = special.get_current_delimiter() @@ -26,10 +25,15 @@ def get_toolbar_tokens(): '[F3] Multiline: OFF ')) if mycli.prompt_app.editing_mode == EditingMode.VI: result.append(( - 'class:botton-toolbar.on', + 'class:bottom-toolbar.on', 'Vi-mode ({})'.format(_get_vi_mode()) )) + if mycli.toolbar_error_message: + result.append( + ('class:bottom-toolbar', ' ' + mycli.toolbar_error_message)) + mycli.toolbar_error_message = None + if show_fish_help(): result.append( ('class:bottom-toolbar', ' Right-arrow to complete suggestion')) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index 8eb3de9b..5d5f40fc 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -3,7 +3,7 @@ from collections import OrderedDict from .sqlcompleter import SQLCompleter -from .sqlexecute import SQLExecute +from .sqlexecute import SQLExecute, ServerSpecies class CompletionRefresher(object): @@ -113,6 +113,8 @@ def refresh_users(completer, executor): @refresher('functions') def refresh_functions(completer, executor): completer.extend_functions(executor.functions()) + if executor.server_info.species == ServerSpecies.TiDB: + completer.extend_functions(completer.tidb_functions, builtin=True) @refresher('special_commands') def refresh_special(completer, executor): @@ -121,3 +123,8 @@ def refresh_special(completer, executor): @refresher('show_commands') def refresh_show_commands(completer, executor): completer.extend_show_items(executor.show_candidates()) + +@refresher('keywords') +def refresh_keywords(completer, executor): + if executor.server_info.species == ServerSpecies.TiDB: + completer.extend_keywords(completer.tidb_keywords, replace=True) diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index 4a24c82b..443233fd 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -1,6 +1,6 @@ import logging from prompt_toolkit.enums import EditingMode -from prompt_toolkit.filters import completion_is_selected +from prompt_toolkit.filters import completion_is_selected, emacs_mode from prompt_toolkit.key_binding import KeyBindings _logger = logging.getLogger(__name__) @@ -61,6 +61,46 @@ def _(event): else: b.start_completion(select_first=False) + @kb.add('c-x', 'p', filter=emacs_mode) + def _(event): + """ + Prettify and indent current statement, usually into multiple lines. + + Only accepts buffers containing single SQL statements. + """ + _logger.debug('Detected /> key.') + + b = event.app.current_buffer + cursorpos_relative = b.cursor_position / max(1, len(b.text)) + pretty_text = mycli.handle_prettify_binding(b.text) + if len(pretty_text) > 0: + b.text = pretty_text + cursorpos_abs = int(round(cursorpos_relative * len(b.text))) + while 0 < cursorpos_abs < len(b.text) \ + and b.text[cursorpos_abs] in (' ', '\n'): + cursorpos_abs -= 1 + b.cursor_position = min(cursorpos_abs, len(b.text)) + + @kb.add('c-x', 'u', filter=emacs_mode) + def _(event): + """ + Unprettify and dedent current statement, usually into one line. + + Only accepts buffers containing single SQL statements. + """ + _logger.debug('Detected /< key.') + + b = event.app.current_buffer + cursorpos_relative = b.cursor_position / max(1, len(b.text)) + unpretty_text = mycli.handle_unprettify_binding(b.text) + if len(unpretty_text) > 0: + b.text = unpretty_text + cursorpos_abs = int(round(cursorpos_relative * len(b.text))) + while 0 < cursorpos_abs < len(b.text) \ + and b.text[cursorpos_abs] in (' ', '\n'): + cursorpos_abs -= 1 + b.cursor_position = min(cursorpos_abs, len(b.text)) + @kb.add('enter', filter=completion_is_selected) def _(event): """Makes the enter key work as the tab key only when showing the menu. diff --git a/mycli/magic.py b/mycli/magic.py index aad229a5..e1611bcc 100644 --- a/mycli/magic.py +++ b/mycli/magic.py @@ -19,8 +19,16 @@ def load_ipython_extension(ipython): def mycli_line_magic(line): _logger.debug('mycli magic called: %r', line) parsed = sql.parse.parse(line, {}) - conn = sql.connection.Connection(parsed['connection']) - + # "get" was renamed to "set" in ipython-sql: + # https://github.com/catherinedevlin/ipython-sql/commit/f4283c65aaf68f961e84019e8b939e4a3c501d43 + if hasattr(sql.connection.Connection, "get"): + conn = sql.connection.Connection.get(parsed["connection"]) + else: + try: + conn = sql.connection.Connection.set(parsed["connection"]) + # a new positional argument was added to Connection.set in version 0.4.0 of ipython-sql + except TypeError: + conn = sql.connection.Connection.set(parsed["connection"], False) try: # A corresponding mycli object already exists mycli = conn._mycli diff --git a/mycli/main.py b/mycli/main.py index 6d3641db..7033294e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -24,6 +24,7 @@ from cli_helpers.utils import strip_ansi import click import sqlparse +import sqlglot from mycli.packages.parseutils import is_dropping_database, is_destructive from prompt_toolkit.completion import DynamicCompleter from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode @@ -92,6 +93,7 @@ class MyCli(object): default_prompt = '\\t \\u@\\h:\\d> ' + default_prompt_splitln = '\\u@\\h\\n(\\t):\\d>' max_len_prompt = 45 defaults_suffix = None @@ -123,6 +125,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.logfile = logfile self.defaults_suffix = defaults_suffix self.login_path = login_path + self.toolbar_error_message = None # self.cnf_files is a class variable that stores the list of mysql # config files to read in at launch. @@ -425,6 +428,7 @@ def connect(self, database='', user='', passwd='', host='', port='', port = 3306 if not host or host == 'localhost': socket = ( + socket or cnf['socket'] or cnf['default_socket'] or guess_socket_location() @@ -582,6 +586,34 @@ def handle_clip_command(self, text): return True return False + def handle_prettify_binding(self, text): + try: + statements = sqlglot.parse(text, read='mysql') + except Exception as e: + statements = [] + if len(statements) == 1 and statements[0]: + pretty_text = statements[0].sql(pretty=True, pad=4, dialect='mysql') + else: + pretty_text = '' + self.toolbar_error_message = 'Prettify failed to parse statement' + if len(pretty_text) > 0: + pretty_text = pretty_text + ';' + return pretty_text + + def handle_unprettify_binding(self, text): + try: + statements = sqlglot.parse(text, read='mysql') + except Exception as e: + statements = [] + if len(statements) == 1 and statements[0]: + unpretty_text = statements[0].sql(pretty=False, dialect='mysql') + else: + unpretty_text = '' + self.toolbar_error_message = 'Unprettify failed to parse statement' + if len(unpretty_text) > 0: + unpretty_text = unpretty_text + ';' + return unpretty_text + def run_cli(self): iterations = 0 sqlexecute = self.sqlexecute @@ -613,7 +645,7 @@ def run_cli(self): def get_message(): prompt = self.get_prompt(self.prompt_format) if self.prompt_format == self.default_prompt and len(prompt) > self.max_len_prompt: - prompt = self.get_prompt('\\d> ') + prompt = self.get_prompt(self.default_prompt_splitln) prompt = prompt.replace("\\x1b", "\x1b") return ANSI(prompt) @@ -724,7 +756,7 @@ def one_iteration(text=None): except KeyboardInterrupt: pass if self.beep_after_seconds > 0 and t >= self.beep_after_seconds: - self.echo('\a', err=True, nl=False) + self.bell() if special.is_timing_enabled(): self.echo('Time: %0.03fs' % t) except KeyboardInterrupt: @@ -865,6 +897,11 @@ def echo(self, s, **kwargs): self.log_output(s) click.secho(s, **kwargs) + def bell(self): + """Print a bell on the stderr. + """ + click.secho('\a', err=True, nl=False) + def get_output_margin(self, status=None): """Get the output margin (number of rows for the prompt, footer and timing message.""" @@ -938,8 +975,9 @@ def configure_pager(self): os.environ['LESS'] = '-RXF' cnf = self.read_my_cnf_files(self.cnf_files, ['pager', 'skip-pager']) - if cnf['pager']: - special.set_pager(cnf['pager']) + cnf_pager = cnf['pager'] or self.config['main']['pager'] + if cnf_pager: + special.set_pager(cnf_pager) self.explicit_pager = True else: self.explicit_pager = False @@ -1002,7 +1040,7 @@ def run_query(self, query, new_line=True): for result in results: title, cur, headers, status = result self.formatter.query = query - output = self.format_output(title, cur, headers) + output = self.format_output(title, cur, headers, special.is_expanded_output()) for line in output: click.echo(line, nl=new_line) @@ -1089,6 +1127,8 @@ def get_last_query(self): @click.option('--ssh-config-path', help='Path to ssh configuration.', default=os.path.expanduser('~') + '/.ssh/config') @click.option('--ssh-config-host', help='Host to connect to ssh server reading from ssh configuration.') +@click.option('--ssl', 'ssl_enable', is_flag=True, + help='Enable SSL for connection (automatically enabled with other flags).') @click.option('--ssl-ca', help='CA file in PEM format.', type=click.Path(exists=True)) @click.option('--ssl-capath', help='CA directory.') @@ -1115,7 +1155,7 @@ def get_last_query(self): help='list of DSN configured into the [alias_dsn] section of myclirc file.') @click.option('--list-ssh-config', 'list_ssh_config', is_flag=True, help='list ssh configurations in the ssh config (requires paramiko).') -@click.option('-R', '--prompt', 'prompt', +@click.option('-R', '--prompt', 'prompt', default=MyCli.default_prompt, help='Prompt format (Default: "{0}").'.format( MyCli.default_prompt)) @click.option('-l', '--logfile', type=click.File(mode='a', encoding='utf-8'), @@ -1150,7 +1190,7 @@ def get_last_query(self): def cli(database, user, host, port, socket, password, dbname, version, verbose, prompt, logfile, defaults_group_suffix, defaults_file, login_path, auto_vertical_output, local_infile, - ssl_ca, ssl_capath, ssl_cert, ssl_key, ssl_cipher, + ssl_enable, ssl_ca, ssl_capath, ssl_cert, ssl_key, ssl_cipher, tls_version, ssl_verify_server_cert, table, csv, warn, execute, myclirc, dsn, list_dsn, ssh_user, ssh_host, ssh_port, ssh_password, ssh_key_filename, list_ssh_config, ssh_config_path, ssh_config_host, @@ -1205,6 +1245,7 @@ def cli(database, user, host, port, socket, password, dbname, database = dbname or database ssl = { + 'enable': ssl_enable, 'ca': ssl_ca and os.path.expanduser(ssl_ca), 'cert': ssl_cert and os.path.expanduser(ssl_cert), 'key': ssl_key and os.path.expanduser(ssl_key), @@ -1243,7 +1284,7 @@ def cli(database, user, host, port, socket, password, dbname, uri = urlparse(dsn_uri) if not database: database = uri.path[1:] # ignore the leading fwd slash - if not user: + if not user and uri.username is not None: user = unquote(uri.username) if not password and uri.password is not None: password = unquote(uri.password) @@ -1296,7 +1337,12 @@ def cli(database, user, host, port, socket, password, dbname, try: if csv: mycli.formatter.format_name = 'csv' - elif not table: + if execute.endswith(r'\G'): + execute = execute[:-2] + elif table: + if execute.endswith(r'\G'): + execute = execute[:-2] + else: mycli.formatter.format_name = 'tsv' mycli.run_query(execute) diff --git a/mycli/myclirc b/mycli/myclirc index ffd2226f..cd58dfe2 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -89,6 +89,9 @@ keyword_casing = auto # disabled pager on startup enable_pager = True +# Choose a specific pager +pager = 'less' + # Custom colors for the completion menu, toolbar, etc. [colors] completion-menu.completion.current = 'bg:#ffffff #000000' diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 3656aa69..ab42fb89 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -13,33 +13,178 @@ class SQLCompleter(Completer): - keywords = ['ACCESS', 'ADD', 'ALL', 'ALTER TABLE', 'AND', 'ANY', 'AS', - 'ASC', 'AUTO_INCREMENT', 'BEFORE', 'BEGIN', 'BETWEEN', - 'BIGINT', 'BINARY', 'BY', 'CASE', 'CHANGE MASTER TO', 'CHAR', - 'CHARACTER SET', 'CHECK', 'COLLATE', 'COLUMN', 'COMMENT', - 'COMMIT', 'CONSTRAINT', 'CREATE', 'CURRENT', - 'CURRENT_TIMESTAMP', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', - 'DELETE FROM', 'DESC', 'DESCRIBE', 'DROP', - 'ELSE', 'END', 'ENGINE', 'ESCAPE', 'EXISTS', 'FILE', 'FLOAT', - 'FOR', 'FOREIGN KEY', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', - 'GRANT', 'GROUP BY', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', - 'INCREMENT', 'INDEX', 'INSERT INTO', 'INT', 'INTEGER', - 'INTERVAL', 'INTO', 'IS', 'JOIN', 'KEY', 'LEFT', 'LEVEL', - 'LIKE', 'LIMIT', 'LOCK', 'LOGS', 'LONG', 'MASTER', - 'MEDIUMINT', 'MODE', 'MODIFY', 'NOT', 'NULL', 'NUMBER', - 'OFFSET', 'ON', 'OPTION', 'OR', 'ORDER BY', 'OUTER', 'OWNER', - 'PASSWORD', 'PORT', 'PRIMARY', 'PRIVILEGES', 'PROCESSLIST', - 'PURGE', 'REFERENCES', 'REGEXP', 'RENAME', 'REPAIR', 'RESET', - 'REVOKE', 'RIGHT', 'ROLLBACK', 'ROW', 'ROWS', 'ROW_FORMAT', - 'SAVEPOINT', 'SELECT', 'SESSION', 'SET', 'SHARE', 'SHOW', - 'SLAVE', 'SMALLINT', 'SMALLINT', 'START', 'STOP', 'TABLE', - 'THEN', 'TINYINT', 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', - 'UNION', 'UNIQUE', 'UNSIGNED', 'UPDATE', 'USE', 'USER', - 'USING', 'VALUES', 'VARCHAR', 'VIEW', 'WHEN', 'WHERE', 'WITH'] + keywords = [ + 'SELECT', 'FROM', 'WHERE', 'UPDATE', 'DELETE FROM', 'GROUP BY', + 'JOIN', 'INSERT INTO', 'LIKE', 'LIMIT', 'ACCESS', 'ADD', 'ALL', + 'ALTER TABLE', 'AND', 'ANY', 'AS', 'ASC', 'AUTO_INCREMENT', + 'BEFORE', 'BEGIN', 'BETWEEN', 'BIGINT', 'BINARY', 'BY', 'CASE', + 'CHANGE MASTER TO', 'CHAR', 'CHARACTER SET', 'CHECK', 'COLLATE', + 'COLUMN', 'COMMENT', 'COMMIT', 'CONSTRAINT', 'CREATE', 'CURRENT', + 'CURRENT_TIMESTAMP', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', + 'DESC', 'DESCRIBE', 'DROP', 'ELSE', 'END', 'ENGINE', 'ESCAPE', + 'EXISTS', 'FILE', 'FLOAT', 'FOR', 'FOREIGN KEY', 'FORMAT', 'FULL', + 'FUNCTION', 'GRANT', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', + 'INCREMENT', 'INDEX', 'INT', 'INTEGER', 'INTERVAL', 'INTO', 'IS', + 'KEY', 'LEFT', 'LEVEL', 'LOCK', 'LOGS', 'LONG', 'MASTER', + 'MEDIUMINT', 'MODE', 'MODIFY', 'NOT', 'NULL', 'NUMBER', 'OFFSET', + 'ON', 'OPTION', 'OR', 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', + 'PORT', 'PRIMARY', 'PRIVILEGES', 'PROCESSLIST', 'PURGE', + 'REFERENCES', 'REGEXP', 'RENAME', 'REPAIR', 'RESET', 'REVOKE', + 'RIGHT', 'ROLLBACK', 'ROW', 'ROWS', 'ROW_FORMAT', 'SAVEPOINT', + 'SESSION', 'SET', 'SHARE', 'SHOW', 'SLAVE', 'SMALLINT', 'SMALLINT', + 'START', 'STOP', 'TABLE', 'THEN', 'TINYINT', 'TO', 'TRANSACTION', + 'TRIGGER', 'TRUNCATE', 'UNION', 'UNIQUE', 'UNSIGNED', 'USE', + 'USER', 'USING', 'VALUES', 'VARCHAR', 'VIEW', 'WHEN', 'WITH' + ] + + tidb_keywords = [ + "SELECT", "FROM", "WHERE", "DELETE FROM", "UPDATE", "GROUP BY", + "JOIN", "INSERT INTO", "LIKE", "LIMIT", "ACCOUNT", "ACTION", "ADD", + "ADDDATE", "ADMIN", "ADVISE", "AFTER", "AGAINST", "AGO", + "ALGORITHM", "ALL", "ALTER", "ALWAYS", "ANALYZE", "AND", "ANY", + "APPROX_COUNT_DISTINCT", "APPROX_PERCENTILE", "AS", "ASC", "ASCII", + "ATTRIBUTES", "AUTO_ID_CACHE", "AUTO_INCREMENT", "AUTO_RANDOM", + "AUTO_RANDOM_BASE", "AVG", "AVG_ROW_LENGTH", "BACKEND", "BACKUP", + "BACKUPS", "BATCH", "BEGIN", "BERNOULLI", "BETWEEN", "BIGINT", + "BINARY", "BINDING", "BINDINGS", "BINDING_CACHE", "BINLOG", "BIT", + "BIT_AND", "BIT_OR", "BIT_XOR", "BLOB", "BLOCK", "BOOL", "BOOLEAN", + "BOTH", "BOUND", "BRIEF", "BTREE", "BUCKETS", "BUILTINS", "BY", + "BYTE", "CACHE", "CALL", "CANCEL", "CAPTURE", "CARDINALITY", + "CASCADE", "CASCADED", "CASE", "CAST", "CAUSAL", "CHAIN", "CHANGE", + "CHAR", "CHARACTER", "CHARSET", "CHECK", "CHECKPOINT", "CHECKSUM", + "CIPHER", "CLEANUP", "CLIENT", "CLIENT_ERRORS_SUMMARY", + "CLUSTERED", "CMSKETCH", "COALESCE", "COLLATE", "COLLATION", + "COLUMN", "COLUMNS", "COLUMN_FORMAT", "COLUMN_STATS_USAGE", + "COMMENT", "COMMIT", "COMMITTED", "COMPACT", "COMPRESSED", + "COMPRESSION", "CONCURRENCY", "CONFIG", "CONNECTION", + "CONSISTENCY", "CONSISTENT", "CONSTRAINT", "CONSTRAINTS", + "CONTEXT", "CONVERT", "COPY", "CORRELATION", "CPU", "CREATE", + "CROSS", "CSV_BACKSLASH_ESCAPE", "CSV_DELIMITER", "CSV_HEADER", + "CSV_NOT_NULL", "CSV_NULL", "CSV_SEPARATOR", + "CSV_TRIM_LAST_SEPARATORS", "CUME_DIST", "CURRENT", "CURRENT_DATE", + "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP", + "CURRENT_USER", "CURTIME", "CYCLE", "DATA", "DATABASE", + "DATABASES", "DATE", "DATETIME", "DATE_ADD", "DATE_SUB", "DAY", + "DAY_HOUR", "DAY_MICROSECOND", "DAY_MINUTE", "DAY_SECOND", "DDL", + "DEALLOCATE", "DECIMAL", "DEFAULT", "DEFINER", "DELAYED", + "DELAY_KEY_WRITE", "DENSE_RANK", "DEPENDENCY", "DEPTH", "DESC", + "DESCRIBE", "DIRECTORY", "DISABLE", "DISABLED", "DISCARD", "DISK", + "DISTINCT", "DISTINCTROW", "DIV", "DO", "DOT", "DOUBLE", "DRAINER", + "DROP", "DRY", "DUAL", "DUMP", "DUPLICATE", "DYNAMIC", "ELSE", + "ENABLE", "ENABLED", "ENCLOSED", "ENCRYPTION", "END", "ENFORCED", + "ENGINE", "ENGINES", "ENUM", "ERROR", "ERRORS", "ESCAPE", + "ESCAPED", "EVENT", "EVENTS", "EVOLVE", "EXACT", "EXCEPT", + "EXCHANGE", "EXCLUSIVE", "EXECUTE", "EXISTS", "EXPANSION", + "EXPIRE", "EXPLAIN", "EXPR_PUSHDOWN_BLACKLIST", "EXTENDED", + "EXTRACT", "FALSE", "FAST", "FAULTS", "FETCH", "FIELDS", "FILE", + "FIRST", "FIRST_VALUE", "FIXED", "FLASHBACK", "FLOAT", "FLUSH", + "FOLLOWER", "FOLLOWERS", "FOLLOWER_CONSTRAINTS", "FOLLOWING", + "FOR", "FORCE", "FOREIGN", "FORMAT", "FULL", "FULLTEXT", + "FUNCTION", "GENERAL", "GENERATED", "GET_FORMAT", "GLOBAL", + "GRANT", "GRANTS", "GROUPS", "GROUP_CONCAT", "HASH", "HAVING", + "HELP", "HIGH_PRIORITY", "HISTOGRAM", "HISTOGRAMS_IN_FLIGHT", + "HISTORY", "HOSTS", "HOUR", "HOUR_MICROSECOND", "HOUR_MINUTE", + "HOUR_SECOND", "IDENTIFIED", "IF", "IGNORE", "IMPORT", "IMPORTS", + "IN", "INCREMENT", "INCREMENTAL", "INDEX", "INDEXES", "INFILE", + "INNER", "INPLACE", "INSERT_METHOD", "INSTANCE", + "INSTANT", "INT", "INT1", "INT2", "INT3", "INT4", "INT8", + "INTEGER", "INTERNAL", "INTERSECT", "INTERVAL", "INTO", + "INVISIBLE", "INVOKER", "IO", "IPC", "IS", "ISOLATION", "ISSUER", + "JOB", "JOBS", "JSON", "JSON_ARRAYAGG", "JSON_OBJECTAGG", "KEY", + "KEYS", "KEY_BLOCK_SIZE", "KILL", "LABELS", "LAG", "LANGUAGE", + "LAST", "LASTVAL", "LAST_BACKUP", "LAST_VALUE", "LEAD", "LEADER", + "LEADER_CONSTRAINTS", "LEADING", "LEARNER", "LEARNERS", + "LEARNER_CONSTRAINTS", "LEFT", "LESS", "LEVEL", "LINEAR", "LINES", + "LIST", "LOAD", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP", "LOCATION", + "LOCK", "LOCKED", "LOGS", "LONG", "LONGBLOB", "LONGTEXT", + "LOW_PRIORITY", "MASTER", "MATCH", "MAX", "MAXVALUE", + "MAX_CONNECTIONS_PER_HOUR", "MAX_IDXNUM", "MAX_MINUTES", + "MAX_QUERIES_PER_HOUR", "MAX_ROWS", "MAX_UPDATES_PER_HOUR", + "MAX_USER_CONNECTIONS", "MB", "MEDIUMBLOB", "MEDIUMINT", + "MEDIUMTEXT", "MEMORY", "MERGE", "MICROSECOND", "MIN", "MINUTE", + "MINUTE_MICROSECOND", "MINUTE_SECOND", "MINVALUE", "MIN_ROWS", + "MOD", "MODE", "MODIFY", "MONTH", "NAMES", "NATIONAL", "NATURAL", + "NCHAR", "NEVER", "NEXT", "NEXTVAL", "NEXT_ROW_ID", "NO", + "NOCACHE", "NOCYCLE", "NODEGROUP", "NODE_ID", "NODE_STATE", + "NOMAXVALUE", "NOMINVALUE", "NONCLUSTERED", "NONE", "NORMAL", + "NOT", "NOW", "NOWAIT", "NO_WRITE_TO_BINLOG", "NTH_VALUE", "NTILE", + "NULL", "NULLS", "NUMERIC", "NVARCHAR", "OF", "OFF", "OFFSET", + "ON", "ONLINE", "ONLY", "ON_DUPLICATE", "OPEN", "OPTIMISTIC", + "OPTIMIZE", "OPTION", "OPTIONAL", "OPTIONALLY", + "OPT_RULE_BLACKLIST", "OR", "ORDER", "OUTER", "OUTFILE", "OVER", + "PACK_KEYS", "PAGE", "PARSER", "PARTIAL", "PARTITION", + "PARTITIONING", "PARTITIONS", "PASSWORD", "PERCENT", + "PERCENT_RANK", "PER_DB", "PER_TABLE", "PESSIMISTIC", "PLACEMENT", + "PLAN", "PLAN_CACHE", "PLUGINS", "POLICY", "POSITION", "PRECEDING", + "PRECISION", "PREDICATE", "PREPARE", "PRESERVE", + "PRE_SPLIT_REGIONS", "PRIMARY", "PRIMARY_REGION", "PRIVILEGES", + "PROCEDURE", "PROCESS", "PROCESSLIST", "PROFILE", "PROFILES", + "PROXY", "PUMP", "PURGE", "QUARTER", "QUERIES", "QUERY", "QUICK", + "RANGE", "RANK", "RATE_LIMIT", "READ", "REAL", "REBUILD", "RECENT", + "RECOVER", "RECURSIVE", "REDUNDANT", "REFERENCES", "REGEXP", + "REGION", "REGIONS", "RELEASE", "RELOAD", "REMOVE", "RENAME", + "REORGANIZE", "REPAIR", "REPEAT", "REPEATABLE", "REPLACE", + "REPLAYER", "REPLICA", "REPLICAS", "REPLICATION", "REQUIRE", + "REQUIRED", "RESET", "RESPECT", "RESTART", "RESTORE", "RESTORES", + "RESTRICT", "RESUME", "REVERSE", "REVOKE", "RIGHT", "RLIKE", + "ROLE", "ROLLBACK", "ROUTINE", "ROW", "ROWS", "ROW_COUNT", + "ROW_FORMAT", "ROW_NUMBER", "RTREE", "RUN", "RUNNING", "S3", + "SAMPLERATE", "SAMPLES", "SAN", "SAVEPOINT", "SCHEDULE", "SECOND", + "SECONDARY_ENGINE", "SECONDARY_LOAD", "SECONDARY_UNLOAD", + "SECOND_MICROSECOND", "SECURITY", "SEND_CREDENTIALS_TO_TIKV", + "SEPARATOR", "SEQUENCE", "SERIAL", "SERIALIZABLE", "SESSION", + "SESSION_STATES", "SET", "SETVAL", "SHARD_ROW_ID_BITS", "SHARE", + "SHARED", "SHOW", "SHUTDOWN", "SIGNED", "SIMPLE", "SKIP", + "SKIP_SCHEMA_FILES", "SLAVE", "SLOW", "SMALLINT", "SNAPSHOT", + "SOME", "SOURCE", "SPATIAL", "SPLIT", "SQL", "SQL_BIG_RESULT", + "SQL_BUFFER_RESULT", "SQL_CACHE", "SQL_CALC_FOUND_ROWS", + "SQL_NO_CACHE", "SQL_SMALL_RESULT", "SQL_TSI_DAY", "SQL_TSI_HOUR", + "SQL_TSI_MINUTE", "SQL_TSI_MONTH", "SQL_TSI_QUARTER", + "SQL_TSI_SECOND", "SQL_TSI_WEEK", "SQL_TSI_YEAR", "SSL", + "STALENESS", "START", "STARTING", "STATISTICS", "STATS", + "STATS_AUTO_RECALC", "STATS_BUCKETS", "STATS_COL_CHOICE", + "STATS_COL_LIST", "STATS_EXTENDED", "STATS_HEALTHY", + "STATS_HISTOGRAMS", "STATS_META", "STATS_OPTIONS", + "STATS_PERSISTENT", "STATS_SAMPLE_PAGES", "STATS_SAMPLE_RATE", + "STATS_TOPN", "STATUS", "STD", "STDDEV", "STDDEV_POP", + "STDDEV_SAMP", "STOP", "STORAGE", "STORED", "STRAIGHT_JOIN", + "STRICT", "STRICT_FORMAT", "STRONG", "SUBDATE", "SUBJECT", + "SUBPARTITION", "SUBPARTITIONS", "SUBSTRING", "SUM", "SUPER", + "SWAPS", "SWITCHES", "SYSTEM", "SYSTEM_TIME", "TABLE", "TABLES", + "TABLESAMPLE", "TABLESPACE", "TABLE_CHECKSUM", "TARGET", + "TELEMETRY", "TELEMETRY_ID", "TEMPORARY", "TEMPTABLE", + "TERMINATED", "TEXT", "THAN", "THEN", "TIDB", "TIFLASH", + "TIKV_IMPORTER", "TIME", "TIMESTAMP", "TIMESTAMPADD", + "TIMESTAMPDIFF", "TINYBLOB", "TINYINT", "TINYTEXT", "TLS", "TO", + "TOKUDB_DEFAULT", "TOKUDB_FAST", "TOKUDB_LZMA", "TOKUDB_QUICKLZ", + "TOKUDB_SMALL", "TOKUDB_SNAPPY", "TOKUDB_UNCOMPRESSED", + "TOKUDB_ZLIB", "TOP", "TOPN", "TRACE", "TRADITIONAL", "TRAILING", + "TRANSACTION", "TRIGGER", "TRIGGERS", "TRIM", "TRUE", + "TRUE_CARD_COST", "TRUNCATE", "TYPE", "UNBOUNDED", "UNCOMMITTED", + "UNDEFINED", "UNICODE", "UNION", "UNIQUE", "UNKNOWN", "UNLOCK", + "UNSIGNED", "USAGE", "USE", "USER", "USING", "UTC_DATE", + "UTC_TIME", "UTC_TIMESTAMP", "VALIDATION", "VALUE", "VALUES", + "VARBINARY", "VARCHAR", "VARCHARACTER", "VARIABLES", "VARIANCE", + "VARYING", "VAR_POP", "VAR_SAMP", "VERBOSE", "VIEW", "VIRTUAL", + "VISIBLE", "VOTER", "VOTERS", "VOTER_CONSTRAINTS", "WAIT", + "WARNINGS", "WEEK", "WEIGHT_STRING", "WHEN", "WIDTH", "WINDOW", + "WITH", "WITHOUT", "WRITE", "X509", "XOR", "YEAR", "YEAR_MONTH", + "ZEROFILL" + ] functions = ['AVG', 'CONCAT', 'COUNT', 'DISTINCT', 'FIRST', 'FORMAT', 'FROM_UNIXTIME', 'LAST', 'LCASE', 'LEN', 'MAX', 'MID', - 'MIN', 'NOW', 'ROUND', 'SUM', 'TOP', 'UCASE', 'UNIX_TIMESTAMP'] + 'MIN', 'NOW', 'ROUND', 'SUM', 'TOP', 'UCASE', + 'UNIX_TIMESTAMP' + ] + + # https://docs.pingcap.com/tidb/dev/tidb-functions + tidb_functions = [ + 'TIDB_BOUNDED_STALENESS', 'TIDB_DECODE_KEY', 'TIDB_DECODE_PLAN', + 'TIDB_IS_DDL_OWNER', 'TIDB_PARSE_TSO', 'TIDB_VERSION', + 'TIDB_DECODE_SQL_DIGESTS', 'VITESS_HASH', 'TIDB_SHARD' + ] + show_items = [] @@ -94,9 +239,12 @@ def extend_special_commands(self, special_commands): def extend_database_names(self, databases): self.databases.extend(databases) - def extend_keywords(self, additional_keywords): - self.keywords.extend(additional_keywords) - self.all_completions.update(additional_keywords) + def extend_keywords(self, keywords, replace=False): + if replace: + self.keywords = keywords + else: + self.keywords.extend(keywords) + self.all_completions.update(keywords) def extend_show_items(self, show_items): for show_item in show_items: @@ -172,7 +320,12 @@ def extend_columns(self, column_data, kind): metadata[self.dbname][relname].append(column) self.all_completions.add(column) - def extend_functions(self, func_data): + def extend_functions(self, func_data, builtin=False): + # if 'builtin' is set this is extending the list of builtin functions + if builtin: + self.functions.extend(func_data) + return + # 'func_data' is a generator object. It can throw an exception while # being consumed. This could happen if the user has launched the app # without specifying a database name. This exception must be handled to @@ -224,13 +377,13 @@ def find_matches(text, collection, start_only=False, fuzzy=True, casing=None): if fuzzy: regex = '.*?'.join(map(escape, text)) pat = compile('(%s)' % regex) - for item in sorted(collection): + for item in collection: r = pat.search(item.lower()) if r: completions.append((len(r.group()), r.start(), item)) else: match_end_limit = len(text) if start_only else None - for item in sorted(collection): + for item in collection: match_point = item.lower().find(text, 0, match_end_limit) if match_point >= 0: completions.append((len(text), match_point, item)) @@ -244,7 +397,7 @@ def apply_case(kw): return kw.lower() return (Completion(z if casing is None else apply_case(z), -len(text)) - for x, y, z in sorted(completions)) + for x, y, z in completions) def get_completions(self, document, complete_event, smart_completion=None): word_before_cursor = document.get_word_before_cursor(WORD=True) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 38de4f09..bd5f5d98 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -56,7 +56,7 @@ def from_version_string(cls, version_string): re_species = ( (r'(?P[0-9\.]+)-MariaDB', ServerSpecies.MariaDB), - (r'(?P[0-9\.]+)[a-z0-9]*-TiDB', ServerSpecies.TiDB), + (r'[0-9\.]*-TiDB-v(?P[0-9\.]+)-?(?P[a-z0-9\-]*)', ServerSpecies.TiDB), (r'(?P[0-9\.]+)[a-z0-9]*-(?P[0-9]+$)', ServerSpecies.Percona), (r'(?P[0-9\.]+)[a-z0-9]*-(?P[A-Za-z0-9_]+)', diff --git a/requirements-dev.txt b/requirements-dev.txt index 3d10e9a0..c2cd04b3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,6 @@ twine>=1.12.1 behave>=1.2.4 pexpect>=3.3 coverage>=5.0.4 -codecov>=2.0.9 autopep8==1.3.3 colorama>=0.4.1 git+https://github.com/hayd/pep8radius.git # --error-status option not released @@ -14,3 +13,4 @@ paramiko==2.11.0 pyperclip>=1.8.1 importlib_resources>=5.0.0 pyaes>=1.6.1 +sqlglot>=5.1.3 diff --git a/setup.py b/setup.py index c9dcc44e..2c4f9e18 100755 --- a/setup.py +++ b/setup.py @@ -18,14 +18,14 @@ install_requirements = [ 'click >= 7.0', - # Temporary to suppress paramiko Blowfish warning which breaks CI. - # Pinning cryptography should not be needed after paramiko 2.11.0. - 'cryptography == 36.0.2', + # Pinning cryptography is not needed after paramiko 2.11.0. Correct it + 'cryptography >= 1.0.0', # 'Pygments>=1.6,<=2.11.1', 'Pygments>=1.6', 'prompt_toolkit>=3.0.6,<4.0.0', 'PyMySQL >= 0.9.2', 'sqlparse>=0.3.0,<0.5.0', + 'sqlglot>=5.1.3', 'configobj >= 5.0.5', 'cli_helpers[styles] >= 2.2.1', 'pyperclip >= 1.8.1', @@ -94,7 +94,7 @@ def run_tests(self): author_email='mycli-dev@googlegroups.com', version=version, url='http://mycli.net', - packages=find_packages(), + packages=find_packages(exclude=['test*']), package_data={'mycli': ['myclirc', 'AUTHORS', 'SPONSORS']}, description=description, long_description=description, @@ -103,16 +103,17 @@ def run_tests(self): 'console_scripts': ['mycli = mycli.main:cli'], }, cmdclass={'lint': lint, 'test': test}, - python_requires=">=3.6", + python_requires=">=3.7", classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: Unix', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: SQL', 'Topic :: Database', 'Topic :: Database :: Front-Ends', diff --git a/test/myclirc b/test/myclirc index 261bee6e..0c1a7ad3 100644 --- a/test/myclirc +++ b/test/myclirc @@ -1,12 +1,157 @@ # vi: ft=dosini +[main] -# This file is loaded after mycli/myclirc and should override only those -# variables needed for testing. -# To see what every variable does see mycli/myclirc +# Enables context sensitive auto-completion. If this is disabled the all +# possible completions will be listed. +smart_completion = True -[main] +# Multi-line mode allows breaking up the sql statements into multiple lines. If +# this is set to True, then the end of the statements must have a semi-colon. +# If this is set to False then sql statements can't be split into multiple +# lines. End of line (return) is considered as the end of the statement. +multi_line = False + +# Destructive warning mode will alert you before executing a sql statement +# that may cause harm to the database such as "drop table", "drop database" +# or "shutdown". +destructive_warning = True +# log_file location. log_file = ~/.mycli.test.log + +# Default log level. Possible values: "CRITICAL", "ERROR", "WARNING", "INFO" +# and "DEBUG". "NONE" disables logging. log_level = DEBUG -prompt = '\t \u@\h:\d> ' + +# Log every query and its results to a file. Enable this by uncommenting the +# line below. +# audit_log = ~/.mycli-audit.log + +# Timing of sql statements and table rendering. +timing = True + +# Beep after long-running queries are completed; 0 to disable. +beep_after_seconds = 0 + +# Table format. Possible values: ascii, double, github, +# psql, plain, simple, grid, fancy_grid, pipe, orgtbl, rst, mediawiki, html, +# latex, latex_booktabs, textile, moinmoin, jira, vertical, tsv, csv. +# Recommended: ascii +table_format = ascii + +# Syntax coloring style. Possible values (many support the "-dark" suffix): +# manni, igor, xcode, vim, autumn, vs, rrt, native, perldoc, borland, tango, emacs, +# friendly, monokai, paraiso, colorful, murphy, bw, pastie, paraiso, trac, default, +# fruity. +# Screenshots at http://mycli.net/syntax +# Can be further modified in [colors] +syntax_style = default + +# Keybindings: Possible values: emacs, vi. +# Emacs mode: Ctrl-A is home, Ctrl-E is end. All emacs keybindings are available in the REPL. +# When Vi mode is enabled you can use modal editing features offered by Vi in the REPL. +key_bindings = emacs + +# Enabling this option will show the suggestions in a wider menu. Thus more items are suggested. +wider_completion_menu = False + +# MySQL prompt +# \D - The full current date +# \d - Database name +# \h - Hostname of the server +# \m - Minutes of the current time +# \n - Newline +# \P - AM/PM +# \p - Port +# \R - The current time, in 24-hour military time (0-23) +# \r - The current time, standard 12-hour time (1-12) +# \s - Seconds of the current time +# \t - Product type (Percona, MySQL, MariaDB, TiDB) +# \A - DSN alias name (from the [alias_dsn] section) +# \u - Username +# \x1b[...m - insert ANSI escape sequence +prompt = "\t \u@\h:\d> " +prompt_continuation = -> + +# Skip intro info on startup and outro info on exit less_chatty = True + +# Use alias from --login-path instead of host name in prompt +login_path_as_host = False + +# Cause result sets to be displayed vertically if they are too wide for the current window, +# and using normal tabular format otherwise. (This applies to statements terminated by ; or \G.) +auto_vertical_output = False + +# keyword casing preference. Possible values "lower", "upper", "auto" +keyword_casing = auto + +# disabled pager on startup +enable_pager = True + +# Custom colors for the completion menu, toolbar, etc. +[colors] +completion-menu.completion.current = "bg:#ffffff #000000" +completion-menu.completion = "bg:#008888 #ffffff" +completion-menu.meta.completion.current = "bg:#44aaaa #000000" +completion-menu.meta.completion = "bg:#448888 #ffffff" +completion-menu.multi-column-meta = "bg:#aaffff #000000" +scrollbar.arrow = "bg:#003333" +scrollbar = "bg:#00aaaa" +selected = "#ffffff bg:#6666aa" +search = "#ffffff bg:#4444aa" +search.current = "#ffffff bg:#44aa44" +bottom-toolbar = "bg:#222222 #aaaaaa" +bottom-toolbar.off = "bg:#222222 #888888" +bottom-toolbar.on = "bg:#222222 #ffffff" +search-toolbar = noinherit bold +search-toolbar.text = nobold +system-toolbar = noinherit bold +arg-toolbar = noinherit bold +arg-toolbar.text = nobold +bottom-toolbar.transaction.valid = "bg:#222222 #00ff5f bold" +bottom-toolbar.transaction.failed = "bg:#222222 #ff005f bold" + +# style classes for colored table output +output.header = "#00ff5f bold" +output.odd-row = "" +output.even-row = "" +output.null = "#808080" + +# SQL syntax highlighting overrides +# sql.comment = 'italic #408080' +# sql.comment.multi-line = '' +# sql.comment.single-line = '' +# sql.comment.optimizer-hint = '' +# sql.escape = 'border:#FF0000' +# sql.keyword = 'bold #008000' +# sql.datatype = 'nobold #B00040' +# sql.literal = '' +# sql.literal.date = '' +# sql.symbol = '' +# sql.quoted-schema-object = '' +# sql.quoted-schema-object.escape = '' +# sql.constant = '#880000' +# sql.function = '#0000FF' +# sql.variable = '#19177C' +# sql.number = '#666666' +# sql.number.binary = '' +# sql.number.float = '' +# sql.number.hex = '' +# sql.number.integer = '' +# sql.operator = '#666666' +# sql.punctuation = '' +# sql.string = '#BA2121' +# sql.string.double-quouted = '' +# sql.string.escape = 'bold #BB6622' +# sql.string.single-quoted = '' +# sql.whitespace = '' + +# Favorite queries. +[favorite_queries] +check = 'select "✔"' + +# Use the -d option to reference a DSN. +# Special characters in passwords and other strings can be escaped with URL encoding. +[alias_dsn] +# example_dsn = mysql://[user[:password]@][host][:port][/dbname] diff --git a/test/test_completion_refresher.py b/test/test_completion_refresher.py index cdc2fb5e..31359cf3 100644 --- a/test/test_completion_refresher.py +++ b/test/test_completion_refresher.py @@ -19,7 +19,7 @@ def test_ctor(refresher): assert len(refresher.refreshers) > 0 actual_handlers = list(refresher.refreshers.keys()) expected_handlers = ['databases', 'schemata', 'tables', 'users', 'functions', - 'special_commands', 'show_commands'] + 'special_commands', 'show_commands', 'keywords'] assert expected_handlers == actual_handlers diff --git a/test/test_main.py b/test/test_main.py index c3351eca..321aba81 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -270,7 +270,8 @@ def stub_terminal_size(): def test_list_dsn(): runner = CliRunner() - with NamedTemporaryFile(mode="w") as myclirc: + # keep Windows from locking the file with delete=False + with NamedTemporaryFile(mode="w",delete=False) as myclirc: myclirc.write(dedent("""\ [alias_dsn] test = mysql://test/test @@ -281,11 +282,35 @@ def test_list_dsn(): assert result.output == "test\n" result = runner.invoke(cli, args=args + ['--verbose']) assert result.output == "test : mysql://test/test\n" + + # delete=False means we should try to clean up + try: + if os.path.exists(myclirc.name): + os.remove(myclirc.name) + except Exception as e: + print(f"An error occurred while attempting to delete the file: {e}") + + + + +def test_prettify_statement(): + statement = 'SELECT 1' + m = MyCli() + pretty_statement = m.handle_prettify_binding(statement) + assert pretty_statement == 'SELECT\n 1;' + + +def test_unprettify_statement(): + statement = 'SELECT\n 1' + m = MyCli() + unpretty_statement = m.handle_unprettify_binding(statement) + assert unpretty_statement == 'SELECT 1;' def test_list_ssh_config(): runner = CliRunner() - with NamedTemporaryFile(mode="w") as ssh_config: + # keep Windows from locking the file with delete=False + with NamedTemporaryFile(mode="w",delete=False) as ssh_config: ssh_config.write(dedent("""\ Host test Hostname test.example.com @@ -299,6 +324,13 @@ def test_list_ssh_config(): assert "test\n" in result.output result = runner.invoke(cli, args=args + ['--verbose']) assert "test : test.example.com\n" in result.output + + # delete=False means we should try to clean up + try: + if os.path.exists(ssh_config.name): + os.remove(ssh_config.name) + except Exception as e: + print(f"An error occurred while attempting to delete the file: {e}") def test_dsn(monkeypatch): @@ -452,7 +484,8 @@ def run_query(self, query, new_line=True): runner = CliRunner() # Setup temporary configuration - with NamedTemporaryFile(mode="w") as ssh_config: + # keep Windows from locking the file with delete=False + with NamedTemporaryFile(mode="w",delete=False) as ssh_config: ssh_config.write(dedent("""\ Host test Hostname test.example.com @@ -475,8 +508,8 @@ def run_query(self, query, new_line=True): MockMyCli.connect_args["ssh_user"] == "joe" and \ MockMyCli.connect_args["ssh_host"] == "test.example.com" and \ MockMyCli.connect_args["ssh_port"] == 22222 and \ - MockMyCli.connect_args["ssh_key_filename"] == os.getenv( - "HOME") + "/.ssh/gateway" + MockMyCli.connect_args["ssh_key_filename"] == os.path.expanduser( + "~") + "/.ssh/gateway" # When a user supplies a ssh config host as argument to mycli, # and used command line arguments, use the command line @@ -498,6 +531,13 @@ def run_query(self, query, new_line=True): MockMyCli.connect_args["ssh_host"] == "arg_host" and \ MockMyCli.connect_args["ssh_port"] == 3 and \ MockMyCli.connect_args["ssh_key_filename"] == "/path/to/key" + + # delete=False means we should try to clean up + try: + if os.path.exists(ssh_config.name): + os.remove(ssh_config.name) + except Exception as e: + print(f"An error occurred while attempting to delete the file: {e}") @dbtest diff --git a/test/test_naive_completion.py b/test/test_naive_completion.py index 32b2abdf..0bc3bf87 100644 --- a/test/test_naive_completion.py +++ b/test/test_naive_completion.py @@ -21,7 +21,7 @@ def test_empty_string_completion(completer, complete_event): result = list(completer.get_completions( Document(text=text, cursor_position=position), complete_event)) - assert result == list(map(Completion, sorted(completer.all_completions))) + assert result == list(map(Completion, completer.all_completions)) def test_select_keyword_completion(completer, complete_event): @@ -39,9 +39,7 @@ def test_function_name_completion(completer, complete_event): result = list(completer.get_completions( Document(text=text, cursor_position=position), complete_event)) - assert result == list([ - Completion(text='MASTER', start_position=-2), - Completion(text='MAX', start_position=-2)]) + assert sorted(x.text for x in result) == ["MASTER", "MAX"] def test_column_name_completion(completer, complete_event): @@ -50,7 +48,7 @@ def test_column_name_completion(completer, complete_event): result = list(completer.get_completions( Document(text=text, cursor_position=position), complete_event)) - assert result == list(map(Completion, sorted(completer.all_completions))) + assert result == list(map(Completion, completer.all_completions)) def test_special_name_completion(completer, complete_event): diff --git a/test/test_smart_completion_public_schema_only.py b/test/test_smart_completion_public_schema_only.py index e7d460a8..b60e67c5 100644 --- a/test/test_smart_completion_public_schema_only.py +++ b/test/test_smart_completion_public_schema_only.py @@ -55,8 +55,8 @@ def test_empty_string_completion(completer, complete_event): completer.get_completions( Document(text=text, cursor_position=position), complete_event)) - assert list(map(Completion, sorted(completer.keywords) + - sorted(completer.special_commands))) == result + assert list(map(Completion, completer.keywords + + completer.special_commands)) == result def test_select_keyword_completion(completer, complete_event): @@ -74,10 +74,10 @@ def test_table_completion(completer, complete_event): result = completer.get_completions( Document(text=text, cursor_position=position), complete_event) assert list(result) == list([ - Completion(text='`réveillé`', start_position=0), - Completion(text='`select`', start_position=0), - Completion(text='orders', start_position=0), Completion(text='users', start_position=0), + Completion(text='orders', start_position=0), + Completion(text='`select`', start_position=0), + Completion(text='`réveillé`', start_position=0), ]) @@ -106,9 +106,9 @@ def test_suggested_column_names(completer, complete_event): complete_event)) assert result == list([ Completion(text='*', start_position=0), + Completion(text='id', start_position=0), Completion(text='email', start_position=0), Completion(text='first_name', start_position=0), - Completion(text='id', start_position=0), Completion(text='last_name', start_position=0), ] + list(map(Completion, completer.functions)) + @@ -132,9 +132,9 @@ def test_suggested_column_names_in_function(completer, complete_event): complete_event) assert list(result) == list([ Completion(text='*', start_position=0), + Completion(text='id', start_position=0), Completion(text='email', start_position=0), Completion(text='first_name', start_position=0), - Completion(text='id', start_position=0), Completion(text='last_name', start_position=0)]) @@ -153,9 +153,9 @@ def test_suggested_column_names_with_table_dot(completer, complete_event): complete_event)) assert result == list([ Completion(text='*', start_position=0), + Completion(text='id', start_position=0), Completion(text='email', start_position=0), Completion(text='first_name', start_position=0), - Completion(text='id', start_position=0), Completion(text='last_name', start_position=0)]) @@ -174,9 +174,9 @@ def test_suggested_column_names_with_alias(completer, complete_event): complete_event)) assert result == list([ Completion(text='*', start_position=0), + Completion(text='id', start_position=0), Completion(text='email', start_position=0), Completion(text='first_name', start_position=0), - Completion(text='id', start_position=0), Completion(text='last_name', start_position=0)]) @@ -196,9 +196,9 @@ def test_suggested_multiple_column_names(completer, complete_event): complete_event)) assert result == list([ Completion(text='*', start_position=0), + Completion(text='id', start_position=0), Completion(text='email', start_position=0), Completion(text='first_name', start_position=0), - Completion(text='id', start_position=0), Completion(text='last_name', start_position=0)] + list(map(Completion, completer.functions)) + [Completion(text='u', start_position=0)] + @@ -221,9 +221,9 @@ def test_suggested_multiple_column_names_with_alias(completer, complete_event): complete_event)) assert result == list([ Completion(text='*', start_position=0), + Completion(text='id', start_position=0), Completion(text='email', start_position=0), Completion(text='first_name', start_position=0), - Completion(text='id', start_position=0), Completion(text='last_name', start_position=0)]) @@ -243,9 +243,9 @@ def test_suggested_multiple_column_names_with_dot(completer, complete_event): complete_event)) assert result == list([ Completion(text='*', start_position=0), + Completion(text='id', start_position=0), Completion(text='email', start_position=0), Completion(text='first_name', start_position=0), - Completion(text='id', start_position=0), Completion(text='last_name', start_position=0)]) @@ -256,8 +256,9 @@ def test_suggested_aliases_after_on(completer, complete_event): Document(text=text, cursor_position=position), complete_event)) assert result == list([ + Completion(text='u', start_position=0), Completion(text='o', start_position=0), - Completion(text='u', start_position=0)]) + ]) def test_suggested_aliases_after_on_right_side(completer, complete_event): @@ -268,8 +269,9 @@ def test_suggested_aliases_after_on_right_side(completer, complete_event): Document(text=text, cursor_position=position), complete_event)) assert result == list([ + Completion(text='u', start_position=0), Completion(text='o', start_position=0), - Completion(text='u', start_position=0)]) + ]) def test_suggested_tables_after_on(completer, complete_event): @@ -279,8 +281,9 @@ def test_suggested_tables_after_on(completer, complete_event): Document(text=text, cursor_position=position), complete_event)) assert result == list([ + Completion(text='users', start_position=0), Completion(text='orders', start_position=0), - Completion(text='users', start_position=0)]) + ]) def test_suggested_tables_after_on_right_side(completer, complete_event): @@ -291,8 +294,9 @@ def test_suggested_tables_after_on_right_side(completer, complete_event): Document(text=text, cursor_position=position), complete_event)) assert result == list([ + Completion(text='users', start_position=0), Completion(text='orders', start_position=0), - Completion(text='users', start_position=0)]) + ]) def test_table_names_after_from(completer, complete_event): @@ -302,10 +306,10 @@ def test_table_names_after_from(completer, complete_event): Document(text=text, cursor_position=position), complete_event)) assert result == list([ - Completion(text='`réveillé`', start_position=0), - Completion(text='`select`', start_position=0), - Completion(text='orders', start_position=0), Completion(text='users', start_position=0), + Completion(text='orders', start_position=0), + Completion(text='`select`', start_position=0), + Completion(text='`réveillé`', start_position=0), ]) @@ -317,12 +321,12 @@ def test_auto_escaped_col_names(completer, complete_event): complete_event)) assert result == [ Completion(text='*', start_position=0), - Completion(text='`ABC`', start_position=0), - Completion(text='`insert`', start_position=0), Completion(text='id', start_position=0), + Completion(text='`insert`', start_position=0), + Completion(text='`ABC`', start_position=0), ] + \ list(map(Completion, completer.functions)) + \ - [Completion(text='`select`', start_position=0)] + \ + [Completion(text='select', start_position=0)] + \ list(map(Completion, completer.keywords)) @@ -334,9 +338,9 @@ def test_un_escaped_table_names(completer, complete_event): complete_event)) assert result == list([ Completion(text='*', start_position=0), - Completion(text='`ABC`', start_position=0), - Completion(text='`insert`', start_position=0), Completion(text='id', start_position=0), + Completion(text='`insert`', start_position=0), + Completion(text='`ABC`', start_position=0), ] + list(map(Completion, completer.functions)) + [Completion(text='réveillé', start_position=0)] + diff --git a/test/test_special_iocommands.py b/test/test_special_iocommands.py index 8b6be337..d0ca45ff 100644 --- a/test/test_special_iocommands.py +++ b/test/test_special_iocommands.py @@ -50,25 +50,49 @@ def test_editor_command(): os.environ['EDITOR'] = 'true' os.environ['VISUAL'] = 'true' - mycli.packages.special.open_external_editor(sql=r'select 1') == "select 1" + # Set the editor to Notepad on Windows + if os.name != 'nt': + mycli.packages.special.open_external_editor(sql=r'select 1') == "select 1" + else: + pytest.skip('Skipping on Windows platform.') + def test_tee_command(): mycli.packages.special.write_tee(u"hello world") # write without file set - with tempfile.NamedTemporaryFile() as f: + # keep Windows from locking the file with delete=False + with tempfile.NamedTemporaryFile(delete=False) as f: mycli.packages.special.execute(None, u"tee " + f.name) mycli.packages.special.write_tee(u"hello world") - assert f.read() == b"hello world\n" + if os.name=='nt': + assert f.read() == b"hello world\r\n" + else: + assert f.read() == b"hello world\n" mycli.packages.special.execute(None, u"tee -o " + f.name) mycli.packages.special.write_tee(u"hello world") f.seek(0) - assert f.read() == b"hello world\n" + if os.name=='nt': + assert f.read() == b"hello world\r\n" + else: + assert f.read() == b"hello world\n" mycli.packages.special.execute(None, u"notee") mycli.packages.special.write_tee(u"hello world") f.seek(0) - assert f.read() == b"hello world\n" + if os.name=='nt': + assert f.read() == b"hello world\r\n" + else: + assert f.read() == b"hello world\n" + + # remove temp file + # delete=False means we should try to clean up + try: + if os.path.exists(f.name): + os.remove(f.name) + except Exception as e: + print(f"An error occurred while attempting to delete the file: {e}") + def test_tee_command_error(): @@ -82,6 +106,8 @@ def test_tee_command_error(): @dbtest + +@pytest.mark.skipif(os.name == "nt", reason="Bug: fails on Windows, needs fixing, singleton of FQ not working right") def test_favorite_query(): with db_connection().cursor() as cur: query = u'select "✔"' @@ -98,16 +124,29 @@ def test_once_command(): mycli.packages.special.execute(None, u"\\once /proc/access-denied") mycli.packages.special.write_once(u"hello world") # write without file set - with tempfile.NamedTemporaryFile() as f: + # keep Windows from locking the file with delete=False + with tempfile.NamedTemporaryFile(delete=False) as f: mycli.packages.special.execute(None, u"\\once " + f.name) mycli.packages.special.write_once(u"hello world") - assert f.read() == b"hello world\n" + if os.name=='nt': + assert f.read() == b"hello world\r\n" + else: + assert f.read() == b"hello world\n" mycli.packages.special.execute(None, u"\\once -o " + f.name) mycli.packages.special.write_once(u"hello world line 1") mycli.packages.special.write_once(u"hello world line 2") f.seek(0) - assert f.read() == b"hello world line 1\nhello world line 2\n" + if os.name=='nt': + assert f.read() == b"hello world line 1\r\nhello world line 2\r\n" + else: + assert f.read() == b"hello world line 1\nhello world line 2\n" + # delete=False means we should try to clean up + try: + if os.path.exists(f.name): + os.remove(f.name) + except Exception as e: + print(f"An error occurred while attempting to delete the file: {e}") def test_pipe_once_command(): @@ -118,9 +157,14 @@ def test_pipe_once_command(): mycli.packages.special.execute( None, u"\\pipe_once /proc/access-denied") - mycli.packages.special.execute(None, u"\\pipe_once wc") - mycli.packages.special.write_once(u"hello world") - mycli.packages.special.unset_pipe_once_if_written() + if os.name == 'nt': + mycli.packages.special.execute(None, u'\\pipe_once python -c "import sys; print(len(sys.stdin.read().strip()))"') + mycli.packages.special.write_once(u"hello world") + mycli.packages.special.unset_pipe_once_if_written() + else: + mycli.packages.special.execute(None, u"\\pipe_once wc") + mycli.packages.special.write_once(u"hello world") + mycli.packages.special.unset_pipe_once_if_written() # how to assert on wc output? @@ -128,12 +172,21 @@ def test_parseargfile(): """Test that parseargfile expands the user directory.""" expected = {'file': os.path.join(os.path.expanduser('~'), 'filename'), 'mode': 'a'} - assert expected == mycli.packages.special.iocommands.parseargfile( - '~/filename') + + if os.name=='nt': + assert expected == mycli.packages.special.iocommands.parseargfile( + '~\\filename') + else: + assert expected == mycli.packages.special.iocommands.parseargfile( + '~/filename') expected = {'file': os.path.join(os.path.expanduser('~'), 'filename'), 'mode': 'w'} - assert expected == mycli.packages.special.iocommands.parseargfile( + if os.name=='nt': + assert expected == mycli.packages.special.iocommands.parseargfile( + '-o ~\\filename') + else: + assert expected == mycli.packages.special.iocommands.parseargfile( '-o ~/filename') @@ -162,6 +215,7 @@ def test_watch_query_iteration(): @dbtest +@pytest.mark.skipif(os.name == "nt", reason="Bug: Win handles this differently. May need to refactor watch_query to work for Win") def test_watch_query_full(): """Test that `watch_query`: diff --git a/test/test_sqlexecute.py b/test/test_sqlexecute.py index 38ca5ef6..ca186bcb 100644 --- a/test/test_sqlexecute.py +++ b/test/test_sqlexecute.py @@ -117,6 +117,7 @@ def test_multiple_queries_same_line_syntaxerror(executor): @dbtest +@pytest.mark.skipif(os.name == "nt", reason="Bug: fails on Windows, needs fixing, singleton of FQ not working right") def test_favorite_query(executor): set_expanded_output(False) run(executor, "create table test(a text)") @@ -136,6 +137,7 @@ def test_favorite_query(executor): @dbtest +@pytest.mark.skipif(os.name == "nt", reason="Bug: fails on Windows, needs fixing, singleton of FQ not working right") def test_favorite_query_multiple_statement(executor): set_expanded_output(False) run(executor, "create table test(a text)") @@ -159,6 +161,7 @@ def test_favorite_query_multiple_statement(executor): @dbtest +@pytest.mark.skipif(os.name == "nt", reason="Bug: fails on Windows, needs fixing, singleton of FQ not working right") def test_favorite_query_expanded_output(executor): set_expanded_output(False) run(executor, '''create table test(a text)''') @@ -195,16 +198,21 @@ def test_cd_command_without_a_folder_name(executor): @dbtest def test_system_command_not_found(executor): results = run(executor, 'system xyz') - assert_result_equal(results, status='OSError: No such file or directory', - assert_contains=True) + if os.name=='nt': + assert_result_equal(results, status='OSError: The system cannot find the file specified', + assert_contains=True) + else: + assert_result_equal(results, status='OSError: No such file or directory', + assert_contains=True) @dbtest def test_system_command_output(executor): + eol = os.linesep test_dir = os.path.abspath(os.path.dirname(__file__)) test_file_path = os.path.join(test_dir, 'test.txt') results = run(executor, 'system cat {0}'.format(test_file_path)) - assert_result_equal(results, status='mycli rocks!\n') + assert_result_equal(results, status=f'mycli rocks!{eol}') @dbtest @@ -276,7 +284,8 @@ def test_multiple_results(executor): @pytest.mark.parametrize( 'version_string, species, parsed_version_string, version', ( - ('5.7.25-TiDB-v6.1.0','TiDB', '5.7.25', 50725), + ('5.7.25-TiDB-v6.1.0','TiDB', '6.1.0', 60100), + ('8.0.11-TiDB-v7.2.0-alpha-69-g96e9e68daa', 'TiDB', '7.2.0', 70200), ('5.7.32-35', 'Percona', '5.7.32', 50732), ('5.7.32-0ubuntu0.18.04.1', 'MySQL', '5.7.32', 50732), ('10.5.8-MariaDB-1:10.5.8+maria~focal', 'MariaDB', '10.5.8', 100508), diff --git a/test/test_tabular_output.py b/test/test_tabular_output.py index c20c7de2..bdc1dbf0 100644 --- a/test/test_tabular_output.py +++ b/test/test_tabular_output.py @@ -102,7 +102,7 @@ def description(self): mycli.formatter.query = "SELECT * FROM `table`" output = mycli.format_output(None, FakeCursor(), headers) assert "\n".join(output) == dedent('''\ - INSERT INTO `table` (`letters`, `number`, `optional`, `float`, `binary`) VALUES + INSERT INTO table (`letters`, `number`, `optional`, `float`, `binary`) VALUES ('abc', 1, NULL, 10.0e0, X'aa') , ('d', 456, '1', 0.5e0, X'aabb') ;''') @@ -112,7 +112,7 @@ def description(self): mycli.formatter.query = "SELECT * FROM `database`.`table`" output = mycli.format_output(None, FakeCursor(), headers) assert "\n".join(output) == dedent('''\ - INSERT INTO `database`.`table` (`letters`, `number`, `optional`, `float`, `binary`) VALUES + INSERT INTO database.table (`letters`, `number`, `optional`, `float`, `binary`) VALUES ('abc', 1, NULL, 10.0e0, X'aa') , ('d', 456, '1', 0.5e0, X'aabb') ;''')