From 279680a28377f277b70a15442a6652d9ff29b7b1 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 23 Apr 2016 14:25:23 -0700 Subject: [PATCH 001/824] Enable history search via . --- mycli/key_bindings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index 94de541a9..3dc13b853 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -17,6 +17,7 @@ def mycli_bindings(get_key_bindings, set_key_bindings): key_binding_manager = KeyBindingManager( enable_open_in_editor=True, enable_system_bindings=True, + enable_search=True, enable_abort_and_exit_bindings=True, enable_vi_mode=Condition(lambda cli: get_key_bindings() == 'vi')) From c4b2d16ea96aef60709e0fe6c04ef6355d2e0d78 Mon Sep 17 00:00:00 2001 From: Jonathan Slenders Date: Fri, 6 May 2016 11:26:15 +0200 Subject: [PATCH 002/824] Upgrade to prompt-toolkit 1.0.0 --- mycli/clistyle.py | 17 +++++++---------- mycli/clitoolbar.py | 8 +++----- mycli/key_bindings.py | 15 ++++++--------- mycli/main.py | 22 +++++++++++----------- mycli/myclirc | 1 + setup.py | 2 +- 6 files changed, 29 insertions(+), 36 deletions(-) diff --git a/mycli/clistyle.py b/mycli/clistyle.py index a74df4506..a2a7b2a70 100644 --- a/mycli/clistyle.py +++ b/mycli/clistyle.py @@ -1,7 +1,6 @@ from pygments.token import string_to_tokentype -from pygments.style import Style from pygments.util import ClassNotFound -from prompt_toolkit.styles import default_style_extensions, PygmentsStyle +from prompt_toolkit.styles import default_style_extensions, style_from_dict import pygments.styles @@ -11,12 +10,10 @@ def style_factory(name, cli_style): except ClassNotFound: style = pygments.styles.get_style_by_name('native') - class CLIStyle(Style): - styles = {} + styles = {} + styles.update(style.styles) + styles.update(default_style_extensions) + custom_styles = dict([(string_to_tokentype(x), y) for x, y in cli_style.items()]) + styles.update(custom_styles) - styles.update(style.styles) - styles.update(default_style_extensions) - custom_styles = dict([(string_to_tokentype(x), y) for x, y in cli_style.items()]) - styles.update(custom_styles) - - return PygmentsStyle(CLIStyle) + return style_from_dict(styles) diff --git a/mycli/clitoolbar.py b/mycli/clitoolbar.py index 8abe948ab..b62d8edbe 100644 --- a/mycli/clitoolbar.py +++ b/mycli/clitoolbar.py @@ -1,12 +1,10 @@ from pygments.token import Token -from prompt_toolkit.enums import DEFAULT_BUFFER +from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode -def create_toolbar_tokens_func(get_key_bindings, get_is_refreshing): +def create_toolbar_tokens_func(get_is_refreshing): """ Return a function that generates the toolbar tokens. """ - assert callable(get_key_bindings) - token = Token.Toolbar def get_toolbar_tokens(cli): @@ -27,7 +25,7 @@ def get_toolbar_tokens(cli): result.append((token, ' (Semi-colon [;] will end the line)')) - if get_key_bindings() == 'vi': + if cli.editing_mode == EditingMode.VI: result.append((token.On, '[F4] Vi-mode')) else: result.append((token.On, '[F4] Emacs-mode')) diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index 3dc13b853..1651347e4 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -1,4 +1,5 @@ import logging +from prompt_toolkit.enums import EditingMode from prompt_toolkit.keys import Keys from prompt_toolkit.key_binding.manager import KeyBindingManager from prompt_toolkit.filters import Condition @@ -7,19 +8,15 @@ _logger = logging.getLogger(__name__) -def mycli_bindings(get_key_bindings, set_key_bindings): +def mycli_bindings(): """ Custom key bindings for mycli. """ - assert callable(get_key_bindings) - assert callable(set_key_bindings) - key_binding_manager = KeyBindingManager( enable_open_in_editor=True, enable_system_bindings=True, enable_search=True, - enable_abort_and_exit_bindings=True, - enable_vi_mode=Condition(lambda cli: get_key_bindings() == 'vi')) + enable_abort_and_exit_bindings=True) @key_binding_manager.registry.add_binding(Keys.F2) def _(event): @@ -45,10 +42,10 @@ def _(event): Toggle between Vi and Emacs mode. """ _logger.debug('Detected F4 key.') - if get_key_bindings() == 'vi': - set_key_bindings('emacs') + if event.cli.editing_mode == EditingMode.VI: + event.cli.editing_mode = EditingMode.EMACS else: - set_key_bindings('vi') + event.cli.editing_mode = EditingMode.VI @key_binding_manager.registry.add_binding(Keys.Tab) def _(event): diff --git a/mycli/main.py b/mycli/main.py index 1ea642f67..cc42d6e82 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -17,7 +17,7 @@ import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction from prompt_toolkit.interface import AcceptAction -from prompt_toolkit.enums import DEFAULT_BUFFER +from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode from prompt_toolkit.shortcuts import create_prompt_layout, create_eventloop from prompt_toolkit.document import Document from prompt_toolkit.filters import Always, HasFocus, IsDone @@ -420,17 +420,12 @@ def run_cli(self): self.refresh_completions() - def set_key_bindings(value): - if value not in ('emacs', 'vi'): - value = 'emacs' - self.key_bindings = value - project_root = os.path.dirname(PACKAGE_ROOT) author_file = os.path.join(project_root, 'AUTHORS') sponsor_file = os.path.join(project_root, 'SPONSORS') - key_binding_manager = mycli_bindings(get_key_bindings=lambda: self.key_bindings, - set_key_bindings=set_key_bindings) + key_binding_manager = mycli_bindings() + print('Version:', __version__) print('Chat: https://gitter.im/dbcli/mycli') print('Mail: https://groups.google.com/forum/#!forum/mycli-users') @@ -443,8 +438,7 @@ def prompt_tokens(cli): def get_continuation_tokens(cli, width): return [(Token.Continuation, ' ' * (width - 3) + '-> ')] - get_toolbar_tokens = create_toolbar_tokens_func(lambda: self.key_bindings, - self.completion_refresher.is_refreshing) + get_toolbar_tokens = create_toolbar_tokens_func(self.completion_refresher.is_refreshing) layout = create_prompt_layout(lexer=MyCliLexer, multiline=True, @@ -462,18 +456,24 @@ def get_continuation_tokens(cli, width): history=FileHistory(os.path.expanduser('~/.mycli-history')), complete_while_typing=Always(), accept_action=AcceptAction.RETURN_DOCUMENT) + if self.key_bindings == 'vi': + editing_mode = EditingMode.VI + else: + editing_mode = EditingMode.EMACS + application = Application(style=style_factory(self.syntax_style, self.cli_style), layout=layout, buffer=buf, key_bindings_registry=key_binding_manager.registry, on_exit=AbortAction.RAISE_EXCEPTION, on_abort=AbortAction.RETRY, + editing_mode=editing_mode, ignore_case=True) self.cli = CommandLineInterface(application=application, eventloop=create_eventloop()) try: while True: - document = self.cli.run() + document = self.cli.run(reset_current_buffer=True) special.set_expanded_output(False) diff --git a/mycli/myclirc b/mycli/myclirc index c59e621c7..0c162e9d3 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -75,6 +75,7 @@ Token.SearchMatch = '#ffffff bg:#4444aa' Token.SearchMatch.Current = '#ffffff bg:#44aa44' # The bottom toolbar. +Token.Toolbar = 'bg:#222222 #aaaaaa' Token.Toolbar.Off = 'bg:#222222 #888888' Token.Toolbar.On = 'bg:#222222 #ffffff' diff --git a/setup.py b/setup.py index b82957543..1876c6a62 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ install_requirements = [ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? - 'prompt_toolkit==0.60', + 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', 'sqlparse >= 0.1.19', 'configobj >= 5.0.6', From ca9ef2df459f5020f41ceaa6496af201a040a300 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 11 May 2016 23:07:27 -0700 Subject: [PATCH 003/824] Update changelog for release 1.7.0. --- changelog.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/changelog.md b/changelog.md index 759c3b4d8..48a148b49 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,20 @@ +1.7.0: +====== + +Features: +--------- + +* Add stdin batch mode. (Thanks: [Thomas Roten]). +* Add warn/no-warn command-line options. (Thanks: [Thomas Roten]). +* Upgrade sqlparse dependency to 0.1.19. (Thanks: [Amjith Ramanujam]). +* Update features list in README.md. (Thanks: [Matheus Rosa]). +* Remove extra \n in features list in README.md. (Thanks: [Matheus Rosa]). + +Bug Fixes: +---------- + +* Enable history search via . (Thanks: [Amjith Ramanujam]). + 1.6.0: ====== From 01bc2c820b088bd1a0f89ea8f6bc2751b5b2b58a Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 12 May 2016 09:32:19 -0700 Subject: [PATCH 004/824] Update changelog for release 1.7.0. --- changelog.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 48a148b49..0964af18a 100644 --- a/changelog.md +++ b/changelog.md @@ -15,6 +15,11 @@ Bug Fixes: * Enable history search via . (Thanks: [Amjith Ramanujam]). +Internal Changes: +----------------- + +* Upgrade prompt_toolkit to 1.0.0. (Thanks: [Jonathan Slenders]) + 1.6.0: ====== @@ -140,7 +145,7 @@ Features: * Add custom styles to color the menus and toolbars. -* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders](https://github.com/jonathanslenders)) +* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders]) Multi-line queries are automatically indented. @@ -310,3 +315,4 @@ Bug Fixes: [Phil Cohen]: https://github.com/phlipper [Terseus]: https://github.com/Terseus [William GARCIA]: https://github.com/willgarcia +[Jonathan Slenders]: https://github.com/jonathanslenders From 3f1e1bc27e8c91c6f9c78db0352624d978753276 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 12 May 2016 21:15:23 -0700 Subject: [PATCH 005/824] Releasing version 1.7.0 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index bcd8d54ef..0e1a38d3c 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.6.0' +__version__ = '1.7.0' From 08d3939a9eb5676c8ee546cf3dcd62c09a78b7ce Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 24 May 2016 11:11:07 -0700 Subject: [PATCH 006/824] Releasing version 1.7.1 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index 0e1a38d3c..48c2f6b0b 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.7.0' +__version__ = '1.7.1' From 6396a3f522c2ce3ac0752174dd3aa8e4e2b4a936 Mon Sep 17 00:00:00 2001 From: Casper Langemeijer Date: Fri, 27 May 2016 12:29:39 +0200 Subject: [PATCH 007/824] New debian release (1.7.0) --- debian/changelog | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/debian/changelog b/debian/changelog index 18da86c59..5135bf68a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,39 @@ +mycli (1.7.0) unstable; urgency=medium + + * Add stdin batch mode. (Thanks: Thomas Roten). + * Add warn/no-warn command-line options. (Thanks: Thomas Roten). + * Upgrade sqlparse dependency to 0.1.19. (Thanks: [Amjith Ramanujam]). + * Update features list in README.md. (Thanks: Matheus Rosa). + * Remove extra \n in features list in README.md. (Thanks: Matheus Rosa). + * Enable history search via . (Thanks: [Amjith Ramanujam]). + * Upgrade prompt_toolkit to 1.0.0. (Thanks: Jonathan Slenders) + + -- Casper Langemeijer Fri, 27 May 2016 12:03:31 +0200 + +mycli (1.6.0) unstable; urgency=medium + + * Change continuation prompt for multi-line mode to match default mysql. + * Add status command to match mysql's status command. (Thanks: Thomas Roten). + * Add SSL support for mycli. (Thanks: Artem Bezsmertnyi). + * Add auto-completion and highlight support for OFFSET keyword. (Thanks: Matheus Rosa). + * Add support for MYSQL_TEST_LOGIN_FILE env variable to specify alternate login file. (Thanks: Thomas Roten). + * Add support for --auto-vertical-output to automatically switch to vertical output if the output doesn't fit in the table format. + * Add support for system-wide config. Now /etc/myclirc will be honored. (Thanks: Thomas Roten). + * Add support for nopager and \n to turn off the pager. (Thanks: Thomas Roten). + * Add support for --local-infile command-line option. (Thanks: Thomas Roten). + * Remove -S from less option which was clobbering the scroll back in history. (Thanks: Thomas Roten). + * Make system command work with Python 3. (Thanks: Thomas Roten). + * Support \G terminator for \f queries. (Thanks: Terseus). + * Upgrade prompt_toolkit to 0.60. + * Add Python 3.5 to test environments. (Thanks: Thomas Roten). + * Remove license meta-data. (Thanks: Thomas Roten). + * Skip binary tests if PyMySQL version does not support it. (Thanks: Thomas Roten). + * Refactor pager handling. (Thanks: Thomas Roten) + * Capture warnings to log file. (Thanks: Mikhail Borisov). + * Make syntax_style a tiny bit more intuitive. (Thanks: Phil Cohen). + + -- Casper Langemeijer Fri, 27 May 2016 12:03:31 +0200 + mycli (1.5.2) unstable; urgency=low * Protect against port number being None when no port is specified in command line. From 1f27566497c33d54b8900c483867ee72ce0a6932 Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Sun, 29 May 2016 21:24:12 +0200 Subject: [PATCH 008/824] add skip_intro config --- mycli/main.py | 12 +++++++----- mycli/myclirc | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index cc42d6e82..8ef4ad691 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -107,6 +107,7 @@ def __init__(self, sqlexecute=None, prompt=None, special.set_timing_enabled(c['main'].as_bool('timing')) self.table_format = c['main']['table_format'] self.syntax_style = c['main']['syntax_style'] + self.skip_intro = c['main'].as_bool('skip_intro') self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') c_dest_warning = c['main'].as_bool('destructive_warning') @@ -426,11 +427,12 @@ def run_cli(self): key_binding_manager = mycli_bindings() - print('Version:', __version__) - print('Chat: https://gitter.im/dbcli/mycli') - print('Mail: https://groups.google.com/forum/#!forum/mycli-users') - print('Home: http://mycli.net') - print('Thanks to the contributor -', thanks_picker([author_file, sponsor_file])) + if not self.skip_intro: + print('Version:', __version__) + print('Chat: https://gitter.im/dbcli/mycli') + print('Mail: https://groups.google.com/forum/#!forum/mycli-users') + print('Home: http://mycli.net') + print('Thanks to the contributor -', thanks_picker([author_file, sponsor_file])) def prompt_tokens(cli): return [(Token.Prompt, self.get_prompt(self.prompt_format))] diff --git a/mycli/myclirc b/mycli/myclirc index 0c162e9d3..f0b29376a 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -58,6 +58,9 @@ wider_completion_menu = False # \n - Newline prompt = '\t \u@\h:\d> ' +# Skip intro info on startup. +skip_intro = False + # Custom colors for the completion menu, toolbar, etc. [colors] # Completion menus. From 1f4f4249b9b681b272e7464e4b48c24e64256526 Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Sun, 29 May 2016 21:39:28 +0200 Subject: [PATCH 009/824] update skip_intro to include outro, and rename to less_chatty --- mycli/main.py | 7 ++++--- mycli/myclirc | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 8ef4ad691..1b220fb80 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -107,7 +107,7 @@ def __init__(self, sqlexecute=None, prompt=None, special.set_timing_enabled(c['main'].as_bool('timing')) self.table_format = c['main']['table_format'] self.syntax_style = c['main']['syntax_style'] - self.skip_intro = c['main'].as_bool('skip_intro') + self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') c_dest_warning = c['main'].as_bool('destructive_warning') @@ -427,7 +427,7 @@ def run_cli(self): key_binding_manager = mycli_bindings() - if not self.skip_intro: + if not self.less_chatty: print('Version:', __version__) print('Chat: https://gitter.im/dbcli/mycli') print('Mail: https://groups.google.com/forum/#!forum/mycli-users') @@ -612,7 +612,8 @@ def get_continuation_tokens(cli, width): self.query_history.append(query) except EOFError: - self.output('Goodbye!') + if not self.less_chatty: + self.output('Goodbye!') def output(self, text, **kwargs): if self.logfile: diff --git a/mycli/myclirc b/mycli/myclirc index f0b29376a..f5f0dd979 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -58,8 +58,8 @@ wider_completion_menu = False # \n - Newline prompt = '\t \u@\h:\d> ' -# Skip intro info on startup. -skip_intro = False +# Skip intro info on startup and outro info on exit +less_chatty = False # Custom colors for the completion menu, toolbar, etc. [colors] From a415fbdd7b4391bd8fa53613a16a787aad164158 Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Sun, 29 May 2016 21:46:23 +0200 Subject: [PATCH 010/824] add log level NONE to use a no-op logging handler --- mycli/main.py | 13 ++++++++++++- mycli/myclirc | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 1b220fb80..e5e1458dc 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -61,6 +61,11 @@ PACKAGE_ROOT = os.path.dirname(__file__) +# no-op logging handler +class NullHandler(logging.Handler): + def emit(self, record): + pass + class MyCli(object): default_prompt = '\\t \\u@\\h:\\d> ' @@ -235,7 +240,13 @@ def initialize_logging(self): 'DEBUG': logging.DEBUG } - handler = logging.FileHandler(os.path.expanduser(log_file)) + # Disable logging if value is NONE by switching to a no-op handler + # Set log level to a high value so it doesn't even waste cycles getting called. + if log_level.upper() == "NONE": + handler = NullHandler() + log_level = "CRITICAL" + else: + handler = logging.FileHandler(os.path.expanduser(log_file)) formatter = logging.Formatter( '%(asctime)s (%(process)d/%(threadName)s) ' diff --git a/mycli/myclirc b/mycli/myclirc index f5f0dd979..7000b573e 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -20,7 +20,7 @@ destructive_warning = True log_file = ~/.mycli.log # Default log level. Possible values: "CRITICAL", "ERROR", "WARNING", "INFO" -# and "DEBUG". +# and "DEBUG". "NONE" disables logging. log_level = INFO # Log every query and its results to a file. Enable this by uncommenting the From 73a5732f96d99307328b6458e44b812e997aea7b Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Sun, 29 May 2016 22:31:26 +0200 Subject: [PATCH 011/824] mimic mysql and support MYCLI_HISTFILE environment variable --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index e5e1458dc..75a344b90 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -466,7 +466,7 @@ def get_continuation_tokens(cli, width): ]) with self._completer_lock: buf = CLIBuffer(always_multiline=self.multi_line, completer=self.completer, - history=FileHistory(os.path.expanduser('~/.mycli-history')), + history=FileHistory(os.path.expanduser(os.environ.get('MYCLI_HISTFILE', '~/.mycli-history'))), complete_while_typing=Always(), accept_action=AcceptAction.RETURN_DOCUMENT) if self.key_bindings == 'vi': From 012d173867b31dbf86d94ecd73120eb553e2a963 Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Thu, 9 Jun 2016 15:05:43 +0200 Subject: [PATCH 012/824] add prompt_continuation support --- mycli/main.py | 4 +++- mycli/myclirc | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 75a344b90..26ca86795 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -138,6 +138,7 @@ def __init__(self, sqlexecute=None, prompt=None, prompt_cnf = self.read_my_cnf_files(self.cnf_files, ['prompt'])['prompt'] self.prompt_format = prompt or prompt_cnf or c['main']['prompt'] or \ self.default_prompt + self.prompt_continuation_format = c['main']['prompt_continuation'] self.query_history = [] @@ -449,7 +450,8 @@ def prompt_tokens(cli): return [(Token.Prompt, self.get_prompt(self.prompt_format))] def get_continuation_tokens(cli, width): - return [(Token.Continuation, ' ' * (width - 3) + '-> ')] + continuation_prompt = self.get_prompt(self.prompt_continuation_format) + return [(Token.Continuation, ' ' * (width - len(continuation_prompt)) + continuation_prompt)] get_toolbar_tokens = create_toolbar_tokens_func(self.completion_refresher.is_refreshing) diff --git a/mycli/myclirc b/mycli/myclirc index 7000b573e..ee3490770 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -57,6 +57,7 @@ wider_completion_menu = False # \d - Database name # \n - Newline prompt = '\t \u@\h:\d> ' +prompt_continuation = '-> ' # Skip intro info on startup and outro info on exit less_chatty = False From 3623eab6ca31df567870fa1374ae67356af37517 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Mon, 13 Jun 2016 16:01:42 -0700 Subject: [PATCH 013/824] Display login-path instead of host in prompt. Connect #271. --- mycli/main.py | 6 ++++-- mycli/myclirc | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 26ca86795..5a8541f6c 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -80,7 +80,7 @@ class MyCli(object): ] system_config_files = [ - '/etc/myclirc', + '/etc/myclirc', ] default_config_file = os.path.join(PACKAGE_ROOT, 'myclirc') @@ -117,6 +117,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') c_dest_warning = c['main'].as_bool('destructive_warning') self.destructive_warning = c_dest_warning if warn is None else warn + self.login_path_as_host = c['main'].as_bool('login_path_as_host') # Write user config if system config wasn't the last config loaded. if c.filename not in self.system_config_files: @@ -686,8 +687,9 @@ def get_completions(self, text, cursor_positition): def get_prompt(self, string): sqlexecute = self.sqlexecute + host = self.login_path if self.login_path and self.login_path_as_host else sqlexecute.host string = string.replace('\\u', sqlexecute.user or '(none)') - string = string.replace('\\h', sqlexecute.host or '(none)') + string = string.replace('\\h', host or '(none)') string = string.replace('\\d', sqlexecute.dbname or '(none)') string = string.replace('\\t', sqlexecute.server_type()[0] or 'mycli') string = string.replace('\\n', "\n") diff --git a/mycli/myclirc b/mycli/myclirc index ee3490770..3a6911c72 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -62,6 +62,9 @@ prompt_continuation = '-> ' # Skip intro info on startup and outro info on exit less_chatty = False +# Use alias from --login-path instead of host name in prompt +login_path_as_host = False + # Custom colors for the completion menu, toolbar, etc. [colors] # Completion menus. From 0fee40c7b8c8d425a0ab29d6e341049d4d4241e2 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Mon, 13 Jun 2016 16:58:25 -0700 Subject: [PATCH 014/824] Completion inside function can handle operands. --- mycli/packages/completion_engine.py | 4 +++- tests/test_completion_engine.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 9f6a2c321..91a22c90a 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -133,6 +133,8 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier else: token_v = token.value.lower() + is_operand = lambda x: x and any([x.endswith(op) for op in ['+', '-', '*', '/']]) + if not token: return [{'type': 'keyword'}, {'type': 'special'}] elif token_v.endswith('('): @@ -270,7 +272,7 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier return [{'type': 'database'}] elif token_v == 'tableformat': return [{'type': 'table_format'}] - elif token_v.endswith(',') or token_v in ['=', 'and', 'or']: + elif token_v.endswith(',') or is_operand(token_v) or token_v in ['=', 'and', 'or']: prev_keyword, text_before_cursor = find_prev_keyword(text_before_cursor) if prev_keyword: return suggest_based_on_last_token( diff --git a/tests/test_completion_engine.py b/tests/test_completion_engine.py index ee1c46e2d..294fdbd56 100644 --- a/tests/test_completion_engine.py +++ b/tests/test_completion_engine.py @@ -67,6 +67,16 @@ def test_lparen_suggests_cols(): assert suggestion == [ {'type': 'column', 'tables': [(None, 'tbl', None)]}] +def test_operand_inside_function_suggests_cols1(): + suggestion = suggest_type('SELECT MAX(col1 + FROM tbl', 'SELECT MAX(col1 + ') + assert suggestion == [ + {'type': 'column', 'tables': [(None, 'tbl', None)]}] + +def test_operand_inside_function_suggests_cols2(): + suggestion = suggest_type('SELECT MAX(col1 + col2 + FROM tbl', 'SELECT MAX(col1 + col2 + ') + assert suggestion == [ + {'type': 'column', 'tables': [(None, 'tbl', None)]}] + def test_select_suggests_cols_and_funcs(): suggestions = suggest_type('SELECT ', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ From 08a65279eb5eeb4b29c8542201dea533f18c5aaf Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Wed, 15 Jun 2016 01:07:46 -0300 Subject: [PATCH 015/824] Remove unsupported keywords This commit remove keywords that are not supported by MySQL 5.7, 5.5 and 5.6. --- mycli/sqlcompleter.py | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index ccc1dabc8..599dc9e52 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -19,32 +19,23 @@ class SQLCompleter(Completer): keywords = ['ACCESS', 'ADD', 'ALL', 'ALTER TABLE', 'AND', 'ANY', 'AS', - 'ASC', 'AUDIT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', - 'CASE', 'CHANGE MASTER TO', 'CHAR', 'CHECK', 'CLUSTER', - 'COLUMN', 'COMMENT', 'COMPRESS', 'COMMIT', 'CONNECT', 'COPY', - 'CREATE', 'CURRENT', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', - 'DELETE FROM', 'DELIMITER', 'DESC', 'DESCRIBE', 'DISTINCT', - 'DROP', 'ELSE', 'ENCODING', 'END', 'ESCAPE', 'EXCLUSIVE', - 'EXISTS', 'EXTENSION', 'FILE', 'FLOAT', 'FOR', 'FORMAT', - 'FORCE_QUOTE', 'FORCE_NOT_NULL', 'FREEZE', 'FROM', 'FULL', - 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HEADER', 'HOST', - 'IDENTIFIED', 'IMMEDIATE', 'IN', 'INCREMENT', 'INDEX', - 'INITIAL', 'INSERT INTO', 'INTEGER', 'INTERSECT', 'INTO', - 'INTERVAL', 'IS', 'JOIN', 'LEFT', 'LEVEL', 'LIKE', 'LIMIT', - 'LOCK', 'LOG', 'LOGS', 'LONG', 'MASTER', 'MINUS', 'MODE', - 'MODIFY', 'NOAUDIT', 'NOCOMPRESS', 'NOT', 'NOWAIT', 'NULL', - 'NUMBER', 'OIDS', 'OF', 'OFFLINE', 'OFFSET', 'ON', 'ONLINE', - 'OPTION', 'OR', 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', - 'PCTFREE', 'PORT', 'PRIMARY', 'PRIOR', 'PRIVILEGES', - 'PROCESSLIST', 'PURGE', 'QUOTE', 'RAW', 'RENAME', 'REPAIR', - 'RESOURCE', 'RESET', 'REVOKE', 'RIGHT', 'ROLLBACK', 'ROW', - 'ROWID', 'ROWNUM', 'ROWS', 'SELECT', 'SESSION', 'SET', 'SHARE', - 'SHOW', 'SIZE', 'SLAVE', 'SLAVES', 'SMALLINT', 'START', 'STOP', - 'SUCCESSFUL', 'SYNONYM', 'SYSDATE', 'TABLE', 'TEMPLATE', - 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UID', - 'UNION', 'UNIQUE', 'UPDATE', 'USE', 'USER', 'USING', - 'VALIDATE', 'VALUES', 'VARCHAR', 'VARCHAR2', 'VIEW', 'WHEN', - 'WHENEVER', 'WHERE', 'WITH'] + 'ASC', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', + 'CASE', 'CHAR', 'CHECK', 'COLUMN', 'COMMENT', 'COMMIT', + 'CHANGE MASTER TO', 'CREATE', 'CURRENT', 'DATABASE', 'DATE', + 'DECIMAL', 'DEFAULT', 'DELETE FROM', 'DELIMITER', 'DESC', + 'DESCRIBE', 'DISTINCT', 'DROP', 'ELSE', 'END', 'ESCAPE', 'EXISTS', + 'FILE', 'FLOAT', 'FOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', + 'GROUP BY', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', 'INCREMENT', 'INDEX', + 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'LEFT', + 'LEVEL', 'LIKE', 'LIMIT', 'LOCK', 'LOGS', 'LONG', 'MASTER', 'MODE', + 'MODIFY', 'NOT', 'NULL', 'NUMBER', 'OFFSET', 'ON', 'OPTION', 'OR', + 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', 'PORT', 'PRIMARY', + 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'RENAME', 'REPAIR', 'RESET', + 'REVOKE', 'RIGHT', 'ROLLBACK','ROW', 'ROWS', 'SELECT', 'SESSION', 'SET', + 'SHARE', 'SHOW', 'SLAVE', 'SMALLINT', 'START', 'STOP', 'TABLE', 'THEN', + 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UNION', 'UNIQUE', 'UPDATE', + 'USE', 'USER', 'USING', 'VALUES', 'VARCHAR', 'VIEW', 'WHEN', 'WHERE', + 'WITH'] functions = ['AVG', 'COUNT', 'DISTINCT', 'FIRST', 'FORMAT', 'LAST', 'LCASE', 'LEN', 'MAX', 'MIN', 'MID', 'NOW', 'ROUND', 'SUM', From e17c44f347dd1eec0f4c9eddea7da2320c4a0e2d Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 18 Jul 2016 00:29:10 -0300 Subject: [PATCH 016/824] Add support for --execute param --- mycli/main.py | 34 +++++++++++++++++++++++++--------- tests/test_main.py | 12 ++++++++++++ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 5a8541f6c..4330a195b 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -695,6 +695,16 @@ def get_prompt(self, string): string = string.replace('\\n', "\n") return string + def run_query(self, query, table_format=None): + """Runs query""" + results = self.sqlexecute.run(query) + for result in results: + title, cur, headers, status = result + table_format = self.table_format if table_format else None + output = format_output(title, cur, headers, None, table_format) + for line in output: + click.echo(line) + @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') @click.option('-P', '--port', envvar='MYSQL_TCP_PORT', type=int, help='Port number to use for connection. Honors ' @@ -740,11 +750,13 @@ def get_prompt(self, string): help='Enable/disable LOAD DATA LOCAL INFILE.') @click.option('--login-path', type=str, help='Read this path from the login file.') +@click.option('--execute', type=str, help='Execute query to the database.') @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, login_path, auto_vertical_output, local_infile, ssl_ca, ssl_capath, - ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, warn): + ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, warn, + execute): if version: print('Version:', __version__) @@ -781,6 +793,15 @@ def cli(database, user, host, port, socket, password, dbname, '\thost: %r' '\tport: %r', database, user, host, port) + # --execute argument + if execute: + try: + mycli.run_query(execute, table_format=table) + exit(0) + except Exception as e: + click.secho(str(e), err=True, fg='red') + exit(1) + if sys.stdin.isatty(): mycli.run_cli() else: @@ -796,17 +817,12 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - results = mycli.sqlexecute.run(stdin_text) - for result in results: - title, cur, headers, status = result - table_format = mycli.table_format if table else None - output = format_output(title, cur, headers, None, table_format) - for line in output: - click.echo(line) + mycli.run_query(stdin_text, table_format=table) except Exception as e: click.secho(str(e), err=True, fg='red') exit(1) + def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): output = [] if title: # Only print the title if it's not None. @@ -818,7 +834,7 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max elif table_format is not None: rows = list(cur) tabulated, frows = tabulate(rows, headers, tablefmt=table_format, - missingval='') + missingval='') if (max_width and rows and content_exceeds_width(frows[0], max_width) and headers): diff --git a/tests/test_main.py b/tests/test_main.py index 22ef48d0c..d2c351417 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -33,6 +33,18 @@ def test_format_output_no_table(): expected = ['Title', 'head1\thead2', 'abc\tdef', 'test status'] assert results == expected +@dbtest +def test_execute_arg(executor): + run(executor, 'create table test (a text)') + run(executor, 'insert into test values("abc")') + + sql = 'select * from test;' + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['--execute', sql]) + + assert result.exit_code == 0 + assert 'abc' in result.output + @dbtest def test_batch_mode(executor): run(executor, '''create table test(a text)''') From afbeec030ce5d73a42dfd01de59ba0da520607fc Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 18 Jul 2016 00:29:40 -0300 Subject: [PATCH 017/824] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 0cab00de1..12588dc82 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ .vagrant *.pyc *.deb +*.swp +.cache/ From 0e13dfcfb59e92adc132dabadb4740e8a18c97bc Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Tue, 19 Jul 2016 19:35:20 -0300 Subject: [PATCH 018/824] Add '-e' as an option for the '--execute' argument --- mycli/main.py | 3 ++- tests/test_main.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 4330a195b..82d817f66 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -750,7 +750,8 @@ def run_query(self, query, table_format=None): help='Enable/disable LOAD DATA LOCAL INFILE.') @click.option('--login-path', type=str, help='Read this path from the login file.') -@click.option('--execute', type=str, help='Execute query to the database.') +@click.option('-e', '--execute', type=str, + help='Execute query to the database.') @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, diff --git a/tests/test_main.py b/tests/test_main.py index d2c351417..f174a45e1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -40,11 +40,17 @@ def test_execute_arg(executor): sql = 'select * from test;' runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['-e', sql]) + + assert result.exit_code == 0 + assert 'abc' in result.output + result = runner.invoke(cli, args=CLI_ARGS + ['--execute', sql]) assert result.exit_code == 0 assert 'abc' in result.output + @dbtest def test_batch_mode(executor): run(executor, '''create table test(a text)''') From 9688f652d8a4d730426e40e66527c94b7f16f763 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 31 Jul 2016 22:28:47 -0700 Subject: [PATCH 019/824] Pin sqlparse to 0.1.19 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1876c6a62..f21bdb4c9 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse >= 0.1.19', + 'sqlparse == 0.1.19', 'configobj >= 5.0.6', ] From 91055a2b1a90099e16459c0e258f7647a6a52dd7 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 1 Aug 2016 07:23:58 -0700 Subject: [PATCH 020/824] Changelog and authors update for 1.8.0 --- AUTHORS | 23 ++++++++++++----------- changelog.md | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index a6552890c..85a53e67b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -3,39 +3,40 @@ Many thanks to the following contributors. Core Developers: ---------------- - * Iryna Cherniavska * Thomas Roten - * Darik Gamble + * Iryna Cherniavska * Matheus Rosa + * Darik Gamble Contributors: ------------- * Steve Robbins - * Daniel West * Shoma Suzuki + * Daniel West + * Scrappy Soft * Daniel Black * Jonathan Bruno + * Casper Langemeijer + * Jonathan Slenders * Artem Bezsmertnyi - * Heath Naylor * Mikhail Borisov + * Heath Naylor * Phil Cohen - * Yasuhiro Matsumoto - * bjarnagin - * jbruno - * mrdeathless - * Abirami P * spacewander * Adam Chainz - * Casper Langemeijer * Johannes Hoff - * Jonathan Slenders * Kacper Kwapisz * Lennart Weller * Martijn Engler * Terseus * Tyler Kuipers * William GARCIA + * Yasuhiro Matsumoto + * bjarnagin + * jbruno + * mrdeathless + * Abirami P Creator: -------- diff --git a/changelog.md b/changelog.md index 0964af18a..91cf208d7 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,22 @@ +1.8.0: +====== + +Features: +--------- + +* Add support for --execute/-e commandline arg. (Thanks: [Matheus Rosa]). +* Add `less_chatty` config option to skip the intro messages. (Thanks: [Scrappy Soft]). +* Support MYCLI_HISTFILE environment variable to specify where to write the history file. (Thanks: [Scrappy Soft]). +* Add `prompt_continuation` config option to allow configuring the continuation prompt for multi-line queries. (Thanks: [Scrappy Soft]). +* Display login-path instead of host in prompt. (Thanks: [Irina Truong]). + +Bug Fixes: +---------- + +* Pin sqlparse to version 0.1.19 since the new version is breaking completion. (Thanks: [Amjith Ramanujam]). +* Remove unsupported keywords. (Thanks: [Matheus Rosa]). +* Fix completion suggestion inside functions with operands. (Thanks: [Irina Truong]). + 1.7.0: ====== @@ -316,3 +335,5 @@ Bug Fixes: [Terseus]: https://github.com/Terseus [William GARCIA]: https://github.com/willgarcia [Jonathan Slenders]: https://github.com/jonathanslenders +[Casper Langemeijer]: https://github.com/langemeijer +[Scrappy Soft]: https://github.com/scrappysoft From 9104a19d5b8e8a39697902bb5a5e3532aef0b1ac Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 3 Aug 2016 11:13:11 -0700 Subject: [PATCH 021/824] Releasing version 1.8.0 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index 48c2f6b0b..b28097579 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.7.1' +__version__ = '1.8.0' From 25316468b821ca14f175f6746c439d873f565129 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 9 Aug 2016 06:34:48 -0700 Subject: [PATCH 022/824] Reset the show items when completion is refreshed. --- mycli/sqlcompleter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 599dc9e52..e92332738 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -192,6 +192,7 @@ def set_dbname(self, dbname): def reset_completions(self): self.databases = [] + self.show_items = [] self.dbname = '' self.dbmetadata = {'tables': {}, 'views': {}, 'functions': {}} self.all_completions = set(self.keywords + self.functions) From eb6b8aa2430d11e08c6234c2c0841d3965f22904 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 19 Jul 2016 22:11:57 -0700 Subject: [PATCH 023/824] Custom converters for time/date types. Having invalid time/date values in database causes mycli to display their values as . Having custom converters will handle that case. --- mycli/sqlexecute.py | 11 ++++++++- tests/test_sqlexecute.py | 52 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index de977fe6f..38116496d 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -2,6 +2,9 @@ import pymysql import sqlparse from .packages import connection, special +from pymysql.constants import FIELD_TYPE +from pymysql.converters import (convert_mysql_timestamp, convert_datetime, + convert_timedelta, convert_date) _logger = logging.getLogger(__name__) @@ -61,13 +64,19 @@ def connect(self, database=None, user=None, password=None, host=None, '\tlocal_infile: %r' '\tssl: %r', database, user, host, port, socket, charset, local_infile, ssl) + conv = { + FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or '0000-00-00 00:00:00'), + FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or '0000-00-00 00:00:00'), + FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or '00:00:00'), + FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or '0000-00-00'), + } conn = connection.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, use_unicode=True, charset=charset, autocommit=True, client_flag=pymysql.constants.CLIENT.INTERACTIVE, cursorclass=connection.Cursor, local_infile=local_infile, - ssl=ssl) + conv=conv, ssl=ssl) if hasattr(self, 'conn'): self.conn.close() self.conn = conn diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index 4a6f99101..69b9a6d26 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -277,3 +277,55 @@ def test_favorite_query_multiline_statement(executor): results = run(executor, "\\fd test-ad") assert results == ['test-ad: Deleted'] + +@dbtest +def test_timestamp_null(executor): + run(executor, '''create table ts_null(a timestamp)''') + run(executor, '''insert into ts_null values(0)''') + results = run(executor, '''select * from ts_null''', join=True) + assert results == dedent("""\ + +---------------------+ + | a | + |---------------------| + | 0000-00-00 00:00:00 | + +---------------------+ + 1 row in set""") + +@dbtest +def test_datetime_null(executor): + run(executor, '''create table dt_null(a datetime)''') + run(executor, '''insert into dt_null values(0)''') + results = run(executor, '''select * from dt_null''', join=True) + assert results == dedent("""\ + +---------------------+ + | a | + |---------------------| + | 0000-00-00 00:00:00 | + +---------------------+ + 1 row in set""") + +@dbtest +def test_date_null(executor): + run(executor, '''create table date_null(a date)''') + run(executor, '''insert into date_null values(0)''') + results = run(executor, '''select * from date_null''', join=True) + assert results == dedent("""\ + +------------+ + | a | + |------------| + | 0000-00-00 | + +------------+ + 1 row in set""") + +@dbtest +def test_time_null(executor): + run(executor, '''create table time_null(a time)''') + run(executor, '''insert into time_null values(0)''') + results = run(executor, '''select * from time_null''', join=True) + assert results == dedent("""\ + +----------+ + | a | + |----------| + | 00:00:00 | + +----------+ + 1 row in set""") From e3c41835a84a6371bd25f574cf7a6e82fd6cbe1f Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 9 Aug 2016 06:38:17 -0700 Subject: [PATCH 024/824] Fallback to the raw object for invalid time values. --- mycli/sqlexecute.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 38116496d..d44a11fec 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -65,10 +65,10 @@ def connect(self, database=None, user=None, password=None, host=None, '\tssl: %r', database, user, host, port, socket, charset, local_infile, ssl) conv = { - FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or '0000-00-00 00:00:00'), - FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or '0000-00-00 00:00:00'), - FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or '00:00:00'), - FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or '0000-00-00'), + FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), + FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), + FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), + FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), } conn = connection.connect(database=db, user=user, password=password, From 6dc8fff09513a36e6d893974f5764959e339362b Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 26 Aug 2016 14:04:30 -0700 Subject: [PATCH 025/824] Update readme to change the install instructions. --- README.md | 39 +++++---------------------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index d7de272d6..04e315396 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,6 @@ A command line client for MySQL that can do auto-completion and syntax highlight HomePage: [http://mycli.net](http://mycli.net) -Debian Packages via [PackageCloud.io](https://packagecloud.io/amjith/mycli). - ![Completion](screenshots/tables.png) ![CompletionGif](screenshots/main.gif) @@ -32,7 +30,11 @@ or $ brew update && brew install mycli # Only on OS X ``` -Check the [detailed install instructions](#detailed-install-instructions) for debian packages or getting started with pip. +or + +``` +$ sudo apt-get install mycli # Only on debian or ubuntu +``` ### Usage @@ -114,37 +116,6 @@ Twitter: [@amjithr](http://twitter.com/amjithr) ## Detailed Install Instructions: -### Debian/Ubuntu Package: - -The debian package for `mycli` is hosted on [packagecloud.io](https://packagecloud.io/amjith/mycli). - -Add the gpg key for packagecloud for package verification. - -``` -curl https://packagecloud.io/gpg.key | apt-key add - -``` - -Install a package called apt-transport-https to make it possible for apt to fetch packages over https. - -``` -apt-get install -y apt-transport-https -``` - -Add the mycli package repo to the apt source. - -``` -echo "deb https://packagecloud.io/amjith/mycli/ubuntu/ trusty main" | sudo tee -a /etc/apt/sources.list -``` - -Update the apt sources and install mycli. - -``` -$ sudo apt-get update -$ sudo apt-get install mycli -``` - -Now `mycli` can be upgraded easily by using ``sudo apt-get upgrade mycli``. - ### RHEL, Centos, Fedora: I haven't built an RPM package for mycli yet. So please use `pip` to install `mycli`. You can install pip on your system using: From 895e109ca169e240e0f26e087b93c3b47df2232a Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 17 Sep 2016 10:10:26 +0200 Subject: [PATCH 026/824] Support python-sqlparse 0.2 --- mycli/packages/completion_engine.py | 9 +++++---- setup.py | 2 +- tests/test_sqlexecute.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 91a22c90a..774bb1ef1 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -2,6 +2,7 @@ import sys import sqlparse from sqlparse.sql import Comparison, Identifier, Where +from sqlparse.compat import text_type from .parseutils import last_word, extract_tables, find_prev_keyword from .special import parse_special_command @@ -56,7 +57,7 @@ def suggest_type(full_text, text_before_cursor): stmt_start, stmt_end = 0, 0 for statement in parsed: - stmt_len = len(statement.to_unicode()) + stmt_len = len(text_type(statement)) stmt_start, stmt_end = stmt_end, stmt_end + stmt_len if stmt_end >= current_pos: @@ -79,7 +80,7 @@ def suggest_type(full_text, text_before_cursor): if tok1 and tok1.value == '\\': return suggest_special(text_before_cursor) - last_token = statement and statement.token_prev(len(statement.tokens)) or '' + last_token = statement and statement.token_prev(len(statement.tokens))[1] or '' return suggest_based_on_last_token(last_token, text_before_cursor, full_text, identifier) @@ -157,7 +158,7 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier # Check for a subquery expression (cases 3 & 4) where = p.tokens[-1] - prev_tok = where.token_prev(len(where.tokens) - 1) + idx, prev_tok = where.token_prev(len(where.tokens) - 1) if isinstance(prev_tok, Comparison): # e.g. "SELECT foo FROM bar WHERE foo = ANY(" @@ -170,7 +171,7 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier return column_suggestions # Get the token before the parens - prev_tok = p.token_prev(len(p.tokens) - 1) + idx, prev_tok = p.token_prev(len(p.tokens) - 1) if prev_tok and prev_tok.value and prev_tok.value.lower() == 'using': # tbl1 INNER JOIN tbl2 USING (col1, col2) tables = extract_tables(full_text) diff --git a/setup.py b/setup.py index f21bdb4c9..31f2fde4c 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse == 0.1.19', + 'sqlparse >= 0.2.0', 'configobj >= 5.0.6', ] diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index 69b9a6d26..d9ed62532 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -90,10 +90,10 @@ def test_invalid_column_name(executor): @dbtest def test_unicode_support_in_output(executor): run(executor, "create table unicodechars(t text)") - run(executor, "insert into unicodechars (t) values ('é')") + run(executor, u"insert into unicodechars (t) values ('é')") # See issue #24, this raises an exception without proper handling - assert u'é' in run(executor, "select * from unicodechars", join=True) + assert u'é' in run(executor, u"select * from unicodechars", join=True) @dbtest def test_expanded_output(executor): @@ -247,7 +247,7 @@ def test_cd_command_current_dir(executor): @dbtest def test_unicode_support(executor): - assert u'日本語' in run(executor, "SELECT '日本語' AS japanese;", join=True) + assert u'日本語' in run(executor, u"SELECT '日本語' AS japanese;", join=True) @dbtest def test_favorite_query_multiline_statement(executor): From 9b30aba3318014456eab2400109d41b7ac348ff5 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 17 Sep 2016 21:21:24 -0700 Subject: [PATCH 027/824] Add an try/except for AS keyword crash. --- mycli/packages/completion_engine.py | 44 ++++++++++++++++------------- tests/test_completion_engine.py | 8 ++++++ 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 774bb1ef1..31ef8744c 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -28,26 +28,32 @@ def suggest_type(full_text, text_before_cursor): identifier = None - # If we've partially typed a word then word_before_cursor won't be an empty - # string. In that case we want to remove the partially typed string before - # sending it to the sqlparser. Otherwise the last token will always be the - # partially typed string which renders the smart completion useless because - # it will always return the list of keywords as completion. - if word_before_cursor: - if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': - parsed = sqlparse.parse(text_before_cursor) + # This is a temporary hack; the exception handling + # here should be removed once sqlparse has been fixed + try: + # If we've partially typed a word then word_before_cursor won't be an empty + # string. In that case we want to remove the partially typed string before + # sending it to the sqlparser. Otherwise the last token will always be the + # partially typed string which renders the smart completion useless because + # it will always return the list of keywords as completion. + if word_before_cursor: + if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': + parsed = sqlparse.parse(text_before_cursor) + else: + parsed = sqlparse.parse( + text_before_cursor[:-len(word_before_cursor)]) + + # word_before_cursor may include a schema qualification, like + # "schema_name.partial_name" or "schema_name.", so parse it + # separately + p = sqlparse.parse(word_before_cursor)[0] + + if p.tokens and isinstance(p.tokens[0], Identifier): + identifier = p.tokens[0] else: - parsed = sqlparse.parse( - text_before_cursor[:-len(word_before_cursor)]) - - # word_before_cursor may include a schema qualification, like - # "schema_name.partial_name" or "schema_name.", so parse it - # separately - p = sqlparse.parse(word_before_cursor)[0] - if p.tokens and isinstance(p.tokens[0], Identifier): - identifier = p.tokens[0] - else: - parsed = sqlparse.parse(text_before_cursor) + parsed = sqlparse.parse(text_before_cursor) + except (TypeError, AttributeError): + return [] if len(parsed) > 1: # Multiple statements being edited -- isolate the current one by diff --git a/tests/test_completion_engine.py b/tests/test_completion_engine.py index 294fdbd56..9b8771083 100644 --- a/tests/test_completion_engine.py +++ b/tests/test_completion_engine.py @@ -442,3 +442,11 @@ def test_cross_join(): {'type': 'table', 'schema': []}, {'type': 'view', 'schema': []}, {'type': 'schema'}]) + +@pytest.mark.parametrize('expression', [ + 'SELECT 1 AS ', + 'SELECT 1 FROM tabl AS ', +]) +def test_after_as(expression): + suggestions = suggest_type(expression, expression) + assert set(suggestions) == set() From d1e8c849d374a130b86296716375dcb69afc757b Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 17 Sep 2016 21:21:49 -0700 Subject: [PATCH 028/824] Pin the version of sqlparse. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 31f2fde4c..82ff1dd6c 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse >= 0.2.0', + 'sqlparse == 0.2.0', 'configobj >= 5.0.6', ] From 6836753244a9bc34af4cad3e3da222fe454c6e0f Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 18 Sep 2016 07:35:17 -0700 Subject: [PATCH 029/824] Make the dependency of sqlparse slightly more liberal. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 82ff1dd6c..3d80c96ca 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse == 0.2.0', + 'sqlparse>=0.2.0,<0.2.2', 'configobj >= 5.0.6', ] From 9c3ce514e5305251e049b93a4ce8ae93629ad54e Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 18 Sep 2016 22:29:38 -0700 Subject: [PATCH 030/824] Remove duplicate listing of DISTINCT keyword. --- mycli/sqlcompleter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index e92332738..c12ada771 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -23,7 +23,7 @@ class SQLCompleter(Completer): 'CASE', 'CHAR', 'CHECK', 'COLUMN', 'COMMENT', 'COMMIT', 'CHANGE MASTER TO', 'CREATE', 'CURRENT', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', 'DELETE FROM', 'DELIMITER', 'DESC', - 'DESCRIBE', 'DISTINCT', 'DROP', 'ELSE', 'END', 'ESCAPE', 'EXISTS', + 'DESCRIBE', 'DROP', 'ELSE', 'END', 'ESCAPE', 'EXISTS', 'FILE', 'FLOAT', 'FOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', 'INCREMENT', 'INDEX', 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'LEFT', From 220617bff1da5103e0f9e15528aabc3787848e56 Mon Sep 17 00:00:00 2001 From: Adam Chainz Date: Fri, 14 Oct 2016 11:50:07 +0100 Subject: [PATCH 031/824] Release as a universal wheel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By releasing as a [Python wheel](http://pythonwheels.com/) as well as a source distribution, you can speed up end user’s installs. After merging this command, to release you just need to run `python setup.py clean sdist bdist_wheel upload`. --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 From cf2b606856fd5aefe5d1607e2733187802ca8ae1 Mon Sep 17 00:00:00 2001 From: Joseph Caillet Date: Thu, 20 Oct 2016 10:41:50 +0200 Subject: [PATCH 032/824] Update README.md Documented workaround for the issues described here : https://github.com/dbcli/mycli/issues/281 --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 04e315396..016c5396f 100644 --- a/README.md +++ b/README.md @@ -153,3 +153,22 @@ Thanks to [PyMysql](http://www.pymysql.org/) for a pure python adapter to MySQL Tests have been run on OS X and Linux. THIS HAS NOT BEEN TESTED IN WINDOWS, but the libraries used in this app are Windows compatible. This means it should work without any modifications. If you're unable to run it on Windows, please file a bug. I will try my best to fix it. + +### Use with pager (mysql workaround) +As described [here](https://github.com/dbcli/mycli/issues/281), " we only read the [client] section of my.cnf not the [mysql] section". + +So, if you want to use a pager, your .my.cnf file should looks like this: + +``` +[mysql] +pager = mypager +[client] +pager = mypager +``` + +instead of just this : + +``` +[mysql] +pager = mypager +``` From df411698623bd0936cccb0db034313235f5cdfa7 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 21 Oct 2016 20:51:21 -0700 Subject: [PATCH 033/824] Update changelog for release 1.8.1. --- AUTHORS | 1 + changelog.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/AUTHORS b/AUTHORS index 85a53e67b..316fa8979 100644 --- a/AUTHORS +++ b/AUTHORS @@ -15,6 +15,7 @@ Contributors: * Shoma Suzuki * Daniel West * Scrappy Soft + * Dick Marinus * Daniel Black * Jonathan Bruno * Casper Langemeijer diff --git a/changelog.md b/changelog.md index 91cf208d7..2bd345e87 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,18 @@ +1.8.1: +====== + +Bug Fixes: +---------- +* Remove duplicate listing of DISTINCT keyword. (Thanks: [Amjith Ramanujam]). +* Add an try/except for AS keyword crash. (Thanks: [Amjith Ramanujam]). +* Support python-sqlparse 0.2. (Thanks: [Dick Marinus]). +* Fallback to the raw object for invalid time values. (Thanks: [Amjith Ramanujam]). +* Reset the show items when completion is refreshed. (Thanks: [Amjith Ramanujam]). + +Internal Changes: +----------------- +* Make the dependency of sqlparse slightly more liberal. (Thanks: [Amjith Ramanujam]). + 1.8.0: ====== @@ -337,3 +352,4 @@ Bug Fixes: [Jonathan Slenders]: https://github.com/jonathanslenders [Casper Langemeijer]: https://github.com/langemeijer [Scrappy Soft]: https://github.com/scrappysoft +[Dick Marinus]: https://github.com/meeuw From eec63f0ab3f216f10dbdc1c7137e2e929702297d Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 24 Oct 2016 06:45:22 -0700 Subject: [PATCH 034/824] Releasing version 1.8.1 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index b28097579..e8b6b090b 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.8.0' +__version__ = '1.8.1' From 95de1846ddccaa98ce5887566073f09330decb27 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 24 Oct 2016 06:53:00 -0700 Subject: [PATCH 035/824] Update the release script. --- release.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/release.py b/release.py index 7ced9bd9f..622056274 100644 --- a/release.py +++ b/release.py @@ -70,6 +70,12 @@ def register_with_pypi(): def create_source_tarball(): run_step('python', 'setup.py', 'sdist') +def create_python_wheel(): + run_step('python', 'setup.py', 'sdist', 'bdist_wheel') + +def upload_source_tarball(): + run_step('python', 'setup.py', 'sdist', 'upload') + def push_to_github(): run_step('git', 'push', 'origin', 'master') @@ -121,5 +127,7 @@ def checklist(questions): create_git_tag('v%s' % ver) register_with_pypi() create_source_tarball() + create_python_wheel() push_to_github() push_tags_to_github() + upload_source_tarball() From b975116b8fa099c68065d89e3d60c16bb8ce5437 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 29 Oct 2016 18:23:48 +0100 Subject: [PATCH 036/824] Add config option for auto-vertical-output in myclirc --- mycli/myclirc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mycli/myclirc b/mycli/myclirc index 3a6911c72..66e1ee16f 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -65,6 +65,9 @@ less_chatty = False # Use alias from --login-path instead of host name in prompt login_path_as_host = False +# Enable auto-verical-output +auto_vertical_output = False + # Custom colors for the completion menu, toolbar, etc. [colors] # Completion menus. From db5412f2d9c0570676722aaa3849f2ffda5f746e Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 29 Oct 2016 18:24:32 +0100 Subject: [PATCH 037/824] Update main.py to read the auto-vertical-output config --- mycli/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 82d817f66..8ebdc09c8 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -94,7 +94,6 @@ def __init__(self, sqlexecute=None, prompt=None, self.logfile = logfile self.defaults_suffix = defaults_suffix self.login_path = login_path - self.auto_vertical_output = auto_vertical_output # self.cnf_files is a class variable that stores the list of mysql # config files to read in at launch. @@ -119,6 +118,10 @@ def __init__(self, sqlexecute=None, prompt=None, self.destructive_warning = c_dest_warning if warn is None else warn self.login_path_as_host = c['main'].as_bool('login_path_as_host') + # read from cli argument or user config file + self.auto_vertical_output = auto_vertical_output or \ + c['main'].as_bool('auto_vertical_output') + # Write user config if system config wasn't the last config loaded. if c.filename not in self.system_config_files: write_default_config(self.default_config_file, self.user_config_file) From 6c6dfdd4eeeb7192abb607b85963612f92178c87 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 01:12:39 +0100 Subject: [PATCH 038/824] Add CSV support for batch output --- mycli/main.py | 53 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 82d817f66..418668ccd 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -5,6 +5,7 @@ import os import os.path import sys +import csv import traceback import logging import threading @@ -13,6 +14,12 @@ from random import choice from io import open +# support StringIO for Python 2 and 3 +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + import click import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction @@ -744,6 +751,8 @@ def run_query(self, query, table_format=None): help='Automatically switch to vertical output mode if the result is wider than the terminal width.') @click.option('-t', '--table', is_flag=True, help='Display batch output in table format.') +@click.option('--csv', is_flag=True, + help='Display batch output in CSV format.') @click.option('--warn/--no-warn', default=None, help='Warn before running a destructive query.') @click.option('--local-infile', type=bool, @@ -756,8 +765,8 @@ def run_query(self, query, table_format=None): def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, login_path, auto_vertical_output, local_infile, ssl_ca, ssl_capath, - ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, warn, - execute): + ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, csv, + warn, execute): if version: print('Version:', __version__) @@ -818,7 +827,21 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - mycli.run_query(stdin_text, table_format=table) + results = mycli.sqlexecute.run(stdin_text) + table_format = None + + if csv: + table_format = 'csv' + new_line = False + elif table: + new_line = True + + for result in results: + title, cur, headers, status = result + output = format_output(title, cur, headers, status, table_format) + for line in output: + click.echo(line, nl=new_line) + except Exception as e: click.secho(str(e), err=True, fg='red') exit(1) @@ -832,7 +855,24 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max headers = [utf8tounicode(x) for x in headers] if expanded: output.append(expanded_table(cur, headers)) - elif table_format is not None: + if table_format is None: + output.append('\t'.join(headers)) + for row in cur: + output.append('\t'.join([str(r) for r in row])) + elif table_format == 'csv': + content = StringIO() + writer = csv.writer(content) + writer.writerow(headers) + output.append(content.getvalue()) + content.truncate(0) + for row in cur: + row = ['null' if val is None else val.encode('utf-8') for val in row] + writer.writerow(row) + output.append(content.getvalue()) + content.truncate(0) + output.append('\n') + content.close() + else: rows = list(cur) tabulated, frows = tabulate(rows, headers, tablefmt=table_format, missingval='') @@ -842,12 +882,9 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max output.append(expanded_table(rows, headers)) else: output.append(tabulated) - else: - output.append('\t'.join(headers)) - for row in cur: - output.append('\t'.join([str(r) for r in row])) if status: # Only print the status if it's not None. output.append(status) + return output def content_exceeds_width(row, width): From ae9d3d894046083942a002152341a20675adeb76 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 02:44:54 +0000 Subject: [PATCH 039/824] Fix code for batch output --- mycli/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 418668ccd..7b582b374 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -829,16 +829,17 @@ def cli(database, user, host, port, socket, password, dbname, try: results = mycli.sqlexecute.run(stdin_text) table_format = None + new_line = True if csv: table_format = 'csv' new_line = False elif table: - new_line = True + table_format = mycli.table_format for result in results: title, cur, headers, status = result - output = format_output(title, cur, headers, status, table_format) + output = format_output(title, cur, headers, None, table_format) for line in output: click.echo(line, nl=new_line) @@ -855,7 +856,7 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max headers = [utf8tounicode(x) for x in headers] if expanded: output.append(expanded_table(cur, headers)) - if table_format is None: + elif table_format is None: output.append('\t'.join(headers)) for row in cur: output.append('\t'.join([str(r) for r in row])) From 44cc4ac55db0fed8b06bbc0c092e476eb99d00e6 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 02:45:10 +0000 Subject: [PATCH 040/824] Add test for CSV batch output --- tests/test_main.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index f174a45e1..4da21dc98 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -88,6 +88,21 @@ def test_batch_mode_table(executor): assert result.exit_code == 0 assert expected in result.output +@dbtest +def test_batch_mode_csv(executor): + run(executor, '''create table test(a text, b text)''') + run(executor, '''insert into test (a, b) values('abc', 'def'), ('ghi', 'jkl')''') + + sql = 'select * from test;' + + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['--csv'], input=sql) + + expected = 'a,b\nabc,def\nghi,jkl\n\n' + + assert result.exit_code == 0 + assert expected in result.output + def test_query_starts_with(executor): query = 'USE test;' assert query_starts_with(query, ('use', )) is True From b8bf879de64f51f68dc2d1f52de3759d7c46aa5a Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 03:19:02 +0000 Subject: [PATCH 041/824] Fix tests for Python 3.5.2 --- mycli/main.py | 3 +-- tests/test_main.py | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 7b582b374..f45facc20 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -867,11 +867,10 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max output.append(content.getvalue()) content.truncate(0) for row in cur: - row = ['null' if val is None else val.encode('utf-8') for val in row] + row = ['null' if val is None else str(val) for val in row] writer.writerow(row) output.append(content.getvalue()) content.truncate(0) - output.append('\n') content.close() else: rows = list(cur) diff --git a/tests/test_main.py b/tests/test_main.py index 4da21dc98..22deed83b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -98,10 +98,11 @@ def test_batch_mode_csv(executor): runner = CliRunner() result = runner.invoke(cli, args=CLI_ARGS + ['--csv'], input=sql) - expected = 'a,b\nabc,def\nghi,jkl\n\n' + expected = 'a,b\nabc,def\nghi,jkl\n' + result_output = result.output.replace('\x00', '') # python 3 assert result.exit_code == 0 - assert expected in result.output + assert expected in result_output def test_query_starts_with(executor): query = 'USE test;' From 4af283f0a0fba083aeed4313c0fb245a11a393d6 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 03:37:40 +0000 Subject: [PATCH 042/824] Update auto-vertical-output config description in myclirc --- mycli/myclirc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/myclirc b/mycli/myclirc index 66e1ee16f..baf505106 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -65,7 +65,8 @@ less_chatty = False # Use alias from --login-path instead of host name in prompt login_path_as_host = False -# Enable auto-verical-output +# Cause result sets to be displayed vertically if they are too wide for the current window, +# and using normal tabular format otherwise. (This applies to statements terminated by ; or \G.) auto_vertical_output = False # Custom colors for the completion menu, toolbar, etc. From a17da7e44a941190d77851fa29cf2afb3414ca0b Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 04:01:45 +0000 Subject: [PATCH 043/824] Add prompt support for \D, \p and \_ options --- mycli/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 82d817f66..b99325bd3 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -6,6 +6,7 @@ import os.path import sys import traceback +import socket import logging import threading from time import time @@ -693,6 +694,9 @@ def get_prompt(self, string): string = string.replace('\\d', sqlexecute.dbname or '(none)') string = string.replace('\\t', sqlexecute.server_type()[0] or 'mycli') string = string.replace('\\n', "\n") + string = string.replace('\\D', datetime.now().strftime('%a %b %d %H:%M:%S %Y')) + string = string.replace('\\p', socket.gethostbyname(socket.gethostname())) + string = string.replace('\\_', ' ') return string def run_query(self, query, table_format=None): From d4cb1b3746e22b4c7f1b5b42de3935ccbf400243 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 12:38:08 +0000 Subject: [PATCH 044/824] Fix "\p" prompt representation --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index b99325bd3..0a9a5ad0f 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -695,7 +695,7 @@ def get_prompt(self, string): string = string.replace('\\t', sqlexecute.server_type()[0] or 'mycli') string = string.replace('\\n', "\n") string = string.replace('\\D', datetime.now().strftime('%a %b %d %H:%M:%S %Y')) - string = string.replace('\\p', socket.gethostbyname(socket.gethostname())) + string = string.replace('\\p', str(sqlexecute.port)) string = string.replace('\\_', ' ') return string From 24b9e35d5eb669b851b62c5f6b30b826b4c4653e Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 13:22:01 +0000 Subject: [PATCH 045/824] Update import StringIO to cStringIO for performance reasons --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index f45facc20..80cc8c5b3 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -16,7 +16,7 @@ # support StringIO for Python 2 and 3 try: - from StringIO import StringIO + from cStringIO import StringIO except ImportError: from io import StringIO From 365a94819c25ecad1a27a8bf56560f5e98897388 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 13:23:02 +0000 Subject: [PATCH 046/824] Remove unecessary `truncate()` calls when writing CSV --- mycli/main.py | 7 +++---- tests/test_main.py | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 80cc8c5b3..a6fdb1319 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -864,13 +864,12 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max content = StringIO() writer = csv.writer(content) writer.writerow(headers) - output.append(content.getvalue()) - content.truncate(0) + for row in cur: row = ['null' if val is None else str(val) for val in row] writer.writerow(row) - output.append(content.getvalue()) - content.truncate(0) + + output.append(content.getvalue()) content.close() else: rows = list(cur) diff --git a/tests/test_main.py b/tests/test_main.py index 22deed83b..c36e980ab 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -99,10 +99,9 @@ def test_batch_mode_csv(executor): result = runner.invoke(cli, args=CLI_ARGS + ['--csv'], input=sql) expected = 'a,b\nabc,def\nghi,jkl\n' - result_output = result.output.replace('\x00', '') # python 3 assert result.exit_code == 0 - assert expected in result_output + assert expected in result.output def test_query_starts_with(executor): query = 'USE test;' From 10bd6e1e1303c731ec75ba88662389b328822db4 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 19:05:28 +0000 Subject: [PATCH 047/824] Add new param "new_line" to MyCli.run_query() method --- mycli/main.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index a6fdb1319..561a003c7 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -702,15 +702,14 @@ def get_prompt(self, string): string = string.replace('\\n', "\n") return string - def run_query(self, query, table_format=None): + def run_query(self, query, table_format=None, new_line=True): """Runs query""" results = self.sqlexecute.run(query) for result in results: title, cur, headers, status = result - table_format = self.table_format if table_format else None output = format_output(title, cur, headers, None, table_format) for line in output: - click.echo(line) + click.echo(line, nl=new_line) @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') @@ -827,7 +826,6 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - results = mycli.sqlexecute.run(stdin_text) table_format = None new_line = True @@ -837,12 +835,8 @@ def cli(database, user, host, port, socket, password, dbname, elif table: table_format = mycli.table_format - for result in results: - title, cur, headers, status = result - output = format_output(title, cur, headers, None, table_format) - for line in output: - click.echo(line, nl=new_line) - + mycli.run_query(stdin_text, table_format=table_format, new_line=new_line) + exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') exit(1) From 124ca067713a4896a398e25e2367989fbc192cc7 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 19:26:12 +0000 Subject: [PATCH 048/824] Add "tsv" as the default table_format --- mycli/main.py | 6 ++---- tests/test_main.py | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 561a003c7..1cd00a94b 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -848,12 +848,10 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max output.append(title) if cur: headers = [utf8tounicode(x) for x in headers] + table_format = 'tsv' if table_format is None else table_format + if expanded: output.append(expanded_table(cur, headers)) - elif table_format is None: - output.append('\t'.join(headers)) - for row in cur: - output.append('\t'.join([str(r) for r in row])) elif table_format == 'csv': content = StringIO() writer = csv.writer(content) diff --git a/tests/test_main.py b/tests/test_main.py index c36e980ab..c7f939696 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -30,7 +30,8 @@ def test_format_output_auto_expand(): def test_format_output_no_table(): results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], 'test status', None) - expected = ['Title', 'head1\thead2', 'abc\tdef', 'test status'] + + expected = ['Title', u'head1 \thead2\nabc \tdef', 'test status'] assert results == expected @dbtest @@ -65,7 +66,7 @@ def test_batch_mode(executor): result = runner.invoke(cli, args=CLI_ARGS, input=sql) assert result.exit_code == 0 - assert 'count(*)\n3\na\nabc' in result.output + assert ' count(*)\n 3\na\nabc\n' in result.output @dbtest def test_batch_mode_table(executor): From 71f503afb699b2a8365f9e8fddcdddb1960a1e79 Mon Sep 17 00:00:00 2001 From: Darik Gamble Date: Fri, 4 Nov 2016 19:54:34 -0400 Subject: [PATCH 049/824] Bump sqlparse version --- mycli/packages/parseutils.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/packages/parseutils.py b/mycli/packages/parseutils.py index 3cbf4a05d..7f848ad85 100644 --- a/mycli/packages/parseutils.py +++ b/mycli/packages/parseutils.py @@ -64,7 +64,7 @@ def last_word(text, include='alphanum_underscore'): # This code is borrowed from sqlparse example script. # def is_subselect(parsed): - if not parsed.is_group(): + if not parsed.is_group: return False for item in parsed.tokens: if item.ttype is DML and item.value.upper() in ('SELECT', 'INSERT', diff --git a/setup.py b/setup.py index 3d80c96ca..c3693572e 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse>=0.2.0,<0.2.2', + 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.6', ] From 54a8206c5187347a6a6f6ae4db994e0d7864cbcc Mon Sep 17 00:00:00 2001 From: Darik Gamble Date: Sat, 5 Nov 2016 08:06:15 -0400 Subject: [PATCH 050/824] Dangling `as` doesn't get grouped into identifiers in this version of sqlparse, so we need to handle them explicitly --- mycli/packages/completion_engine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 31ef8744c..6e2165dca 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -197,6 +197,9 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier return [{'type': 'column', 'tables': extract_tables(full_text)}] elif token_v in ('set', 'by', 'distinct'): return [{'type': 'column', 'tables': extract_tables(full_text)}] + elif token_v == 'as': + # Don't suggest anything for an alias + return [] elif token_v in ('show'): return [{'type': 'show'}] elif token_v in ('to',): From dbe05e840de5e2627a6007e7736f8b831d7dc131 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 5 Nov 2016 14:21:56 +0000 Subject: [PATCH 051/824] Update --execute arg code and add tests --- mycli/main.py | 7 ++++++- tests/test_main.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 1cd00a94b..65c1b1ff9 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -805,7 +805,12 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - mycli.run_query(execute, table_format=table) + table_format = None + if table: + table_format = mycli.table_format + elif csv: + table_format = 'csv' + mycli.run_query(execute, table_format=table_format) exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') diff --git a/tests/test_main.py b/tests/test_main.py index c7f939696..bed88945e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -51,6 +51,38 @@ def test_execute_arg(executor): assert result.exit_code == 0 assert 'abc' in result.output + expected = 'a\nabc\n' + + assert result.output == expected + + +@dbtest +def test_execute_arg_with_table(executor): + run(executor, 'create table test (a text)') + run(executor, 'insert into test values("abc")') + + sql = 'select * from test;' + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['-e', sql] + ['--table']) + expected = '+-----+\n| a |\n|-----|\n| abc |\n+-----+\n' + + assert result.exit_code == 0 + assert result.output == expected + + +@dbtest +def test_execute_arg_with_csv(executor): + run(executor, 'create table test (a text)') + run(executor, 'insert into test values("abc")') + + sql = 'select * from test;' + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['-e', sql] + ['--csv']) + expected = 'a\nabc\n\n' + + assert result.exit_code == 0 + assert result.output == expected + @dbtest def test_batch_mode(executor): From 00fa9b2c9e1ad0b1c04e8ec53dad9a58c0cb0072 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 5 Nov 2016 16:11:28 +0000 Subject: [PATCH 052/824] Fix tests for --execute command --- tests/test_main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index bed88945e..974eca74f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -53,7 +53,7 @@ def test_execute_arg(executor): expected = 'a\nabc\n' - assert result.output == expected + assert expected in result.output @dbtest @@ -67,7 +67,7 @@ def test_execute_arg_with_table(executor): expected = '+-----+\n| a |\n|-----|\n| abc |\n+-----+\n' assert result.exit_code == 0 - assert result.output == expected + assert expected in result.output @dbtest @@ -81,7 +81,7 @@ def test_execute_arg_with_csv(executor): expected = 'a\nabc\n\n' assert result.exit_code == 0 - assert result.output == expected + assert expected in result.output @dbtest From 4efa6e7072035901b5422276f046e1fe4beea5c0 Mon Sep 17 00:00:00 2001 From: cxbig Date: Fri, 11 Nov 2016 10:03:28 +0100 Subject: [PATCH 053/824] Added 'REGEXP' keyword in completer list --- mycli/sqlcompleter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index c12ada771..e79806771 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -30,7 +30,7 @@ class SQLCompleter(Completer): 'LEVEL', 'LIKE', 'LIMIT', 'LOCK', 'LOGS', 'LONG', 'MASTER', 'MODE', 'MODIFY', 'NOT', 'NULL', 'NUMBER', 'OFFSET', 'ON', 'OPTION', 'OR', 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', 'PORT', 'PRIMARY', - 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'RENAME', 'REPAIR', 'RESET', + 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'REGEXP', 'RENAME', 'REPAIR', 'RESET', 'REVOKE', 'RIGHT', 'ROLLBACK','ROW', 'ROWS', 'SELECT', 'SESSION', 'SET', 'SHARE', 'SHOW', 'SLAVE', 'SMALLINT', 'START', 'STOP', 'TABLE', 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UNION', 'UNIQUE', 'UPDATE', From 05a13b7bb327c379d1c43d932ac6ed67426ba255 Mon Sep 17 00:00:00 2001 From: Jialong Liu Date: Tue, 20 Dec 2016 00:04:45 +0800 Subject: [PATCH 054/824] Fixes #202. --- mycli/main.py | 273 +++++++++++++++++++++++++------------------------- 1 file changed, 137 insertions(+), 136 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 5691ac01b..f585c84b5 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -465,6 +465,142 @@ def get_continuation_tokens(cli, width): continuation_prompt = self.get_prompt(self.prompt_continuation_format) return [(Token.Continuation, ' ' * (width - len(continuation_prompt)) + continuation_prompt)] + def one_iteration(document=None): + if document is None: + document = self.cli.run(reset_current_buffer=True) + + special.set_expanded_output(False) + + # The reason we check here instead of inside the sqlexecute is + # because we want to raise the Exit exception which will be + # caught by the try/except block that wraps the + # sqlexecute.run() statement. + if quit_command(document.text): + raise EOFError + + try: + document = self.handle_editor_command(self.cli, document) + except RuntimeError as e: + logger.error("sql: %r, error: %r", document.text, e) + logger.error("traceback: %r", traceback.format_exc()) + self.output(str(e), err=True, fg='red') + return + + if self.destructive_warning: + destroy = confirm_destructive_query(document.text) + if destroy is None: + pass # Query was not destructive. Nothing to do here. + elif destroy is True: + self.output('Your call!') + else: + self.output('Wise choice!') + return + + # Keep track of whether or not the query is mutating. In case + # of a multi-statement query, the overall query is considered + # mutating if any one of the component statements is mutating + mutating = False + + try: + logger.debug('sql: %r', document.text) + + if self.logfile: + self.logfile.write('\n# %s\n' % datetime.now()) + self.logfile.write(document.text) + self.logfile.write('\n') + + successful = False + start = time() + res = sqlexecute.run(document.text) + successful = True + output = [] + total = 0 + for title, cur, headers, status in res: + logger.debug("headers: %r", headers) + logger.debug("rows: %r", cur) + logger.debug("status: %r", status) + threshold = 1000 + if (is_select(status) and + cur and cur.rowcount > threshold): + self.output('The result set has more than %s rows.' + % threshold, fg='red') + if not click.confirm('Do you want to continue?'): + self.output("Aborted!", err=True, fg='red') + break + + if self.auto_vertical_output: + max_width = self.cli.output.get_size().columns + else: + max_width = None + + formatted = format_output(title, cur, headers, + status, self.table_format, + special.is_expanded_output(), max_width) + + output.extend(formatted) + end = time() + total += end - start + mutating = mutating or is_mutating(status) + except UnicodeDecodeError as e: + import pymysql + if pymysql.VERSION < (0, 6, 7): + message = ('You are running an older version of pymysql.\n' + 'Please upgrade to 0.6.7 or above to view binary data.\n' + 'Try \'pip install -U pymysql\'.') + self.output(message) + else: + raise e + except KeyboardInterrupt: + # Restart connection to the database + sqlexecute.connect() + logger.debug("cancelled query, sql: %r", document.text) + self.output("cancelled query", err=True, fg='red') + except NotImplementedError: + self.output('Not Yet Implemented.', fg="yellow") + except OperationalError as e: + logger.debug("Exception: %r", e) + if (e.args[0] in (2003, 2006, 2013)): + logger.debug('Attempting to reconnect.') + self.output('Reconnecting...', fg='yellow') + try: + sqlexecute.connect() + logger.debug('Reconnected successfully.') + one_iteration(document) + return # OK to just return, cuz the recursion call runs to the end. + except OperationalError as e: + logger.debug('Reconnect failed. e: %r', e) + self.output(str(e), err=True, fg='red') + return # If reconnection failed, don't proceed further. + else: + logger.error("sql: %r, error: %r", document.text, e) + logger.error("traceback: %r", traceback.format_exc()) + self.output(str(e), err=True, fg='red') + except Exception as e: + logger.error("sql: %r, error: %r", document.text, e) + logger.error("traceback: %r", traceback.format_exc()) + self.output(str(e), err=True, fg='red') + else: + try: + if special.is_pager_enabled(): + self.output_via_pager('\n'.join(output)) + else: + self.output('\n'.join(output)) + except KeyboardInterrupt: + pass + if special.is_timing_enabled(): + self.output('Time: %0.03fs' % total) + + # Refresh the table names and column names if necessary. + if need_completion_refresh(document.text): + self.refresh_completions( + reset=need_completion_reset(document.text)) + finally: + if self.logfile is False: + self.output("Warning: This query was not logged.", err=True, fg='red') + query = Query(document.text, successful, mutating) + self.query_history.append(query) + + get_toolbar_tokens = create_toolbar_tokens_func(self.completion_refresher.is_refreshing) layout = create_prompt_layout(lexer=MyCliLexer, @@ -500,142 +636,7 @@ def get_continuation_tokens(cli, width): try: while True: - document = self.cli.run(reset_current_buffer=True) - - special.set_expanded_output(False) - - # The reason we check here instead of inside the sqlexecute is - # because we want to raise the Exit exception which will be - # caught by the try/except block that wraps the - # sqlexecute.run() statement. - if quit_command(document.text): - raise EOFError - - try: - document = self.handle_editor_command(self.cli, document) - except RuntimeError as e: - logger.error("sql: %r, error: %r", document.text, e) - logger.error("traceback: %r", traceback.format_exc()) - self.output(str(e), err=True, fg='red') - continue - if self.destructive_warning: - destroy = confirm_destructive_query(document.text) - if destroy is None: - pass # Query was not destructive. Nothing to do here. - elif destroy is True: - self.output('Your call!') - else: - self.output('Wise choice!') - continue - - # Keep track of whether or not the query is mutating. In case - # of a multi-statement query, the overall query is considered - # mutating if any one of the component statements is mutating - mutating = False - - try: - logger.debug('sql: %r', document.text) - - if self.logfile: - self.logfile.write('\n# %s\n' % datetime.now()) - self.logfile.write(document.text) - self.logfile.write('\n') - - successful = False - start = time() - res = sqlexecute.run(document.text) - successful = True - output = [] - total = 0 - for title, cur, headers, status in res: - logger.debug("headers: %r", headers) - logger.debug("rows: %r", cur) - logger.debug("status: %r", status) - threshold = 1000 - if (is_select(status) and - cur and cur.rowcount > threshold): - self.output('The result set has more than %s rows.' - % threshold, fg='red') - if not click.confirm('Do you want to continue?'): - self.output("Aborted!", err=True, fg='red') - break - - if self.auto_vertical_output: - max_width = self.cli.output.get_size().columns - else: - max_width = None - - formatted = format_output(title, cur, headers, - status, self.table_format, - special.is_expanded_output(), max_width) - - output.extend(formatted) - end = time() - total += end - start - mutating = mutating or is_mutating(status) - except UnicodeDecodeError as e: - import pymysql - if pymysql.VERSION < (0, 6, 7): - message = ('You are running an older version of pymysql.\n' - 'Please upgrade to 0.6.7 or above to view binary data.\n' - 'Try \'pip install -U pymysql\'.') - self.output(message) - else: - raise e - except KeyboardInterrupt: - # Restart connection to the database - sqlexecute.connect() - logger.debug("cancelled query, sql: %r", document.text) - self.output("cancelled query", err=True, fg='red') - except NotImplementedError: - self.output('Not Yet Implemented.', fg="yellow") - except OperationalError as e: - logger.debug("Exception: %r", e) - reconnect = True - if (e.args[0] in (2003, 2006, 2013)): - reconnect = click.prompt('Connection reset. Reconnect (Y/n)', - show_default=False, type=bool, default=True) - if reconnect: - logger.debug('Attempting to reconnect.') - try: - sqlexecute.connect() - logger.debug('Reconnected successfully.') - self.output('Reconnected!\nTry the command again.', fg='green') - except OperationalError as e: - logger.debug('Reconnect failed. e: %r', e) - self.output(str(e), err=True, fg='red') - continue # If reconnection failed, don't proceed further. - else: # If user chooses not to reconnect, don't proceed further. - continue - else: - logger.error("sql: %r, error: %r", document.text, e) - logger.error("traceback: %r", traceback.format_exc()) - self.output(str(e), err=True, fg='red') - except Exception as e: - logger.error("sql: %r, error: %r", document.text, e) - logger.error("traceback: %r", traceback.format_exc()) - self.output(str(e), err=True, fg='red') - else: - try: - if special.is_pager_enabled(): - self.output_via_pager('\n'.join(output)) - else: - self.output('\n'.join(output)) - except KeyboardInterrupt: - pass - if special.is_timing_enabled(): - self.output('Time: %0.03fs' % total) - - # Refresh the table names and column names if necessary. - if need_completion_refresh(document.text): - self.refresh_completions( - reset=need_completion_reset(document.text)) - finally: - if self.logfile is False: - self.output("Warning: This query was not logged.", err=True, fg='red') - query = Query(document.text, successful, mutating) - self.query_history.append(query) - + one_iteration() except EOFError: if not self.less_chatty: self.output('Goodbye!') From fa39a021b83628cae5330bcae98e74eb52ecbc5b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 24 Jan 2017 07:30:53 -0600 Subject: [PATCH 055/824] Honor smart_completion setting. --- mycli/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index f585c84b5..e0560e85e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -155,8 +155,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.query_history = [] # Initialize completer. - smart_completion = c['main'].as_bool('smart_completion') - self.completer = SQLCompleter(smart_completion) + self.smart_completion = c['main'].as_bool('smart_completion') + self.completer = SQLCompleter(self.smart_completion) self._completer_lock = threading.Lock() # Register custom special commands. @@ -443,7 +443,8 @@ def run_cli(self): logger = self.logger self.configure_pager() - self.refresh_completions() + if self.smart_completion: + self.refresh_completions() project_root = os.path.dirname(PACKAGE_ROOT) author_file = os.path.join(project_root, 'AUTHORS') From 1e71e5b97cb2d0313b21fccdbd5b291bd63afd9b Mon Sep 17 00:00:00 2001 From: chainkite Date: Sun, 29 Jan 2017 20:49:20 +0800 Subject: [PATCH 056/824] Fix #333 ctrl-c issue --- mycli/main.py | 9 ++++++++- mycli/sqlexecute.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index e0560e85e..b100d9c82 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -552,9 +552,16 @@ def one_iteration(document=None): else: raise e except KeyboardInterrupt: + # get last connection id + connection_id_to_kill = sqlexecute.connection_id + logger.debug("connection id to kill: %r", connection_id_to_kill) # Restart connection to the database sqlexecute.connect() - logger.debug("cancelled query, sql: %r", document.text) + for title, cur, headers, status in sqlexecute.run('kill %s' % connection_id_to_kill): + status_str = str(status).lower() + if status_str.find('ok') > -1: + logger.debug("cancelled query, connection id: %r, sql: %r", + connection_id_to_kill, document.text) self.output("cancelled query", err=True, fg='red') except NotImplementedError: self.output('Not Yet Implemented.', fg="yellow") diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index d44a11fec..40274827e 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -41,6 +41,7 @@ def __init__(self, database, user, password, host, port, socket, charset, self.local_infile = local_infile self.ssl = ssl self._server_type = None + self.connection_id = None self.connect() def connect(self, database=None, user=None, password=None, host=None, @@ -90,6 +91,8 @@ def connect(self, database=None, user=None, password=None, host=None, self.socket = socket self.charset = charset self.ssl = ssl + # retrieve connection id + self.reset_connection_id() def run(self, statement): """Execute the sql in the database and return the results. The results @@ -221,3 +224,16 @@ def server_type(self): self._server_type = (product_type, version) return self._server_type + + def get_connection_id(self): + if not self.connection_id: + self.reset_connection_id() + return self.connection_id + + def reset_connection_id(self): + # Remember current connection id + _logger.debug('Get current connection id') + res = self.run('select connection_id()') + for title, cur, headers, status in res: + self.connection_id = list(cur)[0][0] + _logger.debug('Current connection id: %s', self.connection_id) From 32fbfb88ac7a8a861962d816f47b57811230c784 Mon Sep 17 00:00:00 2001 From: chainkite Date: Thu, 2 Feb 2017 21:39:48 +0800 Subject: [PATCH 057/824] Fix #333 use fetchone() when get connection id --- mycli/sqlexecute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 40274827e..b28008e82 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -235,5 +235,5 @@ def reset_connection_id(self): _logger.debug('Get current connection id') res = self.run('select connection_id()') for title, cur, headers, status in res: - self.connection_id = list(cur)[0][0] + self.connection_id = cur.fetchone()[0] _logger.debug('Current connection id: %s', self.connection_id) From 9044017530a7c3edc2f2d91988bc9bfe5062b938 Mon Sep 17 00:00:00 2001 From: chainkite Date: Mon, 6 Feb 2017 14:25:24 +0800 Subject: [PATCH 058/824] catch exception when kill query --- mycli/main.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index b100d9c82..8bb7721a6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -557,12 +557,15 @@ def one_iteration(document=None): logger.debug("connection id to kill: %r", connection_id_to_kill) # Restart connection to the database sqlexecute.connect() - for title, cur, headers, status in sqlexecute.run('kill %s' % connection_id_to_kill): - status_str = str(status).lower() - if status_str.find('ok') > -1: - logger.debug("cancelled query, connection id: %r, sql: %r", - connection_id_to_kill, document.text) - self.output("cancelled query", err=True, fg='red') + try: + for title, cur, headers, status in sqlexecute.run('kill %s' % connection_id_to_kill): + status_str = str(status).lower() + if status_str.find('ok') > -1: + logger.debug("cancelled query, connection id: %r, sql: %r", + connection_id_to_kill, document.text) + self.output("cancelled query", err=True, fg='red') + except Exception as e: + self.output("cancel query with error: %r" % e, err=True, fg='red') except NotImplementedError: self.output('Not Yet Implemented.', fg="yellow") except OperationalError as e: From 07e800b030c4a5b6935dea9e44941d673c522192 Mon Sep 17 00:00:00 2001 From: chainkite Date: Thu, 9 Feb 2017 15:05:13 +0800 Subject: [PATCH 059/824] revise error message --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 8bb7721a6..b7ac62f5e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -565,7 +565,7 @@ def one_iteration(document=None): connection_id_to_kill, document.text) self.output("cancelled query", err=True, fg='red') except Exception as e: - self.output("cancel query with error: %r" % e, err=True, fg='red') + self.output('Encountered error while cancelling query: %s' % str(e), err=True, fg='red') except NotImplementedError: self.output('Not Yet Implemented.', fg="yellow") except OperationalError as e: From feaf3295c0cc395da56d4436ab35457e853d7d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Tue, 14 Feb 2017 19:43:09 +0100 Subject: [PATCH 060/824] Change link to pymysql to a working one --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 016c5396f..a15cbc033 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ of this app. [Click](http://click.pocoo.org/3/) is used for command line option parsing and printing error messages. -Thanks to [PyMysql](http://www.pymysql.org/) for a pure python adapter to MySQL database. +Thanks to [PyMysql](https://github.com/PyMySQL/PyMySQL) for a pure python adapter to MySQL database. [Tabulate](https://pypi.python.org/pypi/tabulate) library is used for pretty printing the output of tables. From 99cc5cf2c5b7a560c4f9ee459e3286ca7b554f96 Mon Sep 17 00:00:00 2001 From: John Sterling Date: Tue, 14 Feb 2017 17:18:28 -0500 Subject: [PATCH 061/824] 316: users needs to be cleared prior to refresh to prevent repeatedly appending the same users every time a refreh is triggered --- mycli/sqlcompleter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index e79806771..da7f34f87 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -192,6 +192,7 @@ def set_dbname(self, dbname): def reset_completions(self): self.databases = [] + self.users = [] self.show_items = [] self.dbname = '' self.dbmetadata = {'tables': {}, 'views': {}, 'functions': {}} From a2e63ddd6e0eb143380781dbaaf40f412441d0fb Mon Sep 17 00:00:00 2001 From: Gilbert Consellado Date: Thu, 16 Feb 2017 16:37:57 +0800 Subject: [PATCH 062/824] Add some keywords on the keywords list --- mycli/sqlcompleter.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index da7f34f87..c31f3ed02 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -19,21 +19,21 @@ class SQLCompleter(Completer): keywords = ['ACCESS', 'ADD', 'ALL', 'ALTER TABLE', 'AND', 'ANY', 'AS', - 'ASC', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', - 'CASE', 'CHAR', 'CHECK', 'COLUMN', 'COMMENT', 'COMMIT', - 'CHANGE MASTER TO', 'CREATE', 'CURRENT', 'DATABASE', 'DATE', + 'ASC', 'AUTO_INCREMENT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', + 'CASE', 'CHAR', 'CHECK', 'COLUMN', 'COMMENT', 'COMMIT', 'CONSTRAINT', + 'CHANGE MASTER TO', 'CHARACTER SET', 'COLLATE', 'CREATE', 'CURRENT', 'CURRENT_TIMESTAMP', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', 'DELETE FROM', 'DELIMITER', 'DESC', - 'DESCRIBE', 'DROP', 'ELSE', 'END', 'ESCAPE', 'EXISTS', - 'FILE', 'FLOAT', 'FOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', + 'DESCRIBE', 'DROP', 'ELSE', 'END', 'ENGINE', 'ESCAPE', 'EXISTS', + 'FILE', 'FLOAT', 'FOR', 'FOREIGN', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', 'INCREMENT', 'INDEX', - 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'LEFT', + 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'KEY', 'LEFT', 'LEVEL', 'LIKE', 'LIMIT', 'LOCK', 'LOGS', 'LONG', 'MASTER', 'MODE', 'MODIFY', 'NOT', 'NULL', 'NUMBER', 'OFFSET', 'ON', 'OPTION', 'OR', 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', 'PORT', 'PRIMARY', - 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'REGEXP', 'RENAME', 'REPAIR', 'RESET', - 'REVOKE', 'RIGHT', 'ROLLBACK','ROW', 'ROWS', 'SELECT', 'SESSION', 'SET', + 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'REFERENCES', 'REGEXP', 'RENAME', 'REPAIR', 'RESET', + 'REVOKE', 'RIGHT', 'ROLLBACK','ROW', 'ROWS', 'ROW_FORMAT', 'SELECT', 'SESSION', 'SET', 'SHARE', 'SHOW', 'SLAVE', 'SMALLINT', 'START', 'STOP', 'TABLE', 'THEN', - 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UNION', 'UNIQUE', 'UPDATE', + 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UNION', 'UNIQUE', 'UNSIGNED', 'UPDATE', 'USE', 'USER', 'USING', 'VALUES', 'VARCHAR', 'VIEW', 'WHEN', 'WHERE', 'WITH'] From e943050fc78ce2f5ec73ac4e8505271c934a43f5 Mon Sep 17 00:00:00 2001 From: Gilbert Consellado Date: Thu, 16 Feb 2017 19:46:21 +0800 Subject: [PATCH 063/824] Change the "FOREIGN" keyword to "FOREIGN KEY" --- mycli/sqlcompleter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index c31f3ed02..f5fc3c903 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -24,7 +24,7 @@ class SQLCompleter(Completer): 'CHANGE MASTER TO', 'CHARACTER SET', 'COLLATE', 'CREATE', 'CURRENT', 'CURRENT_TIMESTAMP', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', 'DELETE FROM', 'DELIMITER', 'DESC', 'DESCRIBE', 'DROP', 'ELSE', 'END', 'ENGINE', 'ESCAPE', 'EXISTS', - 'FILE', 'FLOAT', 'FOR', 'FOREIGN', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', + 'FILE', 'FLOAT', 'FOR', 'FOREIGN KEY', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', 'INCREMENT', 'INDEX', 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'KEY', 'LEFT', 'LEVEL', 'LIKE', 'LIMIT', 'LOCK', 'LOGS', 'LONG', 'MASTER', 'MODE', From aeac636cddb8ace3054901e2f55f9025c5b80c2b Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 18 Feb 2017 09:24:37 -0800 Subject: [PATCH 064/824] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a15cbc033..6b682a2c4 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ If you already know how to install python packages, then you can install it via You might need sudo on linux. ``` -$ pip install mycli +$ pip install -U mycli ``` or From 247b24a7c53c9d8312227b5bef5d28610afd0058 Mon Sep 17 00:00:00 2001 From: 0x4ec7 <0x4ec7@gmail.com> Date: Tue, 21 Feb 2017 19:05:51 +0800 Subject: [PATCH 065/824] Fix: Crash if statement starts with # --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index b7ac62f5e..a91eba5bb 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -954,7 +954,7 @@ def query_starts_with(query, prefixes): """Check if the query starts with any item from *prefixes*.""" prefixes = [prefix.lower() for prefix in prefixes] formatted_sql = sqlparse.format(query.lower(), strip_comments=True) - return formatted_sql.split()[0] in prefixes + return bool(formatted_sql) and formatted_sql.split()[0] in prefixes def queries_start_with(queries, prefixes): """Check if any queries start with any item from *prefixes*.""" From 75166ae1bfcc2c58b0da36b29ffbfe91fdb7d7b1 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 21 Feb 2017 20:06:17 +0100 Subject: [PATCH 066/824] FIX: disable align (space padding) for table format tsv #348 --- mycli/packages/tabulate.py | 46 +++++++++++++++++++++----------------- tests/test_main.py | 4 ++-- tests/test_tabulate.py | 8 +++++++ 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index f0874d525..978f89439 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -87,7 +87,7 @@ def _is_file(f): TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", "linebetweenrows", "linebelow", "headerrow", "datarow", - "padding", "with_header_hide"]) + "padding", "with_header_hide", "with_align"]) def _pipe_segment_with_colons(align, colwidth): @@ -162,13 +162,13 @@ def escape_char(c): headerrow=DataRow("", " ", ""), datarow=DataRow("", " ", ""), padding=0, - with_header_hide=["lineabove", "linebelow"]), + with_header_hide=["lineabove", "linebelow"], with_align=True), "plain": TableFormat(lineabove=None, linebelowheader=None, linebetweenrows=None, linebelow=None, headerrow=DataRow("", " ", ""), datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None), + padding=0, with_header_hide=None, with_align=True), "grid": TableFormat(lineabove=Line("+", "-", "+", "+"), linebelowheader=Line("+", "=", "+", "+"), @@ -176,7 +176,7 @@ def escape_char(c): linebelow=Line("+", "-", "+", "+"), headerrow=DataRow("|", "|", "|"), datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=True), "fancy_grid": TableFormat(lineabove=Line("╒", "═", "╤", "╕"), linebelowheader=Line("╞", "═", "╪", "╡"), @@ -184,7 +184,7 @@ def escape_char(c): linebelow=Line("╘", "═", "╧", "╛"), headerrow=DataRow("│", "│", "│"), datarow=DataRow("│", "│", "│"), - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=True), "pipe": TableFormat(lineabove=_pipe_line_with_colons, linebelowheader=_pipe_line_with_colons, @@ -193,7 +193,7 @@ def escape_char(c): headerrow=DataRow("|", "|", "|"), datarow=DataRow("|", "|", "|"), padding=1, - with_header_hide=["lineabove"]), + with_header_hide=["lineabove"], with_align=True), "orgtbl": TableFormat(lineabove=None, linebelowheader=Line("|", "-", "+", "|"), @@ -201,7 +201,7 @@ def escape_char(c): linebelow=None, headerrow=DataRow("|", "|", "|"), datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=True), "psql": TableFormat(lineabove=Line("+", "-", "+", "+"), linebelowheader=Line("|", "-", "+", "|"), @@ -209,7 +209,7 @@ def escape_char(c): linebelow=Line("+", "-", "+", "+"), headerrow=DataRow("|", "|", "|"), datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=True), "rst": TableFormat(lineabove=Line("", "=", " ", ""), linebelowheader=Line("", "=", " ", ""), @@ -217,7 +217,7 @@ def escape_char(c): linebelow=Line("", "=", " ", ""), headerrow=DataRow("", " ", ""), datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None), + padding=0, with_header_hide=None, with_align=True), "mediawiki": TableFormat(lineabove=Line("{| class=\"wikitable\" style=\"text-align: left;\"", "", "", "\n|+ \n|-"), @@ -226,7 +226,7 @@ def escape_char(c): linebelow=Line("|}", "", "", ""), headerrow=partial(_mediawiki_row_with_attrs, "!"), datarow=partial(_mediawiki_row_with_attrs, "|"), - padding=0, with_header_hide=None), + padding=0, with_header_hide=None, with_align=True), "html": TableFormat(lineabove=Line("", "", "", ""), linebelowheader=None, @@ -234,7 +234,7 @@ def escape_char(c): linebelow=Line("
", "", "", ""), headerrow=partial(_html_row_with_attrs, "th"), datarow=partial(_html_row_with_attrs, "td"), - padding=0, with_header_hide=None), + padding=0, with_header_hide=None, with_align=False), "latex": TableFormat(lineabove=_latex_line_begin_tabular, linebelowheader=Line("\\hline", "", "", ""), @@ -242,7 +242,7 @@ def escape_char(c): linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), headerrow=_latex_row, datarow=_latex_row, - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=False), "latex_booktabs": TableFormat(lineabove=partial(_latex_line_begin_tabular, booktabs=True), linebelowheader=Line("\\midrule", "", "", ""), @@ -250,13 +250,13 @@ def escape_char(c): linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", ""), headerrow=_latex_row, datarow=_latex_row, - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=False), "tsv": TableFormat(lineabove=None, linebelowheader=None, linebetweenrows=None, linebelow=None, headerrow=DataRow("", "\t", ""), datarow=DataRow("", "\t", ""), - padding=0, with_header_hide=None)} + padding=0, with_header_hide=None, with_align=False)} tabulate_formats = list(sorted(_table_formats.keys())) @@ -904,8 +904,14 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", else: width_fn = wcswidth - # align columns - aligns = [numalign if ct in [int,float] else stralign for ct in coltypes] + if not isinstance(tablefmt, TableFormat): + tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) + + if tablefmt.with_align: + # align columns + aligns = [numalign if ct in [int,float] else stralign for ct in coltypes] + else: + aligns = [False for ct in coltypes] minwidths = [width_fn(h) + MIN_PADDING for h in headers] if headers else [0]*len(cols) cols = [_align_column(c, a, minw, has_invisible) for c, a, minw in zip(cols, aligns, minwidths)] @@ -913,7 +919,10 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", if headers: # align headers and add headers t_cols = cols or [['']] * len(headers) - t_aligns = aligns or [stralign] * len(headers) + if tablefmt.with_align: + t_aligns = aligns or [stralign] * len(headers) + else: + t_aligns = [False for ct in coltypes] minwidths = [max(minw, width_fn(c[0])) for minw, c in zip(minwidths, t_cols)] headers = [_align_header(h, a, minw) for h, a, minw in zip(headers, t_aligns, minwidths)] @@ -922,9 +931,6 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", minwidths = [width_fn(c[0]) for c in cols] rows = list(zip(*cols)) - if not isinstance(tablefmt, TableFormat): - tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) - return _format_table(tablefmt, headers, rows, minwidths, aligns), rows diff --git a/tests/test_main.py b/tests/test_main.py index 974eca74f..49e8be3ff 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -31,7 +31,7 @@ def test_format_output_no_table(): results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], 'test status', None) - expected = ['Title', u'head1 \thead2\nabc \tdef', 'test status'] + expected = ['Title', u'head1\thead2\nabc\tdef', 'test status'] assert results == expected @dbtest @@ -98,7 +98,7 @@ def test_batch_mode(executor): result = runner.invoke(cli, args=CLI_ARGS, input=sql) assert result.exit_code == 0 - assert ' count(*)\n 3\na\nabc\n' in result.output + assert 'count(*)\n3\na\nabc\n' in result.output @dbtest def test_batch_mode_table(executor): diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py index 351261971..e0ddc407d 100644 --- a/tests/test_tabulate.py +++ b/tests/test_tabulate.py @@ -12,3 +12,11 @@ def test_dont_strip_leading_whitespace(): |---------| | abc | +---------+ ''').strip() +def test_dont_add_whitespace(): + data = [[3, 4]] + headers = ['1', '2'] + tbl, _ = tabulate(data, headers, tablefmt='tsv') + assert tbl == dedent(''' + 1\t2 + 3\t4 + ''').strip() From c86568ef3b6931e0cb5df186be000da23acbde06 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sun, 26 Feb 2017 17:02:22 +0100 Subject: [PATCH 067/824] fix doctest --- conftest.py | 13 ++++++- mycli/packages/tabulate.py | 78 +++++++++++++++++++------------------- pytest.ini | 2 + tests/pytest.ini | 2 - 4 files changed, 53 insertions(+), 42 deletions(-) create mode 100644 pytest.ini delete mode 100644 tests/pytest.ini diff --git a/conftest.py b/conftest.py index cee5a17e4..5f72d7107 100644 --- a/conftest.py +++ b/conftest.py @@ -1,2 +1,11 @@ -# https://pytest.org/latest/example/pythoncollection.html -collect_ignore = ["setup.py"] +import sys +collect_ignore = [ + "setup.py", + "mycli/magic.py", + "mycli/packages/parseutils.py", +] +if sys.version_info[0] > 2: + collect_ignore.extend([ + "mycli/packages/counter.py", + "mycli/packages/ordereddict.py", + ]) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index 978f89439..fa9718267 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -270,14 +270,15 @@ def simple_separated_format(separator): """Construct a simple TableFormat with columns separated by a separator. >>> tsv = simple_separated_format("\\t") ; \ - tabulate([["foo", 1], ["spam", 23]], tablefmt=tsv) == 'foo \\t 1\\nspam\\t23' - True - + print(tabulate([["foo", 1], ["spam", 23]], tablefmt=tsv)[0].replace('\\t', r'\\t')) + foo\\t1 + spam\\t23 """ return TableFormat(None, None, None, None, headerrow=DataRow('', separator, ''), datarow=DataRow('', separator, ''), - padding=0, with_header_hide=None) + padding=0, with_header_hide=None, + with_align=False) def _isconvertible(conv, string): @@ -498,16 +499,17 @@ def _column_type(strings, has_invisible=True): def _format(val, valtype, floatfmt, missingval=""): - """Format a value accoding to its type. + u"""Format a value accoding to its type. Unicode is supported: >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', '\u0446\u0438\u0444\u0440\u0430'] ; \ tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] ; \ - good_result = '\\u0431\\u0443\\u043a\\u0432\\u0430 \\u0446\\u0438\\u0444\\u0440\\u0430\\n------- -------\\n\\u0430\\u0437 2\\n\\u0431\\u0443\\u043a\\u0438 4' ; \ - tabulate(tbl, headers=hrow) == good_result - True - + print(tabulate(tbl, headers=hrow)[0]) + буква цифра + ------- ------- + аз 2 + буки 4 """ if val is None: return missingval @@ -656,7 +658,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", missingval=""): """Format a fixed width table for pretty printing. - >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])) + >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])[0]) --- --------- 1 2.34 -56 8.999 @@ -686,7 +688,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", with the plain-text format of R and Pandas' dataframes. >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], - ... headers="firstrow")) + ... headers="firstrow")[0]) sex age ----- ----- ----- Alice F 24 @@ -714,7 +716,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", >>> print(tabulate([["spam", 1, None], ... ["eggs", 42, 3.14], - ... ["other", None, 2.7]], missingval="?")) + ... ["other", None, 2.7]], missingval="?")[0]) ----- -- ---- spam 1 ? eggs 42 3.14 @@ -730,25 +732,25 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", it separates columns with a double space: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "plain")) + ... ["strings", "numbers"], "plain")[0]) strings numbers spam 41.9999 eggs 451 - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="plain")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="plain")[0]) spam 41.9999 eggs 451 "simple" format is like Pandoc simple_tables: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "simple")) + ... ["strings", "numbers"], "simple")[0]) strings numbers --------- --------- spam 41.9999 eggs 451 - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="simple")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="simple")[0]) ---- -------- spam 41.9999 eggs 451 @@ -758,7 +760,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", Pandoc grid_tables: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "grid")) + ... ["strings", "numbers"], "grid")[0]) +-----------+-----------+ | strings | numbers | +===========+===========+ @@ -767,7 +769,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", | eggs | 451 | +-----------+-----------+ - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="grid")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="grid")[0]) +------+----------+ | spam | 41.9999 | +------+----------+ @@ -777,7 +779,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", "fancy_grid" draws a grid using box-drawing characters: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "fancy_grid")) + ... ["strings", "numbers"], "fancy_grid")[0]) ╒═══════════╤═══════════╕ │ strings │ numbers │ ╞═══════════╪═══════════╡ @@ -790,13 +792,13 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", pipe_tables: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "pipe")) + ... ["strings", "numbers"], "pipe")[0]) | strings | numbers | |:----------|----------:| | spam | 41.9999 | | eggs | 451 | - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="pipe")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="pipe")[0]) |:-----|---------:| | spam | 41.9999 | | eggs | 451 | @@ -807,14 +809,14 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", intersections: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "orgtbl")) + ... ["strings", "numbers"], "orgtbl")[0]) | strings | numbers | |-----------+-----------| | spam | 41.9999 | | eggs | 451 | - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="orgtbl")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="orgtbl")[0]) | spam | 41.9999 | | eggs | 451 | @@ -822,7 +824,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", note that reStructuredText accepts also "grid" tables: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "rst")) + ... ["strings", "numbers"], "rst")[0]) ========= ========= strings numbers ========= ========= @@ -830,7 +832,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", eggs 451 ========= ========= - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")[0]) ==== ======== spam 41.9999 eggs 451 @@ -840,7 +842,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", MediaWiki-based sites: >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="mediawiki")) + ... headers="firstrow", tablefmt="mediawiki")[0]) {| class="wikitable" style="text-align: left;" |+ |- @@ -854,31 +856,31 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", "html" produces HTML markup: >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="html")) + ... headers="firstrow", tablefmt="html")[0]) - - - + + +
strings numbers
spam 41.9999
eggs 451
stringsnumbers
spam41.9999
eggs451
"latex" produces a tabular environment of LaTeX document markup: - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex")) - \\begin{tabular}{lr} + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex")[0]) + \\begin{tabular}{ll} \\hline - spam & 41.9999 \\\\ - eggs & 451 \\\\ + spam & 41.9999 \\\\ + eggs & 451 \\\\ \\hline \\end{tabular} "latex_booktabs" produces a tabular environment of LaTeX document markup using the booktabs.sty package: - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex_booktabs")) - \\begin{tabular}{lr} + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex_booktabs")[0]) + \\begin{tabular}{ll} \\toprule - spam & 41.9999 \\\\ - eggs & 451 \\\\ + spam & 41.9999 \\\\ + eggs & 451 \\\\ \\bottomrule \end{tabular} diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..7c5b52b77 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts=--capture=sys --showlocals --doctest-modules diff --git a/tests/pytest.ini b/tests/pytest.ini deleted file mode 100644 index f78774051..000000000 --- a/tests/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts=--capture=sys --showlocals \ No newline at end of file From c3ed9a67aeb1a30ddf0ed67203def3aa4c9753ab Mon Sep 17 00:00:00 2001 From: John Sterling Date: Wed, 1 Mar 2017 21:15:18 -0500 Subject: [PATCH 068/824] #288: only use the default less flags if they are not already set --- mycli/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index a91eba5bb..6bb91b755 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -665,8 +665,9 @@ def output_via_pager(self, text): click.echo_via_pager(text) def configure_pager(self): - # Provide sane defaults for less. - os.environ['LESS'] = '-RXF' + # Provide sane defaults for less if they are empty. + if not os.environ.get('LESS'): + os.environ['LESS'] = '-RXF' cnf = self.read_my_cnf_files(self.cnf_files, ['pager', 'skip-pager']) if cnf['pager']: From 221b50ce19dc1a28ac79cfc39a54f2e9e18b2878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Thu, 2 Mar 2017 18:46:47 +0100 Subject: [PATCH 069/824] Replace obsolete pycrypto with pycryptodome fork The pycrypto library is abandoned since 2014 with a lot of bugs open. It has been forked into pycryptodome which is mostly API-compatible with pycrypto. FWICS, the bits used by mycli work fine with the new library. Therefore, switch to using pycryptodome instead of obsolete pycrypto. I've also enabled it for Windows since the latter library officially supports this system and provides binary wheels for it, so the dependency should no longer be a problem. Since the actual Python code isn't changed, mycli can still formally work with pycrypto or without the library at all. However, pip will now install the new library. --- mycli/config.py | 4 ++-- mycli/main.py | 2 +- setup.py | 7 +------ tests/test_config.py | 12 ++++++------ 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 284a8b400..97ead1fcd 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -19,7 +19,7 @@ class CryptoError(Exception): """ - Exception to signal about pycrypto not available. + Exception to signal about pycrypto(dome) not available. """ pass @@ -126,7 +126,7 @@ def read_and_decrypt_mylogin_cnf(f): :rtype: io.BytesIO or None """ if AES is None: - raise CryptoError('pycrypto is not available.') + raise CryptoError('pycrypto(dome) is not available.') # Number of bytes used to store the length of ciphertext. MAX_CIPHER_STORE_LEN = 4 diff --git a/mycli/main.py b/mycli/main.py index 6bb91b755..f0e175fd2 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -174,7 +174,7 @@ def __init__(self, sqlexecute=None, prompt=None, # There was an error reading the login path file. print('Error: Unable to read login path file.') except CryptoError: - click.secho('Warning: .mylogin.cnf was not read: pycrypto ' + click.secho('Warning: .mylogin.cnf was not read: pycrypto(dome) ' 'module is not available.') self.cli = None diff --git a/setup.py b/setup.py index c3693572e..caa140647 100644 --- a/setup.py +++ b/setup.py @@ -18,14 +18,9 @@ 'PyMySQL >= 0.6.2', 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.6', + 'pycryptodome', ] -# pycrypto is a hard package to install on Windows, so we make it an optional -# dependency. When it's installed, we can read mylogin.cnf, when it is not -# available, we skip reading mylogin.cnf and print a warning message. -if platform.system() != 'Windows': - install_requirements.append('pycrypto >= 2.6.1') - setup( name='mycli', author='Amjith Ramanujam', diff --git a/tests/test_config.py b/tests/test_config.py index efd66d017..7879dd109 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,7 +11,7 @@ open_mylogin_cnf, read_and_decrypt_mylogin_cnf, str_to_bool) -with_pycrypto = ['pycrypto' in set([package.project_name for package in +with_pycryptodome = ['pycryptodome' in set([package.project_name for package in pip.get_installed_distributions()])] LOGIN_PATH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -26,13 +26,13 @@ def open_bmylogin_cnf(name): return buf -@pytest.mark.skipif(with_pycrypto, reason='requires pycrypto missing') +@pytest.mark.skipif(with_pycryptodome, reason='requires pycryptodome missing') def test_read_mylogin_cnf_without_crypto(): with pytest.raises(CryptoError): mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) -@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_read_mylogin_cnf(): """Tests that a login path file can be read and decrypted.""" mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) @@ -44,14 +44,14 @@ def test_read_mylogin_cnf(): assert word in contents -@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_decrypt_blank_mylogin_cnf(): """Test that a blank login path file is handled correctly.""" mylogin_cnf = read_and_decrypt_mylogin_cnf(BytesIO()) assert mylogin_cnf is None -@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_corrupted_login_key(): """Test that a corrupted login path key is handled correctly.""" buf = open_bmylogin_cnf(LOGIN_PATH_FILE) @@ -68,7 +68,7 @@ def test_corrupted_login_key(): assert mylogin_cnf is None -@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_corrupted_pad(): """Tests that a login path file with a corrupted pad is partially read.""" buf = open_bmylogin_cnf(LOGIN_PATH_FILE) From 557be1dacd703a171d9ac3f161b65fc1076664c5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 2 Mar 2017 22:57:53 -0600 Subject: [PATCH 070/824] Add Python 3.6 to tests and setup file. --- .travis.yml | 1 + setup.py | 1 + tox.ini | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5450b982b..826c01d53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" env: - PYMYSQL_VERSION=0.6.7 diff --git a/setup.py b/setup.py index caa140647..54a2404d3 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Programming Language :: SQL', 'Topic :: Database', 'Topic :: Database :: Front-Ends', diff --git a/tox.ini b/tox.ini index a2d769118..d3f9aef14 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34, py35 +envlist = py26, py27, py33, py34, py35, py36 [testenv] deps = pytest mock From 12c94227ee0dc920d18806b60de11d44b2a2376d Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 25 Feb 2017 21:00:44 +0100 Subject: [PATCH 071/824] tests for mycli.packages.special.iocommands --- tests/test_special_iocommands.py | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/test_special_iocommands.py diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py new file mode 100644 index 000000000..5dbbc1602 --- /dev/null +++ b/tests/test_special_iocommands.py @@ -0,0 +1,37 @@ +import mycli.packages.special +import os +def test_set_get_pager(): + mycli.packages.special.set_pager_enabled(True) + assert mycli.packages.special.is_pager_enabled() + mycli.packages.special.set_pager_enabled(False) + assert not mycli.packages.special.is_pager_enabled() + mycli.packages.special.set_pager('less') + assert os.environ['PAGER'] == "less" + mycli.packages.special.set_pager(False) + assert os.environ['PAGER'] == "less" + del os.environ['PAGER'] + mycli.packages.special.set_pager(False) + mycli.packages.special.disable_pager() + assert not mycli.packages.special.is_pager_enabled() + +def test_set_get_timing(): + mycli.packages.special.set_timing_enabled(True) + assert mycli.packages.special.is_timing_enabled() + mycli.packages.special.set_timing_enabled(False) + assert not mycli.packages.special.is_timing_enabled() + +def test_set_get_expanded_output(): + mycli.packages.special.set_expanded_output(True) + assert mycli.packages.special.is_expanded_output() + mycli.packages.special.set_expanded_output(False) + assert not mycli.packages.special.is_expanded_output() + +def test_editor_command(): + assert mycli.packages.special.editor_command(r'hello\e') + assert mycli.packages.special.editor_command(r'\ehello') + assert not mycli.packages.special.editor_command(r'hello') + + assert mycli.packages.special.get_filename(r'\e filename') == "filename" + + os.environ['EDITOR'] = 'true' + mycli.packages.special.open_external_editor(r'select 1') == "select 1" From 97d31af5f37e19e8a60ccdf24731913203e52315 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 5 Mar 2017 12:49:38 -0600 Subject: [PATCH 072/824] Drop Python 2.6 support. --- .travis.yml | 1 - conftest.py | 5 - debian/control | 2 +- debian/mycli.triggers | 1 - mycli/clistyle.py | 2 +- mycli/completion_refresher.py | 5 +- mycli/main.py | 10 +- mycli/packages/counter.py | 190 ---------------------------------- mycli/packages/ordereddict.py | 127 ----------------------- mycli/sqlcompleter.py | 7 +- setup.py | 1 - tox.ini | 2 +- 12 files changed, 8 insertions(+), 345 deletions(-) delete mode 100644 mycli/packages/counter.py delete mode 100644 mycli/packages/ordereddict.py diff --git a/.travis.yml b/.travis.yml index 826c01d53..516df2604 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.6" - "2.7" - "3.3" - "3.4" diff --git a/conftest.py b/conftest.py index 5f72d7107..d2cd1336c 100644 --- a/conftest.py +++ b/conftest.py @@ -4,8 +4,3 @@ "mycli/magic.py", "mycli/packages/parseutils.py", ] -if sys.version_info[0] > 2: - collect_ignore.extend([ - "mycli/packages/counter.py", - "mycli/packages/ordereddict.py", - ]) diff --git a/debian/control b/debian/control index f1602021a..418383263 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Standards-Version: 3.9.5 Package: mycli Architecture: any -Pre-Depends: dpkg (>= 1.16.1), python2.7-minimal | python2.6-minimal, ${misc:Pre-Depends} +Pre-Depends: dpkg (>= 1.16.1), python2.7-minimal, ${misc:Pre-Depends} Depends: ${python:Depends}, ${misc:Depends} Description: CLI for MySQL Database. With auto-completion and syntax highlighting. CLI for MySQL Database. With auto-completion and syntax highlighting. diff --git a/debian/mycli.triggers b/debian/mycli.triggers index 084bfd36d..b0b1d2184 100644 --- a/debian/mycli.triggers +++ b/debian/mycli.triggers @@ -1,7 +1,6 @@ # Register interest in Python interpreter changes (Python 2 for now); and # don't make the Python package dependent on the virtualenv package # processing (noawait) -interest-noawait /usr/bin/python2.6 interest-noawait /usr/bin/python2.7 # Also provide a symbolic trigger for all dh-virtualenv packages diff --git a/mycli/clistyle.py b/mycli/clistyle.py index a2a7b2a70..ef7c1c9d8 100644 --- a/mycli/clistyle.py +++ b/mycli/clistyle.py @@ -13,7 +13,7 @@ def style_factory(name, cli_style): styles = {} styles.update(style.styles) styles.update(default_style_extensions) - custom_styles = dict([(string_to_tokentype(x), y) for x, y in cli_style.items()]) + custom_styles = {string_to_tokentype(x): y for x, y in cli_style.items()} styles.update(custom_styles) return style_from_dict(styles) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index b5b4142ed..33afa009c 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -1,9 +1,6 @@ import threading from .packages.special.main import COMMANDS -try: - from collections import OrderedDict -except ImportError: - from .packages.ordereddict import OrderedDict +from collections import OrderedDict from .sqlcompleter import SQLCompleter from .sqlexecute import SQLExecute diff --git a/mycli/main.py b/mycli/main.py index f0e175fd2..e852fddc4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -271,11 +271,7 @@ def initialize_logging(self): root_logger.addHandler(handler) root_logger.setLevel(level_map[log_level.upper()]) - # Only capture warnings on Python 2.7 and later. - try: - logging.captureWarnings(True) - except AttributeError: - pass + logging.captureWarnings(True) root_logger.debug('Initializing mycli logging.') root_logger.debug('Log file %r.', log_file) @@ -309,7 +305,7 @@ def get(key): result = cnf[sect][key] return result - return dict([(x, get(x)) for x in keys]) + return {x: get(x) for x in keys} def merge_ssl_with_cnf(self, ssl, cnf): """Merge SSL configuration dict with cnf dict""" @@ -809,7 +805,7 @@ def cli(database, user, host, port, socket, password, dbname, } # remove empty ssl options - ssl = dict((k, v) for (k, v) in ssl.items() if v is not None) + ssl = {k: v for k, v in ssl.items() if v is not None} if database and '://' in database: mycli.connect_uri(database, local_infile, ssl) else: diff --git a/mycli/packages/counter.py b/mycli/packages/counter.py deleted file mode 100644 index e74db3116..000000000 --- a/mycli/packages/counter.py +++ /dev/null @@ -1,190 +0,0 @@ -from __future__ import print_function -from operator import itemgetter -from heapq import nlargest -from itertools import repeat, ifilter - -class Counter(dict): - '''Dict subclass for counting hashable objects. Sometimes called a bag - or multiset. Elements are stored as dictionary keys and their counts - are stored as dictionary values. - - >>> Counter('zyzygy') - Counter({'y': 3, 'z': 2, 'g': 1}) - - ''' - - def __init__(self, iterable=None, **kwds): - '''Create a new, empty Counter object. And if given, count elements - from an input iterable. Or, initialize the count from another mapping - of elements to their counts. - - >>> c = Counter() # a new, empty counter - >>> c = Counter('gallahad') # a new counter from an iterable - >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping - >>> c = Counter(a=4, b=2) # a new counter from keyword args - - ''' - self.update(iterable, **kwds) - - def __missing__(self, key): - return 0 - - def most_common(self, n=None): - '''List the n most common elements and their counts from the most - common to the least. If n is None, then list all element counts. - - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('r', 2), ('b', 2)] - - ''' - if n is None: - return sorted(self.iteritems(), key=itemgetter(1), reverse=True) - return nlargest(n, self.iteritems(), key=itemgetter(1)) - - def elements(self): - '''Iterator over elements repeating each as many times as its count. - - >>> c = Counter('ABCABC') - >>> sorted(c.elements()) - ['A', 'A', 'B', 'B', 'C', 'C'] - - If an element's count has been set to zero or is a negative number, - elements() will ignore it. - - ''' - for elem, count in self.iteritems(): - for _ in repeat(None, count): - yield elem - - # Override dict methods where the meaning changes for Counter objects. - - @classmethod - def fromkeys(cls, iterable, v=None): - raise NotImplementedError( - 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') - - def update(self, iterable=None, **kwds): - '''Like dict.update() but add counts instead of replacing them. - - Source can be an iterable, a dictionary, or another Counter instance. - - >>> c = Counter('which') - >>> c.update('witch') # add elements from another iterable - >>> d = Counter('watch') - >>> c.update(d) # add elements from another counter - >>> c['h'] # four 'h' in which, witch, and watch - 4 - - ''' - if iterable is not None: - if hasattr(iterable, 'iteritems'): - if self: - self_get = self.get - for elem, count in iterable.iteritems(): - self[elem] = self_get(elem, 0) + count - else: - dict.update(self, iterable) # fast path when counter is empty - else: - self_get = self.get - for elem in iterable: - self[elem] = self_get(elem, 0) + 1 - if kwds: - self.update(kwds) - - def copy(self): - 'Like dict.copy() but returns a Counter instance instead of a dict.' - return Counter(self) - - def __delitem__(self, elem): - 'Like dict.__delitem__() but does not raise KeyError for missing values.' - if elem in self: - dict.__delitem__(self, elem) - - def __repr__(self): - if not self: - return '%s()' % self.__class__.__name__ - items = ', '.join(map('%r: %r'.__mod__, self.most_common())) - return '%s({%s})' % (self.__class__.__name__, items) - - # Multiset-style mathematical operations discussed in: - # Knuth TAOCP Volume II section 4.6.3 exercise 19 - # and at http://en.wikipedia.org/wiki/Multiset - # - # Outputs guaranteed to only include positive counts. - # - # To strip negative and zero counts, add-in an empty counter: - # c += Counter() - - def __add__(self, other): - '''Add counts from two counters. - - >>> Counter('abbb') + Counter('bcc') - Counter({'b': 4, 'c': 2, 'a': 1}) - - - ''' - if not isinstance(other, Counter): - return NotImplemented - result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] + other[elem] - if newcount > 0: - result[elem] = newcount - return result - - def __sub__(self, other): - ''' Subtract count, but keep only results with positive counts. - - >>> Counter('abbbc') - Counter('bccd') - Counter({'b': 2, 'a': 1}) - - ''' - if not isinstance(other, Counter): - return NotImplemented - result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] - other[elem] - if newcount > 0: - result[elem] = newcount - return result - - def __or__(self, other): - '''Union is the maximum of value in either of the input counters. - - >>> Counter('abbb') | Counter('bcc') - Counter({'b': 3, 'c': 2, 'a': 1}) - - ''' - if not isinstance(other, Counter): - return NotImplemented - _max = max - result = Counter() - for elem in set(self) | set(other): - newcount = _max(self[elem], other[elem]) - if newcount > 0: - result[elem] = newcount - return result - - def __and__(self, other): - ''' Intersection is the minimum of corresponding counts. - - >>> Counter('abbb') & Counter('bcc') - Counter({'b': 1}) - - ''' - if not isinstance(other, Counter): - return NotImplemented - _min = min - result = Counter() - if len(self) < len(other): - self, other = other, self - for elem in ifilter(self.__contains__, other): - newcount = _min(self[elem], other[elem]) - if newcount > 0: - result[elem] = newcount - return result - - -if __name__ == '__main__': - import doctest - print(doctest.testmod()) diff --git a/mycli/packages/ordereddict.py b/mycli/packages/ordereddict.py deleted file mode 100644 index 5b0303f5a..000000000 --- a/mycli/packages/ordereddict.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (c) 2009 Raymond Hettinger -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -from UserDict import DictMixin - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - if last: - key = reversed(self).next() - else: - key = iter(self).next() - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - if len(self) != len(other): - return False - for p, q in zip(self.items(), other.items()): - if p != q: - return False - return True - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index f5fc3c903..41364756c 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -7,12 +7,7 @@ from .packages.special.favoritequeries import favoritequeries from re import compile, escape from .packages.tabulate import table_formats - -try: - from collections import Counter -except ImportError: - # python 2.6 - from .packages.counter import Counter +from collections import Counter _logger = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index 54a2404d3..5c70b9e3e 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,6 @@ 'License :: OSI Approved :: BSD License', 'Operating System :: Unix', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', diff --git a/tox.ini b/tox.ini index d3f9aef14..8d578d8fa 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34, py35, py36 +envlist = py27, py33, py34, py35, py36 [testenv] deps = pytest mock From 0daa21b90419da9fa4e04c7e6fd92be878c8e81f Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 6 Mar 2017 20:25:41 +0100 Subject: [PATCH 073/824] tee special command --- mycli/main.py | 2 ++ mycli/packages/special/iocommands.py | 34 ++++++++++++++++++++++++++++ tests/test_special_iocommands.py | 18 +++++++++++++++ tests/test_sqlexecute.py | 2 +- 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 6bb91b755..e83e7eba8 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -592,6 +592,7 @@ def one_iteration(document=None): self.output(str(e), err=True, fg='red') else: try: + special.write_tee(output) if special.is_pager_enabled(): self.output_via_pager('\n'.join(output)) else: @@ -649,6 +650,7 @@ def one_iteration(document=None): while True: one_iteration() except EOFError: + special.close_tee() if not self.less_chatty: self.output('Goodbye!') diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 415e33d3e..c5f7f8f4d 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -16,6 +16,7 @@ TIMING_ENABLED = False use_expanded_output = False PAGER_ENABLED = True +tee_file = None @export def set_timing_enabled(val): @@ -240,3 +241,36 @@ def execute_system_command(arg, **_): return [(None, None, None, response)] except OSError as e: return [(None, None, None, 'OSError: %s' % e.strerror)] + +@special_command('tee', 'tee [-o] filename', 'write to a output file (optionally override using -o)') +def set_tee(arg, **_): + global tee_file + if arg.startswith('-o '): + mode = "w" + filename = arg[3:] + else: + mode = 'a' + filename = arg + tee_file = open(filename, mode) + return [(None, None, None, "")] + +@export +def close_tee(): + global tee_file + if tee_file: + tee_file.close() + tee_file = None + +@special_command('notee', 'notee', 'stop writing to a output file') +def no_tee(arg, **_): + close_tee() + return [(None, None, None, "")] + +@export +def write_tee(output): + global tee_file + if tee_file: + for buf in output: + tee_file.write(buf) + tee_file.write(u"\n") + tee_file.flush() diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py index 5dbbc1602..98e4cc2f7 100644 --- a/tests/test_special_iocommands.py +++ b/tests/test_special_iocommands.py @@ -1,4 +1,5 @@ import mycli.packages.special +import tempfile import os def test_set_get_pager(): mycli.packages.special.set_pager_enabled(True) @@ -35,3 +36,20 @@ def test_editor_command(): os.environ['EDITOR'] = 'true' mycli.packages.special.open_external_editor(r'select 1') == "select 1" + +def test_tee_command(): + mycli.packages.special.write_tee([u"hello world"]) # write without file set + with tempfile.NamedTemporaryFile() as f: + mycli.packages.special.execute(None, u"tee "+f.name) + mycli.packages.special.write_tee([u"hello world"]) + assert f.read() == b"hello world\n" + + mycli.packages.special.execute(None, u"tee -o "+f.name) + mycli.packages.special.write_tee([u"hello world"]) + f.seek(0) + assert f.read() == b"hello world\n" + + mycli.packages.special.execute(None, u"notee") + mycli.packages.special.write_tee([u"hello world"]) + f.seek(0) + assert f.read() == b"hello world\n" diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index d9ed62532..903e8e6d8 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -213,7 +213,7 @@ def test_favorite_query_expanded_output(executor): @dbtest def test_special_command(executor): results = run(executor, '\\?') - expected_line = u'| help | \\? | Show this help. |\n' + expected_line = u'\n| help' assert len(results) == 1 assert expected_line in results[0] From ab2982ac22254b0a22b942e5abd11e361135cff8 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 6 Mar 2017 21:08:02 +0100 Subject: [PATCH 074/824] add queries to tee output --- mycli/main.py | 4 +++- mycli/packages/special/iocommands.py | 5 ++--- tests/test_special_iocommands.py | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index e83e7eba8..c79382fa4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -505,6 +505,7 @@ def one_iteration(document=None): try: logger.debug('sql: %r', document.text) + special.write_tee(self.get_prompt(self.prompt_format) + document.text) if self.logfile: self.logfile.write('\n# %s\n' % datetime.now()) self.logfile.write(document.text) @@ -592,7 +593,7 @@ def one_iteration(document=None): self.output(str(e), err=True, fg='red') else: try: - special.write_tee(output) + special.write_tee('\n'.join(output)) if special.is_pager_enabled(): self.output_via_pager('\n'.join(output)) else: @@ -655,6 +656,7 @@ def one_iteration(document=None): self.output('Goodbye!') def output(self, text, **kwargs): + special.write_tee(text) if self.logfile: self.logfile.write(utf8tounicode(text)) self.logfile.write('\n') diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index c5f7f8f4d..9582a574d 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -270,7 +270,6 @@ def no_tee(arg, **_): def write_tee(output): global tee_file if tee_file: - for buf in output: - tee_file.write(buf) - tee_file.write(u"\n") + tee_file.write(output) + tee_file.write(u"\n") tee_file.flush() diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py index 98e4cc2f7..78d29b0d3 100644 --- a/tests/test_special_iocommands.py +++ b/tests/test_special_iocommands.py @@ -38,18 +38,18 @@ def test_editor_command(): mycli.packages.special.open_external_editor(r'select 1') == "select 1" def test_tee_command(): - mycli.packages.special.write_tee([u"hello world"]) # write without file set + mycli.packages.special.write_tee(u"hello world") # write without file set with tempfile.NamedTemporaryFile() as f: mycli.packages.special.execute(None, u"tee "+f.name) - mycli.packages.special.write_tee([u"hello world"]) + mycli.packages.special.write_tee(u"hello world") assert f.read() == b"hello world\n" mycli.packages.special.execute(None, u"tee -o "+f.name) - mycli.packages.special.write_tee([u"hello world"]) + mycli.packages.special.write_tee(u"hello world") f.seek(0) assert f.read() == b"hello world\n" mycli.packages.special.execute(None, u"notee") - mycli.packages.special.write_tee([u"hello world"]) + mycli.packages.special.write_tee(u"hello world") f.seek(0) assert f.read() == b"hello world\n" From e53981db0d5a8617260761c11e5c9f38d8f4faa0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 7 Mar 2017 08:57:05 -0600 Subject: [PATCH 075/824] Add error messages for tee command. --- mycli/packages/special/iocommands.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 9582a574d..4d8206432 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -251,7 +251,15 @@ def set_tee(arg, **_): else: mode = 'a' filename = arg - tee_file = open(filename, mode) + + if not filename: + raise TypeError('You must provide a filename.') + + try: + tee_file = open(filename, mode) + except OSError as e: + raise OSError("Cannot write to file '{}': {}".format(e.filename, e.strerror)) + return [(None, None, None, "")] @export From d26f3d745459f8da7e62ca14a5840dcef396c730 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 7 Mar 2017 08:58:11 -0600 Subject: [PATCH 076/824] Grammar fix. --- mycli/packages/special/iocommands.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 4d8206432..650b4ff93 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -242,7 +242,8 @@ def execute_system_command(arg, **_): except OSError as e: return [(None, None, None, 'OSError: %s' % e.strerror)] -@special_command('tee', 'tee [-o] filename', 'write to a output file (optionally override using -o)') +@special_command('tee', 'tee [-o] filename', + 'write to an output file (optionally overwrite using -o)') def set_tee(arg, **_): global tee_file if arg.startswith('-o '): @@ -269,7 +270,7 @@ def close_tee(): tee_file.close() tee_file = None -@special_command('notee', 'notee', 'stop writing to a output file') +@special_command('notee', 'notee', 'stop writing to an output file') def no_tee(arg, **_): close_tee() return [(None, None, None, "")] From 7266570b1c327ee227064f1b1a75216feba1a09b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 7 Mar 2017 09:17:46 -0600 Subject: [PATCH 077/824] Add tee tests for raised errors. --- tests/test_special_iocommands.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py index 78d29b0d3..eddfbf8ac 100644 --- a/tests/test_special_iocommands.py +++ b/tests/test_special_iocommands.py @@ -1,6 +1,12 @@ -import mycli.packages.special -import tempfile import os +import stat +import tempfile + +import pytest + +import mycli.packages.special + + def test_set_get_pager(): mycli.packages.special.set_pager_enabled(True) assert mycli.packages.special.is_pager_enabled() @@ -53,3 +59,12 @@ def test_tee_command(): mycli.packages.special.write_tee(u"hello world") f.seek(0) assert f.read() == b"hello world\n" + +def test_tee_command_error(): + with pytest.raises(TypeError): + mycli.packages.special.execute(None, 'tee') + + with pytest.raises(OSError): + with tempfile.NamedTemporaryFile() as f: + os.chmod(f.name, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + mycli.packages.special.execute(None, 'tee {}'.format(f.name)) From 05190ec683b124f4da13c9c31a4c6b4669f2b9e0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 7 Mar 2017 09:23:01 -0600 Subject: [PATCH 078/824] Catch Python2 tee error. --- mycli/packages/special/iocommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 650b4ff93..4d4fcc655 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -258,7 +258,7 @@ def set_tee(arg, **_): try: tee_file = open(filename, mode) - except OSError as e: + except (IOError, OSError) as e: raise OSError("Cannot write to file '{}': {}".format(e.filename, e.strerror)) return [(None, None, None, "")] From 71cb83899b3ceaa88602445c27a7d117cec058f5 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 7 Mar 2017 20:14:30 +0100 Subject: [PATCH 079/824] Update Detailed Install Instructions for Fedora --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6b682a2c4..6ecca526c 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,17 @@ Twitter: [@amjithr](http://twitter.com/amjithr) ## Detailed Install Instructions: -### RHEL, Centos, Fedora: +### Fedora -I haven't built an RPM package for mycli yet. So please use `pip` to install `mycli`. You can install pip on your system using: +Fedora has a package available for mycli, install it using dnf: + +``` +$ sudo dnf install mycli +``` + +### RHEL, Centos + +I haven't built an RPM package for mycli for RHEL or Centos yet. So please use `pip` to install `mycli`. You can install pip on your system using: ``` $ sudo yum install python-pip From 20f743c3b1353816124cd7ac3d0d18af335ca440 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 11 Mar 2017 13:37:08 +0100 Subject: [PATCH 080/824] pytest environment variables use environment variables for pytest for alternative host/port/etc. --- tests/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index dda72dddf..dd7725cc0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,9 +3,11 @@ from mycli.packages import connection from os import getenv -# TODO: should this be somehow be divined from environment? -USER, HOST, PORT, CHARSET = 'root', 'localhost', 3306, 'utf8' -PASSWORD = getenv('PASSWORD') +PASSWORD = getenv('PYTEST_PASSWORD') +USER = getenv('PYTEST_USER', 'root') +HOST = getenv('PYTEST_HOST', 'localhost') +PORT = getenv('PYTEST_PORT', 3306) +CHARSET = getenv('PYTEST_CHARSET', 'utf8') def db_connection(dbname=None): conn = connection.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, From 14c0bd2177d2ece01bd820f16b97e5d964ab15f2 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 11 Mar 2017 08:08:58 -0800 Subject: [PATCH 081/824] Update AUTHORS file. --- AUTHORS | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 316fa8979..66fdb7cb1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,12 +1,16 @@ -Many thanks to the following contributors. +Project Lead: +------------- + * Thomas Roten + Core Developers: ---------------- - * Thomas Roten * Iryna Cherniavska * Matheus Rosa * Darik Gamble + * Dick Marinus + * Amjith Ramanujam Contributors: ------------- From b78747c4517d9925804113d7719af462642ef9ff Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 17 Mar 2017 20:21:04 -0500 Subject: [PATCH 082/824] Add 1.9.0 changes. --- changelog.md | 71 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/changelog.md b/changelog.md index 2bd345e87..8ee9db42f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,38 @@ +1.9.0: +====== + +Features: +--------- + +* Add tee/notee commands for outputing results to a file. (Thanks: [Dick Marinus]). +* Add date, port, and whitespace options to prompt configuration. (Thanks: [Matheus Rosa]). +* Allow user to specify LESS pager flags. (Thanks: [John Sterling]). +* Add support for auto-reconnect. (Thanks: [Jialong Liu]). +* Add CSV batch output. (Thanks: [Matheus Rosa]). +* Add auto_vertical_output config to myclirc. (Thanks: [Matheus Rosa]). +* Improve Fedora install instructions. (Thanks: [Dick Marinus]). + +Bug Fixes: +---------- + +* Fix crashes occuring from commands starting with #. (Thanks: [Zhidong]). +* Fix broken PyMySQL link in README. (Thanks: [Daniël van Eeden]). +* Add various missing keywords for highlighting and autocompletion. (Thanks: [zer09]). +* Add the missing REGEXP keyword for highlighting and autocompletion. (Thanks: [cxbig]). +* Fix duplicate username entries in completion list. (Thanks: [John Sterling]). +* Remove extra spaces in TSV table format output. (Thanks: [Dick Marinus]). +* Kill running query when interrupted via Ctrl-C. (Thanks: [chainkite]). +* Read the smart_completion config from myclirc. (Thanks: [Thomas Roten]). + +Internal Changes: +----------------- + +* Improve handling of test database credentials. (Thanks: [Dick Marinus]). +* Add Python 3.6 to test environments and PyPI metadata. (Thanks: [Thomas Roten]). +* Drop Python 2.6 support. (Thanks: [Thomas Roten]). +* Swap pycrypto dependency for pycryptodome. (Thanks: [Michał Górny]). +* Bump sqlparse version so pgcli and mycli can be installed together. (Thanks: [darikg]). + 1.8.1: ====== @@ -60,7 +95,7 @@ Internal Changes: Features: --------- -* Change continuation prompt for multi-line mode to match default mysql. +* Change continuation prompt for multi-line mode to match default mysql. * Add `status` command to match mysql's `status` command. (Thanks: [Thomas Roten]). * Add SSL support for `mycli`. (Thanks: [Artem Bezsmertnyi]). * Add auto-completion and highlight support for OFFSET keyword. (Thanks: [Matheus Rosa]). @@ -102,7 +137,7 @@ Bug Fixes: Bug Fixes: ---------- -* Cast the value of port read from my.cnf to int. +* Cast the value of port read from my.cnf to int. 1.5.0: ====== @@ -115,7 +150,7 @@ Features: This feature is only available when `pycrypto` package is installed. * Register the special command `prompt` with the `\R` as alias. (Thanks: [Matheus Rosa]). Users can now change the mysql prompt at runtime using `prompt` command. - eg: + eg: ``` mycli> prompt \u@\h> Changed prompt format to \u@\h> @@ -162,9 +197,9 @@ Internal Changes: Features: --------- -* Add `source` command. This allows running sql statement from a file. +* Add `source` command. This allows running sql statement from a file. - eg: + eg: ``` mycli> source filename.sql ``` @@ -175,13 +210,13 @@ Features: disable the warning before running `DROP` commands. * Add completion support for CHANGE TO and other master/slave commands. This is - still preliminary and it will be enhanced in the future. + still preliminary and it will be enhanced in the future. -* Add custom styles to color the menus and toolbars. +* Add custom styles to color the menus and toolbars. -* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders]) +* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders]) - Multi-line queries are automatically indented. + Multi-line queries are automatically indented. Bug Fixes: ---------- @@ -196,7 +231,7 @@ Bug Fixes: Features: --------- * Add a new special command (\T) to change the table format on the fly. (Thanks: [Jonathan Bruno](https://github.com/brewneaux)) - eg: + eg: ``` mycli> \T tsv ``` @@ -217,7 +252,7 @@ Features: [clientamjith] user = 'amjith' database = 'user_management' - + $ mycli --defaults-group-suffix=amjith # uses the [clientamjith] section in my.cnf ``` @@ -225,7 +260,7 @@ Features: `my.cnf` to use at launch. This also makes it play nice with mysql sandbox. * Make `-p` and `--password` take the password in commandline. This makes mycli - a drop in replacement for mysql. + a drop in replacement for mysql. 1.2.0: ====== @@ -240,7 +275,7 @@ Features: Bug Fixes: --------- -* Prevent Ctrl-C from quitting mycli while the pager is active. +* Prevent Ctrl-C from quitting mycli while the pager is active. * Refresh auto-completions after the database is changed via a CONNECT command. Internal Changes: @@ -289,7 +324,7 @@ Features: * Customizable prompt. (Thanks [Steve Robbins](https://github.com/steverobbins)) * Make `\G` formatting to behave more like mysql. - + Bug Fixes: ---------- @@ -302,7 +337,7 @@ Bug Fixes: Features: --------- -* Upgrade prompt_toolkit to 0.38. This improves the performance of pasting long queries. +* Upgrade prompt_toolkit to 0.38. This improves the performance of pasting long queries. * Add support for reading my.cnf files. * Add editor command \e. * Replace ConfigParser with ConfigObj. @@ -327,12 +362,12 @@ Features: * Add support for connecting via socket. * Add completion for SQL functions. * Add completion support for SHOW statements. -* Made the timing of sql statements human friendly. +* Made the timing of sql statements human friendly. * Automatically prompt for a password if needed. Bug Fixes: ---------- -* Fixed the installation issues with PyMySQL dependency on case-sensitive file systems. +* Fixed the installation issues with PyMySQL dependency on case-sensitive file systems. [Daniel West]: http://github.com/danieljwest [Iryna Cherniavska]: https://github.com/j-bennet @@ -342,7 +377,7 @@ Bug Fixes: [Shoma Suzuki]: https://github.com/shoma [spacewander]: https://github.com/spacewander [Thomas Roten]: https://github.com/tsroten -[Artem Bezsmertnyi]: https://github.com/mrdeathless +[Artem Bezsmertnyi]: https://github.com/mrdeathless [Mikhail Borisov]: https://github.com/borman [Casper Langemeijer]: Casper Langemeijer [Lennart Weller]: https://github.com/lhw From be590c2473b31bc4d585aacc2547000fff5b028b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 17 Mar 2017 20:21:12 -0500 Subject: [PATCH 083/824] Add new contributor names. --- AUTHORS | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/AUTHORS b/AUTHORS index 66fdb7cb1..b5c88bdc6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -42,6 +42,14 @@ Contributors: * jbruno * mrdeathless * Abirami P + * John Sterling + * Jialong Liu + * Zhidong + * Daniël van Eeden + * zer09 + * cxbig + * chainkite + * Michał Górny Creator: -------- From 8aa055896d267ffbb27e87b818816a1ffaba3187 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 17 Mar 2017 20:21:26 -0500 Subject: [PATCH 084/824] Remove duplicate name. --- AUTHORS | 1 - 1 file changed, 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b5c88bdc6..2eb4af10b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -19,7 +19,6 @@ Contributors: * Shoma Suzuki * Daniel West * Scrappy Soft - * Dick Marinus * Daniel Black * Jonathan Bruno * Casper Langemeijer From d5b5e8280d778d57da4b7e659853d05cbf5da3d2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 13:28:37 -0500 Subject: [PATCH 085/824] Create PULL_REQUEST_TEMPLATE.md Adds a pull request template with sections for: - description - checklist --- .github/PULL_REQUEST_TEMPLATE.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..6ea05678c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +## Description + + + + +## Checklist + +- [ ] I've added this contribution to the `changelog.md`. +- [ ] I've added my name to the `AUTHORS` file (or it's already there). From b789fd30acf4b4063dbe4b257a3a0ac68e90e29d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 13:29:39 -0500 Subject: [PATCH 086/824] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6ea05678c..8d498abcd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,5 @@ ## Description - + From afa6a0f4e35802566b50c00bc1f146fe63b86ba3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 18:41:19 -0500 Subject: [PATCH 087/824] Use input for Python 3. --- release.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/release.py b/release.py index 622056274..845c83f8f 100644 --- a/release.py +++ b/release.py @@ -6,6 +6,11 @@ import sys from optparse import OptionParser +try: + input = raw_input +except NameError: + pass + DEBUG = False CONFIRM_STEPS = False DRY_RUN = False @@ -19,7 +24,7 @@ def skip_step(): global CONFIRM_STEPS if CONFIRM_STEPS: - choice = raw_input("--- Confirm step? (y/N) [y] ") + choice = input("--- Confirm step? (y/N) [y] ") if choice.lower() == 'n': return True return False @@ -87,7 +92,7 @@ def push_tags_to_github(): def checklist(questions): for question in questions: - choice = raw_input(question + ' (y/N) [n] ') + choice = input(question + ' (y/N) [n] ') if choice.lower() != 'y': sys.exit(1) @@ -119,7 +124,7 @@ def checklist(questions): CONFIRM_STEPS = popts.confirm_steps DRY_RUN = popts.dry_run - choice = raw_input('Are you sure? (y/N) [n] ') + choice = input('Are you sure? (y/N) [n] ') if choice.lower() != 'y': sys.exit(1) From 885b0b3643da479eed508dcc30ac76200972aeae Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 18:42:13 -0500 Subject: [PATCH 088/824] Add extra newlines for PEP8. --- release.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release.py b/release.py index 845c83f8f..4682b0517 100644 --- a/release.py +++ b/release.py @@ -75,9 +75,11 @@ def register_with_pypi(): def create_source_tarball(): run_step('python', 'setup.py', 'sdist') + def create_python_wheel(): run_step('python', 'setup.py', 'sdist', 'bdist_wheel') + def upload_source_tarball(): run_step('python', 'setup.py', 'sdist', 'upload') From a1eb31f2dc78a18172df29f86416c73d787e32f1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 18:57:28 -0500 Subject: [PATCH 089/824] Make the release script executable. --- release.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 release.py diff --git a/release.py b/release.py old mode 100644 new mode 100755 From 49c23eaadee84a62cf2b81f115c3be189945e755 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 20:07:00 -0500 Subject: [PATCH 090/824] Releasing version 1.9.0 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index e8b6b090b..e5102d301 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.8.1' +__version__ = '1.9.0' From 2016c6d5b4c42c6c2db9a79f19f78a6bd7052fed Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 20:35:30 -0500 Subject: [PATCH 091/824] Use twine for uploading to PyPI. --- release.py | 9 +++++---- requirements-dev.txt | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/release.py b/release.py index 622056274..37865b590 100644 --- a/release.py +++ b/release.py @@ -71,10 +71,11 @@ def create_source_tarball(): run_step('python', 'setup.py', 'sdist') def create_python_wheel(): - run_step('python', 'setup.py', 'sdist', 'bdist_wheel') + run_step('python', 'setup.py', 'bdist_wheel') -def upload_source_tarball(): - run_step('python', 'setup.py', 'sdist', 'upload') + +def upload_distribution_files(): + run_step('twine', 'upload', 'dist/*') def push_to_github(): @@ -130,4 +131,4 @@ def checklist(questions): create_python_wheel() push_to_github() push_tags_to_github() - upload_source_tarball() + upload_distribution_files() diff --git a/requirements-dev.txt b/requirements-dev.txt index ca4019c29..e54eabb9c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ mock pytest tox +twine==1.8.1 From 74d76991aa5e7d27b638654c278fb81e5ce063d0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 20:40:49 -0500 Subject: [PATCH 092/824] Add twine change to change log. --- changelog.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/changelog.md b/changelog.md index 8ee9db42f..f8573009d 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,12 @@ +TBD +=== + +Internal Changes: +----------------- + +* Upload mycli distributions in a safer manner (using twine). (Thanks: [Thomas + Roten]). + 1.9.0: ====== From 127eb6ead4d4399eaa641de633322b74f3cc6a34 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 22:38:26 -0500 Subject: [PATCH 093/824] Merge the distribution functions. --- release.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/release.py b/release.py index 23316b324..9a5d399f9 100755 --- a/release.py +++ b/release.py @@ -72,12 +72,8 @@ def register_with_pypi(): run_step('python', 'setup.py', 'register') -def create_source_tarball(): - run_step('python', 'setup.py', 'sdist') - - -def create_python_wheel(): - run_step('python', 'setup.py', 'bdist_wheel') +def create_distribution_files(): + run_step('python', 'setup.py', 'sdist', 'bdist_wheel') def upload_distribution_files(): @@ -133,8 +129,7 @@ def checklist(questions): commit_for_release('mycli/__init__.py', ver) create_git_tag('v%s' % ver) register_with_pypi() - create_source_tarball() - create_python_wheel() + create_distribution_files() push_to_github() push_tags_to_github() upload_distribution_files() From 9a61fa165678184947e51a2e39a54409d01b12c3 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Mon, 20 Mar 2017 10:45:16 -0700 Subject: [PATCH 094/824] Fixes editor bug with prompt_toolkit 1.0.13. --- mycli/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index db38fa2bc..05456546c 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -422,6 +422,10 @@ def handle_editor_command(self, cli, document): :param document: Document :return: Document """ + # FIXME: using application.pre_run_callables like this here is not the best solution. + # It's internal api of prompt_toolkit that may change. This was added to fix + # https://github.com/dbcli/pgcli/issues/668. We may find a better way to do it in the future. + saved_callables = cli.application.pre_run_callables while special.editor_command(document.text): filename = special.get_filename(document.text) sql, message = special.open_external_editor(filename, @@ -430,8 +434,10 @@ def handle_editor_command(self, cli, document): # Something went wrong. Raise an exception and bail. raise RuntimeError(message) cli.current_buffer.document = Document(sql, cursor_position=len(sql)) - document = cli.run(False) + cli.application.pre_run_callables = [] + document = cli.run() continue + cli.application.pre_run_callables = saved_callables return document def run_cli(self): @@ -464,7 +470,7 @@ def get_continuation_tokens(cli, width): def one_iteration(document=None): if document is None: - document = self.cli.run(reset_current_buffer=True) + document = self.cli.run() special.set_expanded_output(False) From 86d0f9e916439cd5fc1ad5bf6a7f92a06b346f1f Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Mon, 20 Mar 2017 10:49:53 -0700 Subject: [PATCH 095/824] Added fix to changelog. --- changelog.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index f8573009d..d6f4ec96a 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Bug Fixes: +---------- + +* Fix external editor bug (issue #377). (Thanks: [Irina Truong]). + Internal Changes: ----------------- @@ -194,9 +199,9 @@ Bug Fixes: Internal Changes: ----------------- -* Make pycrypto optional and only install it in \*nix systems. (Thanks: [Iryna Cherniavska]). +* Make pycrypto optional and only install it in \*nix systems. (Thanks: [Irina Truong]). * Add badge for PyPI version to README. (Thanks: [Shoma Suzuki]). -* Updated release script with a --dry-run and --confirm-steps option. (Thanks: [Iryna Cherniavska]). +* Updated release script with a --dry-run and --confirm-steps option. (Thanks: [Irina Truong]). * Adds support for PyMySQL 0.6.2 and above. This is useful for debian package builders. (Thanks: [Thomas Roten]). * Disable click warning. @@ -245,7 +250,7 @@ Features: mycli> \T tsv ``` * Add `--defaults-group-suffix` to the command line. This lets the user specify - a group to use in the my.cnf files. (Thanks: [Iryna Cherniavska](http://github.com/j-bennet)) + a group to use in the my.cnf files. (Thanks: [Irina Truong](http://github.com/j-bennet)) In the my.cnf file a user can specify credentials for different databases and invoke mycli with the group name to use the appropriate credentials. @@ -310,7 +315,7 @@ Features: * Fuzzy completion is now case-insensitive. (Thanks: [bjarnagin](https://github.com/bjarnagin)) * Added new-line (`\n`) to the list of special characters to use in prompt. (Thanks: [brewneaux](https://github.com/brewneaux)) -* Honor the `pager` setting in my.cnf files. (Thanks: [Iryna Cherniavska](http://github.com/j-bennet)) +* Honor the `pager` setting in my.cnf files. (Thanks: [Irina Truong](http://github.com/j-bennet)) Bug Fixes: ---------- @@ -379,7 +384,7 @@ Bug Fixes: * Fixed the installation issues with PyMySQL dependency on case-sensitive file systems. [Daniel West]: http://github.com/danieljwest -[Iryna Cherniavska]: https://github.com/j-bennet +[Irina Truong]: https://github.com/j-bennet [Kacper Kwapisz]: https://github.com/KKKas [Martijn Engler]: https://github.com/martijnengler [Matheus Rosa]: https://github.com/mdsrosa From ba0f5bffefcc5767b84a08e5b691c6f4c4dddea2 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Tue, 21 Mar 2017 21:21:37 -0700 Subject: [PATCH 096/824] Bumped minimal requirement of prompt-toolkit to 1.0.10, where reset_current_buffer was deprecated. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5c70b9e3e..5c08d1b59 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ install_requirements = [ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? - 'prompt_toolkit>=1.0.0,<1.1.0', + 'prompt_toolkit>=1.0.10,<1.1.0', 'PyMySQL >= 0.6.2', 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.6', From 29f28e414aac665c3185221480db61955df53e31 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 21 Mar 2017 23:22:51 -0500 Subject: [PATCH 097/824] Open config file using UTF8 encoding. --- mycli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/config.py b/mycli/config.py index 97ead1fcd..df669fe24 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -40,7 +40,7 @@ def read_config_file(f): f = os.path.expanduser(f) try: - config = ConfigObj(f, interpolation=False) + config = ConfigObj(f, interpolation=False, encoding='utf8') except ConfigObjError as e: log(logger, logging.ERROR, "Unable to parse line {0} of config file " "'{1}'.".format(e.line_number, f)) From f44d7453e83c04251651a63f08fab522821569ca Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 21 Mar 2017 23:25:24 -0500 Subject: [PATCH 098/824] Add config utf-8 change to change log. --- changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index d6f4ec96a..1ae1a13f0 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,8 @@ Bug Fixes: ---------- * Fix external editor bug (issue #377). (Thanks: [Irina Truong]). +* Fixed bug so that favorite queries can include unicode characters. (Thanks: + [Thomas Roten]). Internal Changes: ----------------- From f0694e01421852adf66eff05863036cfb2edda48 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Wed, 22 Mar 2017 07:33:56 +0100 Subject: [PATCH 099/824] Add unit test for unicode in favorite queries --- tests/test_special_iocommands.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py index eddfbf8ac..2ed2dbad8 100644 --- a/tests/test_special_iocommands.py +++ b/tests/test_special_iocommands.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import stat import tempfile @@ -5,6 +6,7 @@ import pytest import mycli.packages.special +import utils def test_set_get_pager(): @@ -68,3 +70,9 @@ def test_tee_command_error(): with tempfile.NamedTemporaryFile() as f: os.chmod(f.name, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) mycli.packages.special.execute(None, 'tee {}'.format(f.name)) + +def test_favorite_query(): + with utils.db_connection().cursor() as cur: + query = u'select "✔"' + mycli.packages.special.execute(cur, u'\\fs check {0}'.format(query)) + assert next(mycli.packages.special.execute(cur, u'\\f check'))[0] == "> " + query From 3c95e185a4f51895a3079b6799bd0d73c183f3f3 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Wed, 22 Mar 2017 17:02:03 +0100 Subject: [PATCH 100/824] Fix requirements Pygments lowered to >= 1.6 PyMySQL raised to >= 0.6.7 configobj lowered to 5.0.5 pycryptodome set minimum to 3 --- .travis.yml | 5 +--- changelog.md | 19 +++++++------- mycli/config.py | 13 +-------- mycli/main.py | 29 ++++++-------------- mycli/packages/connection.py | 51 ------------------------------------ mycli/sqlexecute.py | 6 ++--- setup.py | 8 +++--- tests/test_config.py | 11 ++------ tests/test_sqlexecute.py | 4 --- tests/utils.py | 6 ++--- 10 files changed, 32 insertions(+), 120 deletions(-) delete mode 100644 mycli/packages/connection.py diff --git a/.travis.yml b/.travis.yml index 516df2604..1a7991836 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,8 @@ python: - "3.5" - "3.6" -env: - - PYMYSQL_VERSION=0.6.7 - install: - - pip install PyMySQL==$PYMYSQL_VERSION . pytest mock codecov + - pip install PyMySQL . pytest mock codecov script: - coverage run --source mycli -m py.test diff --git a/changelog.md b/changelog.md index 1ae1a13f0..ae59ec7ac 100644 --- a/changelog.md +++ b/changelog.md @@ -7,6 +7,7 @@ Bug Fixes: * Fix external editor bug (issue #377). (Thanks: [Irina Truong]). * Fixed bug so that favorite queries can include unicode characters. (Thanks: [Thomas Roten]). +* Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) Internal Changes: ----------------- @@ -25,7 +26,7 @@ Features: * Allow user to specify LESS pager flags. (Thanks: [John Sterling]). * Add support for auto-reconnect. (Thanks: [Jialong Liu]). * Add CSV batch output. (Thanks: [Matheus Rosa]). -* Add auto_vertical_output config to myclirc. (Thanks: [Matheus Rosa]). +* Add `auto_vertical_output` config to myclirc. (Thanks: [Matheus Rosa]). * Improve Fedora install instructions. (Thanks: [Dick Marinus]). Bug Fixes: @@ -38,7 +39,7 @@ Bug Fixes: * Fix duplicate username entries in completion list. (Thanks: [John Sterling]). * Remove extra spaces in TSV table format output. (Thanks: [Dick Marinus]). * Kill running query when interrupted via Ctrl-C. (Thanks: [chainkite]). -* Read the smart_completion config from myclirc. (Thanks: [Thomas Roten]). +* Read the `smart_completion` config from myclirc. (Thanks: [Thomas Roten]). Internal Changes: ----------------- @@ -72,7 +73,7 @@ Features: * Add support for --execute/-e commandline arg. (Thanks: [Matheus Rosa]). * Add `less_chatty` config option to skip the intro messages. (Thanks: [Scrappy Soft]). -* Support MYCLI_HISTFILE environment variable to specify where to write the history file. (Thanks: [Scrappy Soft]). +* Support `MYCLI_HISTFILE` environment variable to specify where to write the history file. (Thanks: [Scrappy Soft]). * Add `prompt_continuation` config option to allow configuring the continuation prompt for multi-line queries. (Thanks: [Scrappy Soft]). * Display login-path instead of host in prompt. (Thanks: [Irina Truong]). @@ -103,7 +104,7 @@ Bug Fixes: Internal Changes: ----------------- -* Upgrade prompt_toolkit to 1.0.0. (Thanks: [Jonathan Slenders]) +* Upgrade `prompt_toolkit` to 1.0.0. (Thanks: [Jonathan Slenders]) 1.6.0: ====== @@ -131,7 +132,7 @@ Bug Fixes: Internal Changes: ----------------- -* Upgrade prompt_toolkit to 0.60. +* Upgrade `prompt_toolkit` to 0.60. * Add Python 3.5 to test environments. (Thanks: [Thomas Roten]). * Remove license meta-data. (Thanks: [Thomas Roten]). * Skip binary tests if PyMySQL version does not support it. (Thanks: [Thomas Roten]). @@ -230,7 +231,7 @@ Features: * Add custom styles to color the menus and toolbars. -* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders]) +* Upgrade `prompt_toolkit` to 0.46. (Thanks: [Jonathan Slenders]) Multi-line queries are automatically indented. @@ -297,7 +298,7 @@ Bug Fixes: Internal Changes: ----------------- -* Upgrade prompt_toolkit dependency version to 0.45. +* Upgrade `prompt_toolkit` dependency version to 0.45. * Added Travis CI to run the tests automatically. 1.1.1: @@ -329,7 +330,7 @@ Internal Changes: ----------------- * Changed pymysql version to be greater than 0.6.6. -* Upgrade prompt_toolkit version to 0.42. (Thanks: [Yasuhiro Matsumoto](https://github.com/mattn)) +* Upgrade `prompt_toolkit` version to 0.42. (Thanks: [Yasuhiro Matsumoto](https://github.com/mattn)) * Removed the explicit dependency on six. 2015/06/10: @@ -353,7 +354,7 @@ Bug Fixes: Features: --------- -* Upgrade prompt_toolkit to 0.38. This improves the performance of pasting long queries. +* Upgrade `prompt_toolkit` to 0.38. This improves the performance of pasting long queries. * Add support for reading my.cnf files. * Add editor command \e. * Replace ConfigParser with ConfigObj. diff --git a/mycli/config.py b/mycli/config.py index df669fe24..7f5e0cb25 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -11,17 +11,8 @@ basestring except NameError: basestring = str -try: - from Crypto.Cipher import AES -except ImportError: - AES = None - +from Crypto.Cipher import AES -class CryptoError(Exception): - """ - Exception to signal about pycrypto(dome) not available. - """ - pass logger = logging.getLogger(__name__) @@ -125,8 +116,6 @@ def read_and_decrypt_mylogin_cnf(f): :return: the decrypted login path file :rtype: io.BytesIO or None """ - if AES is None: - raise CryptoError('pycrypto(dome) is not available.') # Number of bytes used to store the length of ciphertext. MAX_CIPHER_STORE_LEN = 4 diff --git a/mycli/main.py b/mycli/main.py index 05456546c..04bd26462 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -46,7 +46,7 @@ from .clibuffer import CLIBuffer from .completion_refresher import CompletionRefresher from .config import (write_default_config, get_mylogin_cnf_path, - open_mylogin_cnf, CryptoError, read_config_file, + open_mylogin_cnf, read_config_file, read_config_files, str_to_bool) from .key_bindings import mycli_bindings from .encodingutils import utf8tounicode @@ -165,17 +165,13 @@ def __init__(self, sqlexecute=None, prompt=None, # Load .mylogin.cnf if it exists. mylogin_cnf_path = get_mylogin_cnf_path() if mylogin_cnf_path: - try: - mylogin_cnf = open_mylogin_cnf(mylogin_cnf_path) - if mylogin_cnf_path and mylogin_cnf: - # .mylogin.cnf gets read last, even if defaults_file is specified. - self.cnf_files.append(mylogin_cnf) - elif mylogin_cnf_path and not mylogin_cnf: - # There was an error reading the login path file. - print('Error: Unable to read login path file.') - except CryptoError: - click.secho('Warning: .mylogin.cnf was not read: pycrypto(dome) ' - 'module is not available.') + mylogin_cnf = open_mylogin_cnf(mylogin_cnf_path) + if mylogin_cnf_path and mylogin_cnf: + # .mylogin.cnf gets read last, even if defaults_file is specified. + self.cnf_files.append(mylogin_cnf) + elif mylogin_cnf_path and not mylogin_cnf: + # There was an error reading the login path file. + print('Error: Unable to read login path file.') self.cli = None @@ -545,15 +541,6 @@ def one_iteration(document=None): end = time() total += end - start mutating = mutating or is_mutating(status) - except UnicodeDecodeError as e: - import pymysql - if pymysql.VERSION < (0, 6, 7): - message = ('You are running an older version of pymysql.\n' - 'Please upgrade to 0.6.7 or above to view binary data.\n' - 'Try \'pip install -U pymysql\'.') - self.output(message) - else: - raise e except KeyboardInterrupt: # get last connection id connection_id_to_kill = sqlexecute.connection_id diff --git a/mycli/packages/connection.py b/mycli/packages/connection.py deleted file mode 100644 index 5ece3f9eb..000000000 --- a/mycli/packages/connection.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Connection and cursor wrappers around PyMySQL. - -This module effectively backports PyMySQL functionality and error handling -so that mycli will support Debian's python-pymysql version (0.6.2). -""" - -import pymysql - -Cursor = pymysql.cursors.Cursor -connect = pymysql.connect - - -if pymysql.VERSION[1] == 6 and pymysql.VERSION[2] < 5: - class Cursor(pymysql.cursors.Cursor): - """Makes Cursor a context manager in PyMySQL < 0.6.5.""" - - def __enter__(self): - return self - - def __exit__(self, *exc_info): - del exc_info - self.close() - - -if pymysql.VERSION[1] == 6 and pymysql.VERSION[2] < 3: - class Connection(pymysql.connections.Connection): - """Adds error handling to Connection in PyMySQL < 0.6.3.""" - - def __del__(self): - if self.socket: - try: - self.socket.close() - except: - pass - self.socket = None - self._rfile = None - - def connect(*args, **kwargs): - """Makes connect() use our custom Connection class. - - PyMySQL < 0.6.3 uses the *passwd* argument instead of *password*. This - function renames that keyword or assigns it the default value of '', - which is the same default value PyMySQL gives it. - - See pymysql.connections.Connection.__init__() for more information - about calling this function. - """ - - kwargs['passwd'] = kwargs.pop('password', '') - - return Connection(*args, **kwargs) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index b28008e82..786c8f0ec 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -1,7 +1,7 @@ import logging import pymysql import sqlparse -from .packages import connection, special +from .packages import special from pymysql.constants import FIELD_TYPE from pymysql.converters import (convert_mysql_timestamp, convert_datetime, convert_timedelta, convert_date) @@ -72,11 +72,11 @@ def connect(self, database=None, user=None, password=None, host=None, FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), } - conn = connection.connect(database=db, user=user, password=password, + conn = pymysql.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, use_unicode=True, charset=charset, autocommit=True, client_flag=pymysql.constants.CLIENT.INTERACTIVE, - cursorclass=connection.Cursor, local_infile=local_infile, + local_infile=local_infile, conv=conv, ssl=ssl) if hasattr(self, 'conn'): self.conn.close() diff --git a/setup.py b/setup.py index 5c08d1b59..82cdef6e1 100644 --- a/setup.py +++ b/setup.py @@ -13,12 +13,12 @@ install_requirements = [ 'click >= 4.1', - 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? + 'Pygments >= 1.6', 'prompt_toolkit>=1.0.10,<1.1.0', - 'PyMySQL >= 0.6.2', + 'PyMySQL >= 0.6.7', 'sqlparse>=0.2.2,<0.3.0', - 'configobj >= 5.0.6', - 'pycryptodome', + 'configobj >= 5.0.5', + 'pycryptodome >= 3', ] setup( diff --git a/tests/test_config.py b/tests/test_config.py index 7879dd109..2a0d26c18 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,9 +7,8 @@ import tempfile import pytest -from mycli.config import (CryptoError, get_mylogin_cnf_path, - open_mylogin_cnf, read_and_decrypt_mylogin_cnf, - str_to_bool) +from mycli.config import (get_mylogin_cnf_path, open_mylogin_cnf, + read_and_decrypt_mylogin_cnf, str_to_bool) with_pycryptodome = ['pycryptodome' in set([package.project_name for package in pip.get_installed_distributions()])] @@ -26,12 +25,6 @@ def open_bmylogin_cnf(name): return buf -@pytest.mark.skipif(with_pycryptodome, reason='requires pycryptodome missing') -def test_read_mylogin_cnf_without_crypto(): - with pytest.raises(CryptoError): - mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) - - @pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_read_mylogin_cnf(): """Tests that a login path file can be read and decrypted.""" diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index 903e8e6d8..e9929a70f 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -7,8 +7,6 @@ from utils import run, dbtest, set_expanded_output -pymysql_support_binary = pymysql.VERSION >= (0, 6, 7) - @dbtest def test_conn(executor): run(executor, '''create table test(a text)''') @@ -36,7 +34,6 @@ def test_bools(executor): 1 row in set""") @dbtest -@pytest.mark.skipif(not pymysql_support_binary, reason='pymysql < 0.6.7') def test_binary(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') run(executor, '''INSERT INTO bt VALUES (GeomFromText('LINESTRING(116.37604 39.73979,116.375 39.73965)'));''') @@ -50,7 +47,6 @@ def test_binary(executor): 1 row in set""") @dbtest -@pytest.mark.skipif(not pymysql_support_binary, reason='pymysql < 0.6.7') def test_binary_expanded(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') run(executor, '''INSERT INTO bt VALUES (GeomFromText('LINESTRING(116.37604 39.73979,116.375 39.73965)'));''') diff --git a/tests/utils.py b/tests/utils.py index dd7725cc0..c76cb7ad3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,6 @@ import pytest +import pymysql from mycli.main import format_output, special -from mycli.packages import connection from os import getenv PASSWORD = getenv('PYTEST_PASSWORD') @@ -10,8 +10,8 @@ CHARSET = getenv('PYTEST_CHARSET', 'utf8') def db_connection(dbname=None): - conn = connection.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, - charset=CHARSET, cursorclass=connection.Cursor, + conn = pymysql.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, + charset=CHARSET, local_infile=False) conn.autocommit = True return conn From 59e1c003f9401ec51fdd0c4bd297bf0b3793dc51 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 23 Mar 2017 10:05:44 -0500 Subject: [PATCH 101/824] Open thanks files using UTF-8 encoding. --- changelog.md | 2 ++ mycli/main.py | 2 +- tests/test_main.py | 18 +++++++++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index ae59ec7ac..924d8efa4 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,8 @@ Bug Fixes: * Fixed bug so that favorite queries can include unicode characters. (Thanks: [Thomas Roten]). * Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) +* Fix bug where mycli would not start due to the thanks/credit intro text. + (Thanks: [Thomas Roten]). Internal Changes: ----------------- diff --git a/mycli/main.py b/mycli/main.py index 04bd26462..ba3443955 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -981,7 +981,7 @@ def quit_command(sql): def thanks_picker(files=()): for filename in files: - with open(filename) as f: + with open(filename, encoding='utf-8') as f: contents = f.readlines() return choice([x.split('*')[1].strip() for x in contents if x.startswith('*')]) diff --git a/tests/test_main.py b/tests/test_main.py index 49e8be3ff..14f1c10e2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,10 +1,18 @@ +import os + import click from click.testing import CliRunner from mycli.main import (cli, confirm_destructive_query, format_output, - is_destructive, query_starts_with, queries_start_with) + is_destructive, query_starts_with, queries_start_with, + thanks_picker, PACKAGE_ROOT) from utils import USER, HOST, PORT, PASSWORD, dbtest, run +try: + text_type = basestring +except NameError: + text_type = str + CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, '--password', PASSWORD, '_test_db'] @@ -171,3 +179,11 @@ def test_confirm_destructive_query_notty(executor): sql = 'drop database foo;' assert confirm_destructive_query(sql) is None + +def test_thanks_picker_utf8(): + project_root = os.path.dirname(PACKAGE_ROOT) + author_file = os.path.join(project_root, 'AUTHORS') + sponsor_file = os.path.join(project_root, 'SPONSORS') + + name = thanks_picker((author_file, sponsor_file)) + assert isinstance(name, text_type) From 6ada4d468a6b33e2f6af8a3c0af4fc71ca4cc603 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:27:32 -0500 Subject: [PATCH 102/824] Add basic output formatter class functionality. --- mycli/main.py | 111 ++-- mycli/myclirc | 2 +- mycli/packages/expanded.py | 2 +- mycli/packages/tabulate.py | 1078 ------------------------------------ mycli/sqlcompleter.py | 11 +- setup.py | 1 + tests/test_main.py | 28 +- tests/test_tabulate.py | 22 - tests/utils.py | 17 +- 9 files changed, 81 insertions(+), 1191 deletions(-) delete mode 100644 mycli/packages/tabulate.py delete mode 100644 tests/test_tabulate.py diff --git a/mycli/main.py b/mycli/main.py index ba3443955..b9a3cebe4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -35,8 +35,6 @@ from pygments.token import Token from configobj import ConfigObj, ConfigObjError -from .packages.tabulate import tabulate, table_formats -from .packages.expanded import expanded_table from .packages.special.main import (COMMANDS, NO_QUERY) import mycli.packages.special as special from .sqlcompleter import SQLCompleter @@ -49,6 +47,7 @@ open_mylogin_cnf, read_config_file, read_config_files, str_to_bool) from .key_bindings import mycli_bindings +from .output_formatter import OutputFormatter from .encodingutils import utf8tounicode from .lexer import MyCliLexer from .__init__ import __version__ @@ -144,6 +143,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.completion_refresher = CompletionRefresher() + self.formatter = OutputFormatter() + self.logger = logging.getLogger(__name__) self.initialize_logging() @@ -191,9 +192,9 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - if not arg in table_formats(): + if not arg in self.formatter.supported_formats(): msg = "Table type %s not yet implemented. Allowed types:" % arg - for table_type in table_formats(): + for table_type in self.formatter.supported_formats(): msg += "\n\t%s" % table_type yield (None, None, None, msg) else: @@ -533,9 +534,8 @@ def one_iteration(document=None): else: max_width = None - formatted = format_output(title, cur, headers, - status, self.table_format, - special.is_expanded_output(), max_width) + formatted = self.format_output(title, cur, headers, + status, special.is_expanded_output(), max_width) output.extend(formatted) end = time() @@ -715,15 +715,55 @@ def get_prompt(self, string): string = string.replace('\\_', ' ') return string - def run_query(self, query, table_format=None, new_line=True): + def run_query(self, query, new_line=True): """Runs query""" results = self.sqlexecute.run(query) for result in results: title, cur, headers, status = result - output = format_output(title, cur, headers, None, table_format) + output = self.format_output(title, cur, headers, None) for line in output: click.echo(line, nl=new_line) + def format_output(self, title, cur, headers, status, expanded=False, + max_width=None): + output = [] + if title: # Only print the title if it's not None. + output.append(title) + if cur: + headers = [utf8tounicode(x) for x in headers] + + if expanded: + output.append(self.formatter.format_output(cur, headers, + 'expanded')) + elif self.table_format == 'csv': + content = StringIO() + writer = csv.writer(content) + writer.writerow(headers) + + for row in cur: + row = ['null' if val is None else str(val) for val in row] + writer.writerow(row) + + output.append(content.getvalue()) + content.close() + else: + rows = list(cur) + formatted = self.formatter.format_output(rows, headers, + self.table_format) + if (self.table_format != 'expanded' and + max_width and rows and + content_exceeds_width(rows[0], max_width) and + headers): + output.append(self.formatter.format_output(cur, headers, + 'expanded')) + else: + output.append(formatted) + if status: # Only print the status if it's not None. + output.append(status) + + return output + + @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') @click.option('-P', '--port', envvar='MYSQL_TCP_PORT', type=int, help='Port number to use for connection. Honors ' @@ -818,12 +858,11 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - table_format = None - if table: - table_format = mycli.table_format - elif csv: + table_format = 'tsv' + if csv: table_format = 'csv' - mycli.run_query(execute, table_format=table_format) + mycli.table_format = table_format or mycli.table_format + mycli.run_query(execute) exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') @@ -844,58 +883,22 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - table_format = None + table_format = 'tsv' new_line = True if csv: table_format = 'csv' new_line = False - elif table: - table_format = mycli.table_format - mycli.run_query(stdin_text, table_format=table_format, new_line=new_line) + mycli.table_format = table_format or mycli.table_format + + mycli.run_query(stdin_text, new_line=new_line) exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') exit(1) -def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): - output = [] - if title: # Only print the title if it's not None. - output.append(title) - if cur: - headers = [utf8tounicode(x) for x in headers] - table_format = 'tsv' if table_format is None else table_format - - if expanded: - output.append(expanded_table(cur, headers)) - elif table_format == 'csv': - content = StringIO() - writer = csv.writer(content) - writer.writerow(headers) - - for row in cur: - row = ['null' if val is None else str(val) for val in row] - writer.writerow(row) - - output.append(content.getvalue()) - content.close() - else: - rows = list(cur) - tabulated, frows = tabulate(rows, headers, tablefmt=table_format, - missingval='') - if (max_width and rows and - content_exceeds_width(frows[0], max_width) and - headers): - output.append(expanded_table(rows, headers)) - else: - output.append(tabulated) - if status: # Only print the status if it's not None. - output.append(status) - - return output - def content_exceeds_width(row, width): # Account for 3 characters between each column separator_space = (len(row)*3) diff --git a/mycli/myclirc b/mycli/myclirc index baf505106..ff6e8f734 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -66,7 +66,7 @@ less_chatty = False login_path_as_host = False # Cause result sets to be displayed vertically if they are too wide for the current window, -# and using normal tabular format otherwise. (This applies to statements terminated by ; or \G.) +# and using normal tabular format otherwise. (This applies to statements terminated by ; or \G.) auto_vertical_output = False # Custom colors for the completion menu, toolbar, etc. diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 128e9c690..245e4ca72 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,4 +1,4 @@ -from .tabulate import _text_type +from tabulate import _text_type import binascii def pad(field, total, char=u" "): diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py deleted file mode 100644 index fa9718267..000000000 --- a/mycli/packages/tabulate.py +++ /dev/null @@ -1,1078 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Pretty-print tabular data.""" - -from __future__ import print_function -from __future__ import unicode_literals -from collections import namedtuple -from decimal import Decimal -from platform import python_version_tuple -from wcwidth import wcswidth -import re -import binascii - - -if python_version_tuple()[0] < "3": - from itertools import izip_longest - from functools import partial - _none_type = type(None) - _int_type = int - _long_type = long - _float_type = float - _text_type = unicode - _binary_type = str - - def _is_file(f): - return isinstance(f, file) - -else: - from itertools import zip_longest as izip_longest - from functools import reduce, partial - _none_type = type(None) - _int_type = int - _long_type = int - _float_type = float - _text_type = str - _binary_type = bytes - - import io - def _is_file(f): - return isinstance(f, io.IOBase) - - -__all__ = ["tabulate", "tabulate_formats", "simple_separated_format"] -__version__ = "0.7.4" - - -MIN_PADDING = 2 - - -Line = namedtuple("Line", ["begin", "hline", "sep", "end"]) - - -DataRow = namedtuple("DataRow", ["begin", "sep", "end"]) - - -# A table structure is suppposed to be: -# -# --- lineabove --------- -# headerrow -# --- linebelowheader --- -# datarow -# --- linebewteenrows --- -# ... (more datarows) ... -# --- linebewteenrows --- -# last datarow -# --- linebelow --------- -# -# TableFormat's line* elements can be -# -# - either None, if the element is not used, -# - or a Line tuple, -# - or a function: [col_widths], [col_alignments] -> string. -# -# TableFormat's *row elements can be -# -# - either None, if the element is not used, -# - or a DataRow tuple, -# - or a function: [cell_values], [col_widths], [col_alignments] -> string. -# -# padding (an integer) is the amount of white space around data values. -# -# with_header_hide: -# -# - either None, to display all table elements unconditionally, -# - or a list of elements not to be displayed if the table has column headers. -# -TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", - "linebetweenrows", "linebelow", - "headerrow", "datarow", - "padding", "with_header_hide", "with_align"]) - - -def _pipe_segment_with_colons(align, colwidth): - """Return a segment of a horizontal line with optional colons which - indicate column's alignment (as in `pipe` output format).""" - w = colwidth - if align in ["right", "decimal"]: - return ('-' * (w - 1)) + ":" - elif align == "center": - return ":" + ('-' * (w - 2)) + ":" - elif align == "left": - return ":" + ('-' * (w - 1)) - else: - return '-' * w - - -def _pipe_line_with_colons(colwidths, colaligns): - """Return a horizontal line with optional colons to indicate column's - alignment (as in `pipe` output format).""" - segments = [_pipe_segment_with_colons(a, w) for a, w in zip(colaligns, colwidths)] - return "|" + "|".join(segments) + "|" - - -def _mediawiki_row_with_attrs(separator, cell_values, colwidths, colaligns): - alignment = { "left": '', - "right": 'align="right"| ', - "center": 'align="center"| ', - "decimal": 'align="right"| ' } - # hard-coded padding _around_ align attribute and value together - # rather than padding parameter which affects only the value - values_with_attrs = [' ' + alignment.get(a, '') + c + ' ' - for c, a in zip(cell_values, colaligns)] - colsep = separator*2 - return (separator + colsep.join(values_with_attrs)).rstrip() - - -def _html_row_with_attrs(celltag, cell_values, colwidths, colaligns): - alignment = { "left": '', - "right": ' style="text-align: right;"', - "center": ' style="text-align: center;"', - "decimal": ' style="text-align: right;"' } - values_with_attrs = ["<{0}{1}>{2}".format(celltag, alignment.get(a, ''), c) - for c, a in zip(cell_values, colaligns)] - return "" + "".join(values_with_attrs).rstrip() + "" - - -def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): - alignment = { "left": "l", "right": "r", "center": "c", "decimal": "r" } - tabular_columns_fmt = "".join([alignment.get(a, "l") for a in colaligns]) - return "\n".join(["\\begin{tabular}{" + tabular_columns_fmt + "}", - "\\toprule" if booktabs else "\hline"]) - -LATEX_ESCAPE_RULES = {r"&": r"\&", r"%": r"\%", r"$": r"\$", r"#": r"\#", - r"_": r"\_", r"^": r"\^{}", r"{": r"\{", r"}": r"\}", - r"~": r"\textasciitilde{}", "\\": r"\textbackslash{}", - r"<": r"\ensuremath{<}", r">": r"\ensuremath{>}"} - - -def _latex_row(cell_values, colwidths, colaligns): - def escape_char(c): - return LATEX_ESCAPE_RULES.get(c, c) - escaped_values = ["".join(map(escape_char, cell)) for cell in cell_values] - rowfmt = DataRow("", "&", "\\\\") - return _build_simple_row(escaped_values, rowfmt) - - -_table_formats = {"simple": - TableFormat(lineabove=Line("", "-", " ", ""), - linebelowheader=Line("", "-", " ", ""), - linebetweenrows=None, - linebelow=Line("", "-", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, - with_header_hide=["lineabove", "linebelow"], with_align=True), - "plain": - TableFormat(lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None, with_align=True), - "grid": - TableFormat(lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("+", "=", "+", "+"), - linebetweenrows=Line("+", "-", "+", "+"), - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None, with_align=True), - "fancy_grid": - TableFormat(lineabove=Line("╒", "═", "╤", "╕"), - linebelowheader=Line("╞", "═", "╪", "╡"), - linebetweenrows=Line("├", "─", "┼", "┤"), - linebelow=Line("╘", "═", "╧", "╛"), - headerrow=DataRow("│", "│", "│"), - datarow=DataRow("│", "│", "│"), - padding=1, with_header_hide=None, with_align=True), - "pipe": - TableFormat(lineabove=_pipe_line_with_colons, - linebelowheader=_pipe_line_with_colons, - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, - with_header_hide=["lineabove"], with_align=True), - "orgtbl": - TableFormat(lineabove=None, - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None, with_align=True), - "psql": - TableFormat(lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None, with_align=True), - "rst": - TableFormat(lineabove=Line("", "=", " ", ""), - linebelowheader=Line("", "=", " ", ""), - linebetweenrows=None, - linebelow=Line("", "=", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None, with_align=True), - "mediawiki": - TableFormat(lineabove=Line("{| class=\"wikitable\" style=\"text-align: left;\"", - "", "", "\n|+ \n|-"), - linebelowheader=Line("|-", "", "", ""), - linebetweenrows=Line("|-", "", "", ""), - linebelow=Line("|}", "", "", ""), - headerrow=partial(_mediawiki_row_with_attrs, "!"), - datarow=partial(_mediawiki_row_with_attrs, "|"), - padding=0, with_header_hide=None, with_align=True), - "html": - TableFormat(lineabove=Line("", "", "", ""), - linebelowheader=None, - linebetweenrows=None, - linebelow=Line("
", "", "", ""), - headerrow=partial(_html_row_with_attrs, "th"), - datarow=partial(_html_row_with_attrs, "td"), - padding=0, with_header_hide=None, with_align=False), - "latex": - TableFormat(lineabove=_latex_line_begin_tabular, - linebelowheader=Line("\\hline", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None, with_align=False), - "latex_booktabs": - TableFormat(lineabove=partial(_latex_line_begin_tabular, booktabs=True), - linebelowheader=Line("\\midrule", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None, with_align=False), - "tsv": - TableFormat(lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("", "\t", ""), - datarow=DataRow("", "\t", ""), - padding=0, with_header_hide=None, with_align=False)} - - -tabulate_formats = list(sorted(_table_formats.keys())) - - -_invisible_codes = re.compile(r"\x1b\[\d*m|\x1b\[\d*\;\d*\;\d*m") # ANSI color codes -_invisible_codes_bytes = re.compile(b"\x1b\[\d*m|\x1b\[\d*\;\d*\;\d*m") # ANSI color codes - - -def simple_separated_format(separator): - """Construct a simple TableFormat with columns separated by a separator. - - >>> tsv = simple_separated_format("\\t") ; \ - print(tabulate([["foo", 1], ["spam", 23]], tablefmt=tsv)[0].replace('\\t', r'\\t')) - foo\\t1 - spam\\t23 - """ - return TableFormat(None, None, None, None, - headerrow=DataRow('', separator, ''), - datarow=DataRow('', separator, ''), - padding=0, with_header_hide=None, - with_align=False) - - -def _isconvertible(conv, string): - try: - n = conv(string) - return True - except (ValueError, TypeError): - return False - - -def _isnumber(string): - """ - >>> _isnumber("123.45") - True - >>> _isnumber("123") - True - >>> _isnumber("spam") - False - """ - return _isconvertible(float, string) - - -def _isint(string): - """ - >>> _isint("123") - True - >>> _isint("123.45") - False - """ - return type(string) is _int_type or type(string) is _long_type or \ - (isinstance(string, _binary_type) or isinstance(string, _text_type)) and \ - _isconvertible(int, string) - - -def _type(string, has_invisible=True): - """The least generic type (type(None), int, float, str, unicode). - - >>> _type(None) is type(None) - True - >>> _type("foo") is type("") - True - >>> _type("1") is type(1) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - - """ - - if has_invisible and \ - (isinstance(string, _text_type) or isinstance(string, _binary_type)): - string = _strip_invisible(string) - - if string is None: - return _none_type - if isinstance(string, (bool, Decimal,)): - return _text_type - elif hasattr(string, "isoformat"): # datetime.datetime, date, and time - return _text_type - elif _isint(string): - return int - elif _isnumber(string): - return float - elif isinstance(string, _binary_type): - return _binary_type - else: - return _text_type - - -def _afterpoint(string): - """Symbols after a decimal point, -1 if the string lacks the decimal point. - - >>> _afterpoint("123.45") - 2 - >>> _afterpoint("1001") - -1 - >>> _afterpoint("eggs") - -1 - >>> _afterpoint("123e45") - 2 - - """ - if _isnumber(string): - if _isint(string): - return -1 - else: - pos = string.rfind(".") - pos = string.lower().rfind("e") if pos < 0 else pos - if pos >= 0: - return len(string) - pos - 1 - else: - return -1 # no point - else: - return -1 # not a number - - -def _padleft(width, s, has_invisible=True): - """Flush right. - - >>> _padleft(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430' - True - - """ - lwidth = width - wcswidth(_strip_invisible(s) if has_invisible else s) - return ' ' * lwidth + s - - -def _padright(width, s, has_invisible=True): - """Flush left. - - >>> _padright(6, '\u044f\u0439\u0446\u0430') == '\u044f\u0439\u0446\u0430 ' - True - - """ - rwidth = width - wcswidth(_strip_invisible(s) if has_invisible else s) - return s + ' ' * rwidth - - -def _padboth(width, s, has_invisible=True): - """Center string. - - >>> _padboth(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430 ' - True - - """ - xwidth = width - wcswidth(_strip_invisible(s) if has_invisible else s) - lwidth = xwidth // 2 - rwidth = 0 if xwidth <= 0 else lwidth + xwidth % 2 - return ' ' * lwidth + s + ' ' * rwidth - - -def _strip_invisible(s): - "Remove invisible ANSI color codes." - if isinstance(s, _text_type): - return re.sub(_invisible_codes, "", s) - else: # a bytestring - return re.sub(_invisible_codes_bytes, "", s) - - -def _visible_width(s): - """Visible width of a printed string. ANSI color codes are removed. - - >>> _visible_width('\x1b[31mhello\x1b[0m'), _visible_width("world") - (5, 5) - - """ - if isinstance(s, _text_type) or isinstance(s, _binary_type): - return wcswidth(_strip_invisible(s)) - else: - return wcswidth(_text_type(s)) - - -def _align_column(strings, alignment, minwidth=0, has_invisible=True): - """[string] -> [padded_string] - - >>> list(map(str,_align_column(["12.345", "-1234.5", "1.23", "1234.5", "1e+234", "1.0e234"], "decimal"))) - [' 12.345 ', '-1234.5 ', ' 1.23 ', ' 1234.5 ', ' 1e+234 ', ' 1.0e234'] - - >>> list(map(str,_align_column(['123.4', '56.7890'], None))) - ['123.4', '56.7890'] - - """ - if alignment == "right": - padfn = _padleft - elif alignment == "center": - padfn = _padboth - elif alignment == "decimal": - decimals = [_afterpoint(s) for s in strings] - maxdecimals = max(decimals) - strings = [s + (maxdecimals - decs) * " " - for s, decs in zip(strings, decimals)] - padfn = _padleft - elif not alignment: - return strings - else: - padfn = _padright - - if has_invisible: - width_fn = _visible_width - else: - width_fn = wcswidth - - maxwidth = max(max(map(width_fn, strings)), minwidth) - padded_strings = [padfn(maxwidth, s, has_invisible) for s in strings] - return padded_strings - - -def _more_generic(type1, type2): - types = { _none_type: 0, int: 1, float: 2, _binary_type: 3, _text_type: 4 } - invtypes = { 4: _text_type, 3: _binary_type, 2: float, 1: int, 0: _none_type } - moregeneric = max(types.get(type1, 4), types.get(type2, 4)) - return invtypes[moregeneric] - - -def _column_type(strings, has_invisible=True): - """The least generic type all column values are convertible to. - - >>> _column_type(["1", "2"]) is _int_type - True - >>> _column_type(["1", "2.3"]) is _float_type - True - >>> _column_type(["1", "2.3", "four"]) is _text_type - True - >>> _column_type(["four", '\u043f\u044f\u0442\u044c']) is _text_type - True - >>> _column_type([None, "brux"]) is _text_type - True - >>> _column_type([1, 2, None]) is _int_type - True - >>> import datetime as dt - >>> _column_type([dt.datetime(1991,2,19), dt.time(17,35)]) is _text_type - True - - """ - types = [_type(s, has_invisible) for s in strings ] - return reduce(_more_generic, types, int) - - -def _format(val, valtype, floatfmt, missingval=""): - u"""Format a value accoding to its type. - - Unicode is supported: - - >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', '\u0446\u0438\u0444\u0440\u0430'] ; \ - tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] ; \ - print(tabulate(tbl, headers=hrow)[0]) - буква цифра - ------- ------- - аз 2 - буки 4 - """ - if val is None: - return missingval - - if valtype in [int, _text_type]: - return "{0}".format(val) - elif valtype is _binary_type: - try: - return _text_type(val, "ascii") - except UnicodeDecodeError: - return _text_type('0x' + binascii.hexlify(val).decode('ascii')) - except TypeError: - return _text_type(val) - elif valtype is float: - return format(float(val), floatfmt) - else: - return "{0}".format(val) - - -def _align_header(header, alignment, width): - if alignment == "left": - return _padright(width, header) - elif alignment == "center": - return _padboth(width, header) - elif not alignment: - return "{0}".format(header) - else: - return _padleft(width, header) - - -def _normalize_tabular_data(tabular_data, headers): - """Transform a supported data type to a list of lists, and a list of headers. - - Supported tabular data types: - - * list-of-lists or another iterable of iterables - - * list of named tuples (usually used with headers="keys") - - * list of dicts (usually used with headers="keys") - - * list of OrderedDicts (usually used with headers="keys") - - * 2D NumPy arrays - - * NumPy record arrays (usually used with headers="keys") - - * dict of iterables (usually used with headers="keys") - - * pandas.DataFrame (usually used with headers="keys") - - The first row can be used as headers if headers="firstrow", - column indices can be used as headers if headers="keys". - - """ - - if hasattr(tabular_data, "keys") and hasattr(tabular_data, "values"): - # dict-like and pandas.DataFrame? - if hasattr(tabular_data.values, "__call__"): - # likely a conventional dict - keys = tabular_data.keys() - rows = list(izip_longest(*tabular_data.values())) # columns have to be transposed - elif hasattr(tabular_data, "index"): - # values is a property, has .index => it's likely a pandas.DataFrame (pandas 0.11.0) - keys = tabular_data.keys() - vals = tabular_data.values # values matrix doesn't need to be transposed - names = tabular_data.index - rows = [[v]+list(row) for v,row in zip(names, vals)] - else: - raise ValueError("tabular data doesn't appear to be a dict or a DataFrame") - - if headers == "keys": - headers = list(map(_text_type,keys)) # headers should be strings - - else: # it's a usual an iterable of iterables, or a NumPy array - rows = list(tabular_data) - - if (headers == "keys" and - hasattr(tabular_data, "dtype") and - getattr(tabular_data.dtype, "names")): - # numpy record array - headers = tabular_data.dtype.names - elif (headers == "keys" - and len(rows) > 0 - and isinstance(rows[0], tuple) - and hasattr(rows[0], "_fields")): - # namedtuple - headers = list(map(_text_type, rows[0]._fields)) - elif (len(rows) > 0 - and isinstance(rows[0], dict)): - # dict or OrderedDict - uniq_keys = set() # implements hashed lookup - keys = [] # storage for set - if headers == "firstrow": - firstdict = rows[0] if len(rows) > 0 else {} - keys.extend(firstdict.keys()) - uniq_keys.update(keys) - rows = rows[1:] - for row in rows: - for k in row.keys(): - #Save unique items in input order - if k not in uniq_keys: - keys.append(k) - uniq_keys.add(k) - if headers == 'keys': - headers = keys - elif isinstance(headers, dict): - # a dict of headers for a list of dicts - headers = [headers.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - elif headers == "firstrow": - if len(rows) > 0: - headers = [firstdict.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - else: - headers = [] - elif headers: - raise ValueError('headers for a list of dicts is not a dict or a keyword') - rows = [[row.get(k) for k in keys] for row in rows] - elif headers == "keys" and len(rows) > 0: - # keys are column indices - headers = list(map(_text_type, range(len(rows[0])))) - - # take headers from the first row if necessary - if headers == "firstrow" and len(rows) > 0: - headers = list(map(_text_type, rows[0])) # headers should be strings - rows = rows[1:] - - headers = list(map(_text_type,headers)) - rows = list(map(list,rows)) - - # pad with empty headers for initial columns if necessary - if headers and len(rows) > 0: - nhs = len(headers) - ncols = len(rows[0]) - if nhs < ncols: - headers = [""]*(ncols - nhs) + headers - - return rows, headers - -def table_formats(): - return _table_formats.keys() - -def tabulate(tabular_data, headers=[], tablefmt="simple", - floatfmt="g", numalign="decimal", stralign="left", - missingval=""): - """Format a fixed width table for pretty printing. - - >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])[0]) - --- --------- - 1 2.34 - -56 8.999 - 2 10001 - --- --------- - - The first required argument (`tabular_data`) can be a - list-of-lists (or another iterable of iterables), a list of named - tuples, a dictionary of iterables, an iterable of dictionaries, - a two-dimensional NumPy array, NumPy record array, or a Pandas' - dataframe. - - - Table headers - ------------- - - To print nice column headers, supply the second argument (`headers`): - - - `headers` can be an explicit list of column headers - - if `headers="firstrow"`, then the first row of data is used - - if `headers="keys"`, then dictionary keys or column indices are used - - Otherwise a headerless table is produced. - - If the number of headers is less than the number of columns, they - are supposed to be names of the last columns. This is consistent - with the plain-text format of R and Pandas' dataframes. - - >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], - ... headers="firstrow")[0]) - sex age - ----- ----- ----- - Alice F 24 - Bob M 19 - - - Column alignment - ---------------- - - `tabulate` tries to detect column types automatically, and aligns - the values properly. By default it aligns decimal points of the - numbers (or flushes integer numbers to the right), and flushes - everything else to the left. Possible column alignments - (`numalign`, `stralign`) are: "right", "center", "left", "decimal" - (only for `numalign`), and None (to disable alignment). - - - Table formats - ------------- - - `floatfmt` is a format specification used for columns which - contain numeric data with a decimal point. - - `None` values are replaced with a `missingval` string: - - >>> print(tabulate([["spam", 1, None], - ... ["eggs", 42, 3.14], - ... ["other", None, 2.7]], missingval="?")[0]) - ----- -- ---- - spam 1 ? - eggs 42 3.14 - other ? 2.7 - ----- -- ---- - - Various plain-text table formats (`tablefmt`) are supported: - 'plain', 'simple', 'grid', 'pipe', 'orgtbl', 'rst', 'mediawiki', - 'latex', and 'latex_booktabs'. Variable `tabulate_formats` contains the list of - currently supported formats. - - "plain" format doesn't use any pseudographics to draw tables, - it separates columns with a double space: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "plain")[0]) - strings numbers - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="plain")[0]) - spam 41.9999 - eggs 451 - - "simple" format is like Pandoc simple_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "simple")[0]) - strings numbers - --------- --------- - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="simple")[0]) - ---- -------- - spam 41.9999 - eggs 451 - ---- -------- - - "grid" is similar to tables produced by Emacs table.el package or - Pandoc grid_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "grid")[0]) - +-----------+-----------+ - | strings | numbers | - +===========+===========+ - | spam | 41.9999 | - +-----------+-----------+ - | eggs | 451 | - +-----------+-----------+ - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="grid")[0]) - +------+----------+ - | spam | 41.9999 | - +------+----------+ - | eggs | 451 | - +------+----------+ - - "fancy_grid" draws a grid using box-drawing characters: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "fancy_grid")[0]) - ╒═══════════╤═══════════╕ - │ strings │ numbers │ - ╞═══════════╪═══════════╡ - │ spam │ 41.9999 │ - ├───────────┼───────────┤ - │ eggs │ 451 │ - ╘═══════════╧═══════════╛ - - "pipe" is like tables in PHP Markdown Extra extension or Pandoc - pipe_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "pipe")[0]) - | strings | numbers | - |:----------|----------:| - | spam | 41.9999 | - | eggs | 451 | - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="pipe")[0]) - |:-----|---------:| - | spam | 41.9999 | - | eggs | 451 | - - "orgtbl" is like tables in Emacs org-mode and orgtbl-mode. They - are slightly different from "pipe" format by not using colons to - define column alignment, and using a "+" sign to indicate line - intersections: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "orgtbl")[0]) - | strings | numbers | - |-----------+-----------| - | spam | 41.9999 | - | eggs | 451 | - - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="orgtbl")[0]) - | spam | 41.9999 | - | eggs | 451 | - - "rst" is like a simple table format from reStructuredText; please - note that reStructuredText accepts also "grid" tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "rst")[0]) - ========= ========= - strings numbers - ========= ========= - spam 41.9999 - eggs 451 - ========= ========= - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")[0]) - ==== ======== - spam 41.9999 - eggs 451 - ==== ======== - - "mediawiki" produces a table markup used in Wikipedia and on other - MediaWiki-based sites: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="mediawiki")[0]) - {| class="wikitable" style="text-align: left;" - |+ - |- - ! strings !! align="right"| numbers - |- - | spam || align="right"| 41.9999 - |- - | eggs || align="right"| 451 - |} - - "html" produces HTML markup: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="html")[0]) - - - - -
stringsnumbers
spam41.9999
eggs451
- - "latex" produces a tabular environment of LaTeX document markup: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex")[0]) - \\begin{tabular}{ll} - \\hline - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\hline - \\end{tabular} - - "latex_booktabs" produces a tabular environment of LaTeX document markup - using the booktabs.sty package: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex_booktabs")[0]) - \\begin{tabular}{ll} - \\toprule - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\bottomrule - \end{tabular} - - Also returns a tuple of the raw rows pulled from tabular_data - """ - if tabular_data is None: - tabular_data = [] - list_of_lists, headers = _normalize_tabular_data(tabular_data, headers) - - # format rows and columns, convert numeric values to strings - cols = list(zip(*list_of_lists)) - coltypes = list(map(_column_type, cols)) - cols = [[_format(v, ct, floatfmt, missingval) for v in c] - for c,ct in zip(cols, coltypes)] - - # optimization: look for ANSI control codes once, - # enable smart width functions only if a control code is found - plain_text = '\n'.join(['\t'.join(map(_text_type, headers))] + \ - ['\t'.join(map(_text_type, row)) for row in cols]) - has_invisible = re.search(_invisible_codes, plain_text) - if has_invisible: - width_fn = _visible_width - else: - width_fn = wcswidth - - if not isinstance(tablefmt, TableFormat): - tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) - - if tablefmt.with_align: - # align columns - aligns = [numalign if ct in [int,float] else stralign for ct in coltypes] - else: - aligns = [False for ct in coltypes] - minwidths = [width_fn(h) + MIN_PADDING for h in headers] if headers else [0]*len(cols) - cols = [_align_column(c, a, minw, has_invisible) - for c, a, minw in zip(cols, aligns, minwidths)] - - if headers: - # align headers and add headers - t_cols = cols or [['']] * len(headers) - if tablefmt.with_align: - t_aligns = aligns or [stralign] * len(headers) - else: - t_aligns = [False for ct in coltypes] - minwidths = [max(minw, width_fn(c[0])) for minw, c in zip(minwidths, t_cols)] - headers = [_align_header(h, a, minw) - for h, a, minw in zip(headers, t_aligns, minwidths)] - rows = list(zip(*cols)) - else: - minwidths = [width_fn(c[0]) for c in cols] - rows = list(zip(*cols)) - - return _format_table(tablefmt, headers, rows, minwidths, aligns), rows - - -def _build_simple_row(padded_cells, rowfmt): - "Format row according to DataRow format without padding." - begin, sep, end = rowfmt - return (begin + sep.join(padded_cells) + end).rstrip() - - -def _build_row(padded_cells, colwidths, colaligns, rowfmt): - "Return a string which represents a row of data cells." - if not rowfmt: - return None - if hasattr(rowfmt, "__call__"): - return rowfmt(padded_cells, colwidths, colaligns) - else: - return _build_simple_row(padded_cells, rowfmt) - - -def _build_line(colwidths, colaligns, linefmt): - "Return a string which represents a horizontal line." - if not linefmt: - return None - if hasattr(linefmt, "__call__"): - return linefmt(colwidths, colaligns) - else: - begin, fill, sep, end = linefmt - cells = [fill*w for w in colwidths] - return _build_simple_row(cells, (begin, sep, end)) - - -def _pad_row(cells, padding): - if cells: - pad = " "*padding - padded_cells = [pad + cell + pad for cell in cells] - return padded_cells - else: - return cells - - -def _format_table(fmt, headers, rows, colwidths, colaligns): - """Produce a plain-text representation of the table.""" - lines = [] - hidden = fmt.with_header_hide if (headers and fmt.with_header_hide) else [] - pad = fmt.padding - headerrow = fmt.headerrow - - padded_widths = [(w + 2*pad) for w in colwidths] - padded_headers = _pad_row(headers, pad) - padded_rows = [_pad_row(row, pad) for row in rows] - - if fmt.lineabove and "lineabove" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.lineabove)) - - if padded_headers: - lines.append(_build_row(padded_headers, padded_widths, colaligns, headerrow)) - if fmt.linebelowheader and "linebelowheader" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.linebelowheader)) - - if padded_rows and fmt.linebetweenrows and "linebetweenrows" not in hidden: - # initial rows with a line below - for row in padded_rows[:-1]: - lines.append(_build_row(row, padded_widths, colaligns, fmt.datarow)) - lines.append(_build_line(padded_widths, colaligns, fmt.linebetweenrows)) - # the last row without a line below - lines.append(_build_row(padded_rows[-1], padded_widths, colaligns, fmt.datarow)) - else: - for row in padded_rows: - lines.append(_build_row(row, padded_widths, colaligns, fmt.datarow)) - - if fmt.linebelow and "linebelow" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.linebelow)) - - return "\n".join(lines) - - -def _main(): - """\ - Usage: tabulate [options] [FILE ...] - - Pretty-print tabular data. See also https://bitbucket.org/astanin/python-tabulate - - FILE a filename of the file with tabular data; - if "-" or missing, read data from stdin. - - Options: - - -h, --help show this message - -1, --header use the first row of data as a table header - -s REGEXP, --sep REGEXP use a custom column separator (default: whitespace) - -f FMT, --format FMT set output table format; supported formats: - plain, simple, grid, fancy_grid, pipe, orgtbl, - rst, mediawiki, html, latex, latex_booktabs, tsv - (default: simple) - """ - import getopt - import sys - import textwrap - usage = textwrap.dedent(_main.__doc__) - try: - opts, args = getopt.getopt(sys.argv[1:], - "h1f:s:", - ["help", "header", "format", "separator"]) - except getopt.GetoptError as e: - print(e) - print(usage) - sys.exit(2) - headers = [] - tablefmt = "simple" - sep = r"\s+" - for opt, value in opts: - if opt in ["-1", "--header"]: - headers = "firstrow" - elif opt in ["-f", "--format"]: - if value not in tabulate_formats: - print("%s is not a supported table format" % value) - print(usage) - sys.exit(3) - tablefmt = value - elif opt in ["-s", "--sep"]: - sep = value - elif opt in ["-h", "--help"]: - print(usage) - sys.exit(0) - files = [sys.stdin] if not args else args - for f in files: - if f == "-": - f = sys.stdin - if _is_file(f): - _pprint_file(f, headers=headers, tablefmt=tablefmt, sep=sep) - else: - with open(f) as fobj: - _pprint_file(fobj) - - -def _pprint_file(fobject, headers, tablefmt, sep): - rows = fobject.readlines() - table = [re.split(sep, r.rstrip()) for r in rows] - print(tabulate(table, headers, tablefmt)) - - -if __name__ == "__main__": - _main() diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 41364756c..656e5d531 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -1,13 +1,15 @@ from __future__ import print_function from __future__ import unicode_literals import logging +from re import compile, escape +from collections import Counter + from prompt_toolkit.completion import Completer, Completion + +from .output_formatter import OutputFormatter from .packages.completion_engine import suggest_type from .packages.parseutils import last_word from .packages.special.favoritequeries import favoritequeries -from re import compile, escape -from .packages.tabulate import table_formats -from collections import Counter _logger = logging.getLogger(__name__) @@ -57,7 +59,8 @@ def __init__(self, smart_completion=True): self.name_pattern = compile("^[_a-z][_a-z0-9\$]*$") self.special_commands = [] - self.table_formats = table_formats() + formatter = OutputFormatter() + self.table_formats = formatter.supported_formats() self.reset_completions() def escape_name(self, name): diff --git a/setup.py b/setup.py index 82cdef6e1..3ecc21c48 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', 'pycryptodome >= 3', + 'tabulate >= 0.7.6', ] setup( diff --git a/tests/test_main.py b/tests/test_main.py index 14f1c10e2..cb769a0be 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,7 +3,7 @@ import click from click.testing import CliRunner -from mycli.main import (cli, confirm_destructive_query, format_output, +from mycli.main import (cli, confirm_destructive_query, is_destructive, query_starts_with, queries_start_with, thanks_picker, PACKAGE_ROOT) from utils import USER, HOST, PORT, PASSWORD, dbtest, run @@ -16,32 +16,6 @@ CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, '--password', PASSWORD, '_test_db'] -def test_format_output(): - results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], - 'test status', 'psql') - expected = ['Title', '+---------+---------+\n| head1 | head2 |\n|---------+---------|\n| abc | def |\n+---------+---------+', 'test status'] - assert results == expected - -def test_format_output_auto_expand(): - table_results = format_output('Title', [('abc', 'def')], - ['head1', 'head2'], 'test status', 'psql', - max_width=100) - table = ['Title', '+---------+---------+\n| head1 | head2 |\n|---------+---------|\n| abc | def |\n+---------+---------+', 'test status'] - assert table_results == table - - expanded_results = format_output('Title', [('abc', 'def')], - ['head1', 'head2'], 'test status', 'psql', - max_width=1) - expanded = ['Title', u'***************************[ 1. row ]***************************\nhead1 | abc\nhead2 | def\n', 'test status'] - assert expanded_results == expanded - -def test_format_output_no_table(): - results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], - 'test status', None) - - expected = ['Title', u'head1\thead2\nabc\tdef', 'test status'] - assert results == expected - @dbtest def test_execute_arg(executor): run(executor, 'create table test (a text)') diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py deleted file mode 100644 index e0ddc407d..000000000 --- a/tests/test_tabulate.py +++ /dev/null @@ -1,22 +0,0 @@ -from mycli.packages.tabulate import tabulate -from textwrap import dedent - - -def test_dont_strip_leading_whitespace(): - data = [[' abc']] - headers = ['xyz'] - tbl, _ = tabulate(data, headers, tablefmt='psql') - assert tbl == dedent(''' - +---------+ - | xyz | - |---------| - | abc | - +---------+ ''').strip() -def test_dont_add_whitespace(): - data = [[3, 4]] - headers = ['1', '2'] - tbl, _ = tabulate(data, headers, tablefmt='tsv') - assert tbl == dedent(''' - 1\t2 - 3\t4 - ''').strip() diff --git a/tests/utils.py b/tests/utils.py index c76cb7ad3..5e9e0abb0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,11 @@ -import pytest -import pymysql -from mycli.main import format_output, special from os import getenv +import pymysql +import pytest + +from mycli.main import MyCli, special +from mycli.output_formatter import OutputFormatter + PASSWORD = getenv('PYTEST_PASSWORD') USER = getenv('PYTEST_USER', 'root') HOST = getenv('PYTEST_HOST', 'localhost') @@ -37,8 +40,14 @@ def create_db(dbname): def run(executor, sql, join=False): " Return string output for the sql to be run " result = [] + + # TODO: this needs to go away. `run()` should not test formatted output. + # It should test raw results. + mycli = MyCli() + mycli.table_format = 'psql' for title, rows, headers, status in executor.run(sql): - result.extend(format_output(title, rows, headers, status, 'psql', special.is_expanded_output())) + result.extend(mycli.format_output(title, rows, headers, status, special.is_expanded_output())) + if join: result = '\n'.join(result) return result From 2bc4facb4ae978c7465dcb4128b895224348acb1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:30:40 -0500 Subject: [PATCH 103/824] Add missing class. --- mycli/output_formatter.py | 57 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 mycli/output_formatter.py diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py new file mode 100644 index 000000000..c939aa512 --- /dev/null +++ b/mycli/output_formatter.py @@ -0,0 +1,57 @@ +"""A generic output formatter interface.""" + +from __future__ import unicode_literals + +from tabulate import tabulate + +from .packages.expanded import expanded_table + + +def tabulate_wrapper(data, headers, table_format=None, missing_value=None): + """Wrap tabulate inside a standard function for OutputFormatter.""" + return tabulate(data, headers, tablefmt=table_format, + missingval=missing_value) + + +class OutputFormatter(object): + """A class with a standard interface for various formatting libraries.""" + + def __init__(self): + """Register the supported output formats.""" + self._output_formats = {} + + tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', + 'orgtbl', 'jira', 'psql', 'rst', 'tsv', + 'mediawiki', 'moinmoin', 'html', 'latex', + 'latex_booktabs', 'textile') + for tabulate_format in tabulate_formats: + self.register_output_format(tabulate_format, tabulate_wrapper, + table_format=tabulate_format) + + self.register_output_format('expanded', expanded_table) + + def register_output_format(self, name, function, **kwargs): + """Register a new output format. + + *function* should be a callable that accepts the following arguments: + - *headers*: A list of headers for the output data. + - *data*: The data that needs formatting. + - *kwargs*: Any other keyword arguments for controlling the output. + It should return the formatted output as a string. + """ + self._output_formats[name] = (function, kwargs) + + def supported_formats(self): + """Return the supported output format names.""" + return tuple(self._output_formats.keys()) + + def format_output(self, data, headers, format_name, **kwargs): + """Format the headers and data using a specific formatter. + + *format_name* must be a formatter available in `supported_formats()`. + + All keyword arguments are passed to the specified formatter. + """ + function, fkwargs = self._output_formats[format_name] + fkwargs.update(kwargs) + return function(data, headers, **fkwargs) From 3c783213fa0c2c63668a709644ddded28f2dfad7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:34:10 -0500 Subject: [PATCH 104/824] Make logic more readable. --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index b9a3cebe4..46de4cbff 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -192,7 +192,7 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - if not arg in self.formatter.supported_formats(): + if arg not in self.formatter.supported_formats(): msg = "Table type %s not yet implemented. Allowed types:" % arg for table_type in self.formatter.supported_formats(): msg += "\n\t%s" % table_type From 26ff1f3f23fac653de47e51d4494c494a1fe4cc3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:37:31 -0500 Subject: [PATCH 105/824] Move CSV to output formatter. --- mycli/main.py | 17 ----------------- mycli/output_formatter.py | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 46de4cbff..389f176c6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -15,12 +15,6 @@ from random import choice from io import open -# support StringIO for Python 2 and 3 -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - import click import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction @@ -735,17 +729,6 @@ def format_output(self, title, cur, headers, status, expanded=False, if expanded: output.append(self.formatter.format_output(cur, headers, 'expanded')) - elif self.table_format == 'csv': - content = StringIO() - writer = csv.writer(content) - writer.writerow(headers) - - for row in cur: - row = ['null' if val is None else str(val) for val in row] - writer.writerow(row) - - output.append(content.getvalue()) - content.close() else: rows = list(cur) formatted = self.formatter.format_output(rows, headers, diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index c939aa512..5081a5d61 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -2,6 +2,12 @@ from __future__ import unicode_literals +import csv +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + from tabulate import tabulate from .packages.expanded import expanded_table @@ -13,6 +19,21 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value=None): missingval=missing_value) +def csv_wrapper(data, headers): + content = StringIO() + writer = csv.writer(content) + writer.writerow(headers) + + for row in data: + row = ['null' if val is None else str(val) for val in row] + writer.writerow(row) + + output = content.getvalue() + content.close() + + return output + + class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" @@ -28,6 +49,7 @@ def __init__(self): self.register_output_format(tabulate_format, tabulate_wrapper, table_format=tabulate_format) + self.register_output_format('csv', csv_wrapper) self.register_output_format('expanded', expanded_table) def register_output_format(self, name, function, **kwargs): From 5da417568292477e3603c8124ed733e7be0dd840 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:51:47 -0500 Subject: [PATCH 106/824] Simply format output. --- mycli/main.py | 27 ++++++++++++--------------- mycli/output_formatter.py | 2 +- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 389f176c6..c3485dfc4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -720,27 +720,24 @@ def run_query(self, query, new_line=True): def format_output(self, title, cur, headers, status, expanded=False, max_width=None): + table_format = 'expanded' if expanded else self.table_format output = [] + if title: # Only print the title if it's not None. output.append(title) + if cur: headers = [utf8tounicode(x) for x in headers] - if expanded: - output.append(self.formatter.format_output(cur, headers, - 'expanded')) - else: - rows = list(cur) - formatted = self.formatter.format_output(rows, headers, - self.table_format) - if (self.table_format != 'expanded' and - max_width and rows and - content_exceeds_width(rows[0], max_width) and - headers): - output.append(self.formatter.format_output(cur, headers, - 'expanded')) - else: - output.append(formatted) + rows = list(cur) + formatted = self.formatter.format_output(rows, headers, table_format) + + if (table_format != 'expanded' and max_width and rows and + content_exceeds_width(rows[0], max_width) and headers): + formatted = self.formatter.format_output(rows, headers, 'expanded') + + output.append(formatted) + if status: # Only print the status if it's not None. output.append(status) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5081a5d61..5c2678a0d 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -13,7 +13,7 @@ from .packages.expanded import expanded_table -def tabulate_wrapper(data, headers, table_format=None, missing_value=None): +def tabulate_wrapper(data, headers, table_format=None, missing_value=''): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, missingval=missing_value) From 8918a51db94b46ae9846946685860c51f2ad0061 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:55:39 -0500 Subject: [PATCH 107/824] Fix tsv/table logic. --- mycli/main.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index c3485dfc4..cb40aaf09 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -838,10 +838,12 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - table_format = 'tsv' - if csv: + table_format = None + if table: + table_format = mycli.table_format + elif csv: table_format = 'csv' - mycli.table_format = table_format or mycli.table_format + mycli.table_format = table_format or 'tsv' mycli.run_query(execute) exit(0) except Exception as e: @@ -863,14 +865,16 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - table_format = 'tsv' + table_format = None new_line = True if csv: table_format = 'csv' new_line = False + elif table: + table_format = mycli.table_format - mycli.table_format = table_format or mycli.table_format + mycli.table_format = table_format or 'tsv' mycli.run_query(stdin_text, new_line=new_line) exit(0) From 7b6398a14df84ebaa0801fe000b620a2c19a5949 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:59:01 -0500 Subject: [PATCH 108/824] Use missing_value keyword for csv wrapper. --- mycli/output_formatter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5c2678a0d..5a90709ac 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -19,13 +19,14 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value=''): missingval=missing_value) -def csv_wrapper(data, headers): +def csv_wrapper(data, headers, missing_value='null'): + """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() writer = csv.writer(content) writer.writerow(headers) for row in data: - row = ['null' if val is None else str(val) for val in row] + row = [missing_value if val is None else str(val) for val in row] writer.writerow(row) output = content.getvalue() From f5a305d45652d017e411c1f47cf9e87fe412b4f7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:05:22 -0500 Subject: [PATCH 109/824] Do not use tabulate for TSV formatting. --- mycli/output_formatter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5a90709ac..6b0d8f2a4 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -19,10 +19,10 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value=''): missingval=missing_value) -def csv_wrapper(data, headers, missing_value='null'): +def csv_wrapper(data, headers, missing_value='null', delimiter=','): """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() - writer = csv.writer(content) + writer = csv.writer(content, delimiter=delimiter) writer.writerow(headers) for row in data: @@ -43,14 +43,15 @@ def __init__(self): self._output_formats = {} tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'jira', 'psql', 'rst', 'tsv', - 'mediawiki', 'moinmoin', 'html', 'latex', - 'latex_booktabs', 'textile') + 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', + 'moinmoin', 'html', 'latex', 'latex_booktabs', + 'textile') for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, table_format=tabulate_format) self.register_output_format('csv', csv_wrapper) + self.register_output_format('tsv', csv_wrapper, delimiter='\t') self.register_output_format('expanded', expanded_table) def register_output_format(self, name, function, **kwargs): From 93c6306b16c6316e352c6f6307b45c49602258dd Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:07:11 -0500 Subject: [PATCH 110/824] Add space before expanded. --- mycli/output_formatter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 6b0d8f2a4..29922b866 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -52,6 +52,7 @@ def __init__(self): self.register_output_format('csv', csv_wrapper) self.register_output_format('tsv', csv_wrapper, delimiter='\t') + self.register_output_format('expanded', expanded_table) def register_output_format(self, name, function, **kwargs): From 3a881284339f71033d29e78cef288589efcb5fe1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:25:42 -0500 Subject: [PATCH 111/824] Add preprocessor. --- mycli/output_formatter.py | 32 +++++++++++++++++++++++--------- mycli/packages/expanded.py | 2 +- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 29922b866..9cc602671 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -13,20 +13,23 @@ from .packages.expanded import expanded_table -def tabulate_wrapper(data, headers, table_format=None, missing_value=''): +def override_missing_value(data, missing_value='', **_): + """Override missing values in the data with *missing_value*.""" + return [[missing_value if v is None else v for v in row] for row in data] + + +def tabulate_wrapper(data, headers, table_format=None, **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value) + return tabulate(data, headers, tablefmt=table_format) -def csv_wrapper(data, headers, missing_value='null', delimiter=','): +def csv_wrapper(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() writer = csv.writer(content, delimiter=delimiter) writer.writerow(headers) for row in data: - row = [missing_value if val is None else str(val) for val in row] writer.writerow(row) output = content.getvalue() @@ -48,12 +51,20 @@ def __init__(self): 'textile') for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, - table_format=tabulate_format) + table_format=tabulate_format, + preprocessor=override_missing_value, + missing_value='') - self.register_output_format('csv', csv_wrapper) - self.register_output_format('tsv', csv_wrapper, delimiter='\t') + self.register_output_format('csv', csv_wrapper, + preprocessor=override_missing_value, + missing_value='null') + self.register_output_format('tsv', csv_wrapper, delimiter='\t', + preprocessor=override_missing_value, + missing_value='null') - self.register_output_format('expanded', expanded_table) + self.register_output_format('expanded', expanded_table, + preprocessor=override_missing_value, + missing_value='') def register_output_format(self, name, function, **kwargs): """Register a new output format. @@ -79,4 +90,7 @@ def format_output(self, data, headers, format_name, **kwargs): """ function, fkwargs = self._output_formats[format_name] fkwargs.update(kwargs) + preprocessor = fkwargs.pop('preprocessor', None) + if preprocessor: + data = preprocessor(data, **fkwargs) return function(data, headers, **fkwargs) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 245e4ca72..2b0b7207f 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -19,7 +19,7 @@ def format_field(value): except UnicodeDecodeError: return _text_type('0x' + binascii.hexlify(value).decode('ascii')) -def expanded_table(rows, headers): +def expanded_table(rows, headers, **_): header_len = max([len(x) for x in headers]) max_row_len = 0 results = [] From cff2ddd8b198d4d47d45ef67f5d7493d95b073be Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:33:41 -0500 Subject: [PATCH 112/824] Simplify missing value. --- mycli/output_formatter.py | 6 +++--- mycli/packages/expanded.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 9cc602671..9c3d7d028 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -18,9 +18,10 @@ def override_missing_value(data, missing_value='', **_): return [[missing_value if v is None else v for v in row] for row in data] -def tabulate_wrapper(data, headers, table_format=None, **_): +def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format) + return tabulate(data, headers, tablefmt=table_format, + missingval=missing_value) def csv_wrapper(data, headers, delimiter=',', **_): @@ -52,7 +53,6 @@ def __init__(self): for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, table_format=tabulate_format, - preprocessor=override_missing_value, missing_value='') self.register_output_format('csv', csv_wrapper, diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 2b0b7207f..b39d9507f 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -35,7 +35,6 @@ def expanded_table(rows, headers, **_): max_row_len = row_len for header, value in zip(padded_headers, row): - if value is None: value = '' row_result.append(u"%s %s" % (header, value)) results.append('\n'.join(row_result)) From 5cbdf480e9cc3a65d6ac54619ec797a3cd4e41e0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:36:36 -0500 Subject: [PATCH 113/824] Disable number parsing. --- mycli/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 9c3d7d028..2d73e1a56 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -21,7 +21,7 @@ def override_missing_value(data, missing_value='', **_): def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value) + missingval=missing_value, disable_numparse=True) def csv_wrapper(data, headers, delimiter=',', **_): From cced4902517706581ec93237a4424e7be5a3b2c0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:44:23 -0500 Subject: [PATCH 114/824] Remove unused imports. --- mycli/main.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index cb40aaf09..beeeb7d23 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -5,9 +5,7 @@ import os import os.path import sys -import csv import traceback -import socket import logging import threading from time import time @@ -27,9 +25,8 @@ ConditionalProcessor) from prompt_toolkit.history import FileHistory from pygments.token import Token -from configobj import ConfigObj, ConfigObjError -from .packages.special.main import (COMMANDS, NO_QUERY) +from .packages.special.main import NO_QUERY import mycli.packages.special as special from .sqlcompleter import SQLCompleter from .clitoolbar import create_toolbar_tokens_func @@ -38,8 +35,7 @@ from .clibuffer import CLIBuffer from .completion_refresher import CompletionRefresher from .config import (write_default_config, get_mylogin_cnf_path, - open_mylogin_cnf, read_config_file, - read_config_files, str_to_bool) + open_mylogin_cnf, read_config_files, str_to_bool) from .key_bindings import mycli_bindings from .output_formatter import OutputFormatter from .encodingutils import utf8tounicode From d0f1dbf441ae823d56cadca456b1ff35a35abd88 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:47:06 -0500 Subject: [PATCH 115/824] Remove tabulate license note. --- LICENSE.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 9a41a67d4..8afaa2657 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -27,9 +27,3 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------- - -This program also bundles with it python-tabulate -(https://pypi.python.org/pypi/tabulate) library. This library is licensed under -MIT License. - -------------------------------------------------------------------------------- From 369b5b0795d5391870f260c8563cb6e27e895308 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 11:39:29 +0200 Subject: [PATCH 116/824] Fix tests/test_sqlexecute unit tests --- mycli/output_formatter.py | 17 +++++++++++++++++ tests/test_sqlexecute.py | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 2d73e1a56..1397b54f6 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import csv +import binascii try: from cStringIO import StringIO except ImportError: @@ -18,6 +19,21 @@ def override_missing_value(data, missing_value='', **_): return [[missing_value if v is None else v for v in row] for row in data] +def bytes_to_unicode(data, **_): + results = [] + for row in data: + result = [] + for v in row: + if isinstance(v, bytes): + try: + conv = v.decode('utf8') + except: + v = '0x' + binascii.hexlify(v).decode('ascii') + result.append(v) + results.append(result) + return results + + def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, @@ -52,6 +68,7 @@ def __init__(self): 'textile') for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, + preprocessor=bytes_to_unicode, table_format=tabulate_format, missing_value='') diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index e9929a70f..d964f43ba 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -27,9 +27,9 @@ def test_bools(executor): results = run(executor, '''select * from test''', join=True) assert results == dedent("""\ +-----+ - | a | + | a | |-----| - | 1 | + | 1 | +-----+ 1 row in set""") From ae3712f764aaf22aa5a2fe3c9a277b375f541bc5 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 12:06:01 +0200 Subject: [PATCH 117/824] fix test_main unit tests --- tests/test_main.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index cb769a0be..29501f137 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,6 +8,8 @@ thanks_picker, PACKAGE_ROOT) from utils import USER, HOST, PORT, PASSWORD, dbtest, run +from textwrap import dedent + try: text_type = basestring except NameError: @@ -80,7 +82,7 @@ def test_batch_mode(executor): result = runner.invoke(cli, args=CLI_ARGS, input=sql) assert result.exit_code == 0 - assert 'count(*)\n3\na\nabc\n' in result.output + assert 'count(*)\n3\n\na\nabc\n' in result.output @dbtest def test_batch_mode_table(executor): @@ -95,10 +97,17 @@ def test_batch_mode_table(executor): runner = CliRunner() result = runner.invoke(cli, args=CLI_ARGS + ['-t'], input=sql) - expected = ( - '| count(*) |\n|------------|\n| 3 |\n+------------+\n' - '+-----+\n| a |\n|-----|\n| abc |\n+-----+' - ) + expected = (dedent("""\ + +------------+ + | count(*) | + |------------| + | 3 | + +------------+ + +-----+ + | a | + |-----| + | abc | + +-----+""")) assert result.exit_code == 0 assert expected in result.output From 3dbe5c01cb03fa42b6dc7718b3b4b8bfcfc5c92a Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 12:39:37 +0200 Subject: [PATCH 118/824] use str for python2 delimiter --- mycli/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 1397b54f6..178add2f7 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -43,7 +43,7 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): def csv_wrapper(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() - writer = csv.writer(content, delimiter=delimiter) + writer = csv.writer(content, delimiter=str(delimiter)) writer.writerow(headers) for row in data: From 434aea0d3f31cde2f76309b1eea429e1b97f6527 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 16:37:06 +0200 Subject: [PATCH 119/824] add docstring to bytes_to_unicode function --- mycli/output_formatter.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 178add2f7..2375c0dd6 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """A generic output formatter interface.""" from __future__ import unicode_literals @@ -20,6 +21,12 @@ def override_missing_value(data, missing_value='', **_): def bytes_to_unicode(data, **_): + """Convert bytes that cannot be decoded to utf8 to hexlified string + >>> result = bytes_to_unicode([[b"\\xff", "abc", "✌"]]) + >>> print(" ".join(result[0])) + 0xff abc ✌ + """ + results = [] for row in data: result = [] From be0e2ed85917ec4015bbde46f132b85d511e4e45 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 10:00:13 -0500 Subject: [PATCH 120/824] Add headers to preprocessor. --- mycli/output_formatter.py | 44 +++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 2375c0dd6..ba79e8037 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -15,30 +15,34 @@ from .packages.expanded import expanded_table -def override_missing_value(data, missing_value='', **_): +def override_missing_value(data, headers, missing_value='', **_): """Override missing values in the data with *missing_value*.""" - return [[missing_value if v is None else v for v in row] for row in data] + return ([[missing_value if v is None else v for v in row] for row in data], + headers) -def bytes_to_unicode(data, **_): - """Convert bytes that cannot be decoded to utf8 to hexlified string - >>> result = bytes_to_unicode([[b"\\xff", "abc", "✌"]]) - >>> print(" ".join(result[0])) - 0xff abc ✌ +def bytes_to_hex(b): + """Convert bytes that cannot be decoded to utf8 to hexlified string. + + >>> print(bytes_to_hex(b"\\xff")) + 0xff + >>> print(bytes_to_hex('abc')) + abc + >>> print(bytes_to_hex('✌')) + ✌ """ + if isinstance(b, bytes): + try: + b.decode('utf8') + except: + b = '0x' + binascii.hexlify(b).decode('ascii') + return b - results = [] - for row in data: - result = [] - for v in row: - if isinstance(v, bytes): - try: - conv = v.decode('utf8') - except: - v = '0x' + binascii.hexlify(v).decode('ascii') - result.append(v) - results.append(result) - return results + +def bytes_to_unicode(data, headers, **_): + """Convert all *data* and *headers* to unicode.""" + return ([[bytes_to_hex(v) for v in row] for row in data], + [bytes_to_hex(h) for h in headers]) def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): @@ -116,5 +120,5 @@ def format_output(self, data, headers, format_name, **kwargs): fkwargs.update(kwargs) preprocessor = fkwargs.pop('preprocessor', None) if preprocessor: - data = preprocessor(data, **fkwargs) + data, headers = preprocessor(data, headers, **fkwargs) return function(data, headers, **fkwargs) From f4d05278d7984e7d3cdc9d0361bf3a9f89d89cfc Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:10:20 -0500 Subject: [PATCH 121/824] Move mycli table format to output formatter. --- mycli/main.py | 48 +++++++++++++++++++-------------------- mycli/output_formatter.py | 26 ++++++++++++++++++--- tests/utils.py | 1 - 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index beeeb7d23..fa715d26a 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -106,7 +106,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.table_format = c['main']['table_format'] + self.formatter = OutputFormatter(format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] @@ -133,8 +133,6 @@ def __init__(self, sqlexecute=None, prompt=None, self.completion_refresher = CompletionRefresher() - self.formatter = OutputFormatter() - self.logger = logging.getLogger(__name__) self.initialize_logging() @@ -182,14 +180,16 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - if arg not in self.formatter.supported_formats(): - msg = "Table type %s not yet implemented. Allowed types:" % arg + print('change table: ' + arg) + try: + self.formatter.set_format_name(arg) + yield (None, None, None, + 'Changed table type to {}'.format(arg)) + except ValueError: + msg = 'Table type {} not yet implemented. Allowed types:'.format(arg) for table_type in self.formatter.supported_formats(): - msg += "\n\t%s" % table_type + msg += "\n\t{}".format(table_type) yield (None, None, None, msg) - else: - self.table_format = arg - yield (None, None, None, "Changed table Type to %s" % self.table_format) def change_db(self, arg, **_): if arg is None: @@ -716,7 +716,7 @@ def run_query(self, query, new_line=True): def format_output(self, title, cur, headers, status, expanded=False, max_width=None): - table_format = 'expanded' if expanded else self.table_format + expanded = expanded or self.formatter.get_format_name() == 'expanded' output = [] if title: # Only print the title if it's not None. @@ -726,11 +726,13 @@ def format_output(self, title, cur, headers, status, expanded=False, headers = [utf8tounicode(x) for x in headers] rows = list(cur) - formatted = self.formatter.format_output(rows, headers, table_format) + formatted = self.formatter.format_output( + rows, headers, format_name='expanded' if expanded else None) - if (table_format != 'expanded' and max_width and rows and + if (not expanded and max_width and rows and content_exceeds_width(rows[0], max_width) and headers): - formatted = self.formatter.format_output(rows, headers, 'expanded') + formatted = self.formatter.format_output( + rows, headers, format_name='expanded') output.append(formatted) @@ -834,12 +836,11 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - table_format = None - if table: - table_format = mycli.table_format - elif csv: - table_format = 'csv' - mycli.table_format = table_format or 'tsv' + if csv: + mycli.formatter.set_format_name('csv') + elif not table: + mycli.formatter.set_format_name('tsv') + mycli.run_query(execute) exit(0) except Exception as e: @@ -861,16 +862,13 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - table_format = None new_line = True if csv: - table_format = 'csv' + mycli.formatter.set_format_name('csv') new_line = False - elif table: - table_format = mycli.table_format - - mycli.table_format = table_format or 'tsv' + elif not table: + mycli.formatter.set_format_name('tsv') mycli.run_query(stdin_text, new_line=new_line) exit(0) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index ba79e8037..ebb92b1d7 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -3,8 +3,8 @@ from __future__ import unicode_literals -import csv import binascii +import csv try: from cStringIO import StringIO except ImportError: @@ -69,9 +69,10 @@ def csv_wrapper(data, headers, delimiter=',', **_): class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" - def __init__(self): + def __init__(self, format_name=None): """Register the supported output formats.""" self._output_formats = {} + self._format_name = None tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', @@ -94,6 +95,21 @@ def __init__(self): preprocessor=override_missing_value, missing_value='') + if format_name: + self.set_format_name(format_name) + + def set_format_name(self, format_name): + """Set the OutputFormatter's default format.""" + if format_name in self.supported_formats(): + self._format_name = format_name + else: + raise ValueError('unrecognized format_name: {}'.format( + format_name)) + + def get_format_name(self): + """Get the OutputFormatter's default format.""" + return self._format_name + def register_output_format(self, name, function, **kwargs): """Register a new output format. @@ -109,13 +125,17 @@ def supported_formats(self): """Return the supported output format names.""" return tuple(self._output_formats.keys()) - def format_output(self, data, headers, format_name, **kwargs): + def format_output(self, data, headers, format_name=None, **kwargs): """Format the headers and data using a specific formatter. *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. """ + format_name = format_name or self._format_name + if format_name not in self.supported_formats(): + raise ValueError('unrecognized format: {}'.format(format_name)) + function, fkwargs = self._output_formats[format_name] fkwargs.update(kwargs) preprocessor = fkwargs.pop('preprocessor', None) diff --git a/tests/utils.py b/tests/utils.py index 5e9e0abb0..64006324f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,7 +44,6 @@ def run(executor, sql, join=False): # TODO: this needs to go away. `run()` should not test formatted output. # It should test raw results. mycli = MyCli() - mycli.table_format = 'psql' for title, rows, headers, status in executor.run(sql): result.extend(mycli.format_output(title, rows, headers, status, special.is_expanded_output())) From 90fd5fe54e05b5a9f8e33af35f71a5fcae8ebce6 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:20:34 -0500 Subject: [PATCH 122/824] Use context manager for StringIO. --- mycli/output_formatter.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index ebb92b1d7..6a0564890 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import binascii +import contextlib import csv try: from cStringIO import StringIO @@ -53,17 +54,14 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): def csv_wrapper(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" - content = StringIO() - writer = csv.writer(content, delimiter=str(delimiter)) - writer.writerow(headers) + with contextlib.closing(StringIO()) as content: + writer = csv.writer(content, delimiter=str(delimiter)) - for row in data: - writer.writerow(row) + writer.writerow(headers) + for row in data: + writer.writerow(row) - output = content.getvalue() - content.close() - - return output + return content.getvalue() class OutputFormatter(object): From 9f0ace286f60875d708018a41b10c7015cebcac7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:27:34 -0500 Subject: [PATCH 123/824] Remove register output format method. --- mycli/output_formatter.py | 47 +++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 6a0564890..14339cb77 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -69,7 +69,21 @@ class OutputFormatter(object): def __init__(self, format_name=None): """Register the supported output formats.""" - self._output_formats = {} + self._output_formats = { + 'csv': (csv_wrapper, { + 'preprocessor': override_missing_value, + 'missing_value': '' + }), + 'tsv': (csv_wrapper, { + 'preprocessor': override_missing_value, + 'missing_value': '', + 'delimiter': '\t' + }), + 'expanded': (expanded_table, { + 'preprocessor': override_missing_value, + 'missing_value': '' + }) + } self._format_name = None tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', @@ -77,21 +91,11 @@ def __init__(self, format_name=None): 'moinmoin', 'html', 'latex', 'latex_booktabs', 'textile') for tabulate_format in tabulate_formats: - self.register_output_format(tabulate_format, tabulate_wrapper, - preprocessor=bytes_to_unicode, - table_format=tabulate_format, - missing_value='') - - self.register_output_format('csv', csv_wrapper, - preprocessor=override_missing_value, - missing_value='null') - self.register_output_format('tsv', csv_wrapper, delimiter='\t', - preprocessor=override_missing_value, - missing_value='null') - - self.register_output_format('expanded', expanded_table, - preprocessor=override_missing_value, - missing_value='') + self._output_formats[tabulate_format] = ( + tabulate_wrapper, {'preprocessor': bytes_to_unicode, + 'table_format': tabulate_format, + 'missing_value': ''} + ) if format_name: self.set_format_name(format_name) @@ -108,17 +112,6 @@ def get_format_name(self): """Get the OutputFormatter's default format.""" return self._format_name - def register_output_format(self, name, function, **kwargs): - """Register a new output format. - - *function* should be a callable that accepts the following arguments: - - *headers*: A list of headers for the output data. - - *data*: The data that needs formatting. - - *kwargs*: Any other keyword arguments for controlling the output. - It should return the formatted output as a string. - """ - self._output_formats[name] = (function, kwargs) - def supported_formats(self): """Return the supported output format names.""" return tuple(self._output_formats.keys()) From e7b93b8162b663ad745948ccbecb9f839c78d05f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:31:52 -0500 Subject: [PATCH 124/824] Remove print statement. --- mycli/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index fa715d26a..661dc65d2 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -180,7 +180,6 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - print('change table: ' + arg) try: self.formatter.set_format_name(arg) yield (None, None, None, From b58cad04f7dda089612c5da03c93b87aa5b221a6 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:34:31 -0500 Subject: [PATCH 125/824] Update list of table formats. --- mycli/myclirc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/myclirc b/mycli/myclirc index ff6e8f734..36e9e409a 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -31,7 +31,8 @@ log_level = INFO timing = True # Table format. Possible values: psql, plain, simple, grid, fancy_grid, pipe, -# orgtbl, rst, mediawiki, html, latex, latex_booktabs, tsv. +# orgtbl, jira, rst, mediawiki, moinmoin, html, latex, latex_booktabs, +# textile, csv, tsv. # Recommended: psql, fancy_grid and grid. table_format = psql From cbd5ba0a58d699e402a1ef2c8b4665d3bfc375ae Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:43:03 -0500 Subject: [PATCH 126/824] Move bytes_to_hex to encoding utils package. --- mycli/encodingutils.py | 21 +++++++++++++++++++++ mycli/output_formatter.py | 20 +------------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index 29564d08e..ed8f9a158 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -1,8 +1,10 @@ +import binascii import sys PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 + def unicode2utf8(arg): """ Only in Python 2. Psycopg2 expects the args as bytes not unicode. @@ -13,6 +15,7 @@ def unicode2utf8(arg): return arg.encode('utf-8') return arg + def utf8tounicode(arg): """ Only in Python 2. Psycopg2 returns the error message as utf-8. @@ -22,3 +25,21 @@ def utf8tounicode(arg): if PY2 and isinstance(arg, str): return arg.decode('utf-8') return arg + + +def bytes_to_hex(b): + """Convert bytes that cannot be decoded to utf8 to hexlified string. + + >>> print(bytes_to_hex(b"\\xff")) + 0xff + >>> print(bytes_to_hex('abc')) + abc + >>> print(bytes_to_hex('✌')) + ✌ + """ + if isinstance(b, bytes): + try: + b.decode('utf8') + except: + b = '0x' + binascii.hexlify(b).decode('ascii') + return b diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 14339cb77..a1319fd8e 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals -import binascii import contextlib import csv try: @@ -13,6 +12,7 @@ from tabulate import tabulate +from .encodingutils import bytes_to_hex from .packages.expanded import expanded_table @@ -22,24 +22,6 @@ def override_missing_value(data, headers, missing_value='', **_): headers) -def bytes_to_hex(b): - """Convert bytes that cannot be decoded to utf8 to hexlified string. - - >>> print(bytes_to_hex(b"\\xff")) - 0xff - >>> print(bytes_to_hex('abc')) - abc - >>> print(bytes_to_hex('✌')) - ✌ - """ - if isinstance(b, bytes): - try: - b.decode('utf8') - except: - b = '0x' + binascii.hexlify(b).decode('ascii') - return b - - def bytes_to_unicode(data, headers, **_): """Convert all *data* and *headers* to unicode.""" return ([[bytes_to_hex(v) for v in row] for row in data], From ae648abb91f92645832c54675039ea3a7d8e0154 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:47:39 -0500 Subject: [PATCH 127/824] Fix encoding issue. --- mycli/encodingutils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index ed8f9a158..43ce6c7b6 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + import binascii import sys From 6ce7e158ed969a6d381f10cf0d1551ae63c04fd7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 15:27:56 -0500 Subject: [PATCH 128/824] Do not rely on tabulate for text type. --- mycli/encodingutils.py | 13 ++++++++++--- mycli/packages/expanded.py | 11 ++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index 43ce6c7b6..c5f32e9d2 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -7,6 +7,13 @@ PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 +if PY2: + text_type = unicode + binary_type = str +else: + text_type = str + binary_type = bytes + def unicode2utf8(arg): """ @@ -14,7 +21,7 @@ def unicode2utf8(arg): In Python 3 the args are expected as unicode. """ - if PY2 and isinstance(arg, unicode): + if PY2 and isinstance(arg, text_type): return arg.encode('utf-8') return arg @@ -25,7 +32,7 @@ def utf8tounicode(arg): In Python 3 the errors are returned as unicode. """ - if PY2 and isinstance(arg, str): + if PY2 and isinstance(arg, binary_type): return arg.decode('utf-8') return arg @@ -40,7 +47,7 @@ def bytes_to_hex(b): >>> print(bytes_to_hex('✌')) ✌ """ - if isinstance(b, bytes): + if isinstance(b, binary_type): try: b.decode('utf8') except: diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index b39d9507f..d79b9acd3 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,6 +1,7 @@ -from tabulate import _text_type import binascii +from .encodingutils import binary_type, text_type + def pad(field, total, char=u" "): return field + (char * (total - len(field))) @@ -12,12 +13,12 @@ def get_separator(num, header_len, data_len): def format_field(value): # Returns the field as a text type, otherwise will hexify the string try: - if isinstance(value, bytes): - return _text_type(value, "ascii") + if isinstance(value, binary_type): + return text_type(value, "ascii") else: - return _text_type(value) + return text_type(value) except UnicodeDecodeError: - return _text_type('0x' + binascii.hexlify(value).decode('ascii')) + return text_type('0x' + binascii.hexlify(value).decode('ascii')) def expanded_table(rows, headers, **_): header_len = max([len(x) for x in headers]) From 4a65f7bb98654f058e0f3cae1c06cf2f66bd33e2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 15:28:53 -0500 Subject: [PATCH 129/824] Fix import. --- mycli/packages/expanded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index d79b9acd3..d1108c2eb 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,6 +1,6 @@ import binascii -from .encodingutils import binary_type, text_type +from mycli.encodingutils import binary_type, text_type def pad(field, total, char=u" "): return field + (char * (total - len(field))) From 02f5c521495eec0968d99fdf603be958968a2a5e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:24:03 -0500 Subject: [PATCH 130/824] Move pre-processing to output_formatter. --- mycli/encodingutils.py | 16 +++++++------- mycli/main.py | 2 -- mycli/output_formatter.py | 45 +++++++++++++++++++++++++------------- mycli/packages/expanded.py | 29 ++++++++++-------------- tests/test_expanded.py | 3 ++- 5 files changed, 52 insertions(+), 43 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index c5f32e9d2..aac4aa07d 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -37,19 +37,19 @@ def utf8tounicode(arg): return arg -def bytes_to_hex(b): - """Convert bytes that cannot be decoded to utf8 to hexlified string. +def bytes_to_string(b): + """Convert bytes to a string. Hexlify bytes that can't be decoded. - >>> print(bytes_to_hex(b"\\xff")) + >>> print(bytes_to_string(b"\\xff")) 0xff - >>> print(bytes_to_hex('abc')) + >>> print(bytes_to_string('abc')) abc - >>> print(bytes_to_hex('✌')) + >>> print(bytes_to_string('✌')) ✌ """ if isinstance(b, binary_type): try: - b.decode('utf8') - except: - b = '0x' + binascii.hexlify(b).decode('ascii') + return b.decode('utf8') + except UnicodeDecodeError: + return '0x' + binascii.hexlify(b).decode('ascii') return b diff --git a/mycli/main.py b/mycli/main.py index 661dc65d2..faca8b37d 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -722,8 +722,6 @@ def format_output(self, title, cur, headers, status, expanded=False, output.append(title) if cur: - headers = [utf8tounicode(x) for x in headers] - rows = list(cur) formatted = self.formatter.format_output( rows, headers, format_name='expanded' if expanded else None) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index a1319fd8e..5474b41f6 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -12,20 +12,34 @@ from tabulate import tabulate -from .encodingutils import bytes_to_hex +from . import encodingutils from .packages.expanded import expanded_table +def to_string(value): + """Convert *value* to a string.""" + if isinstance(value, encodingutils.binary_type): + return encodingutils.bytes_to_string(value) + else: + return encodingutils.text_type(value) + + +def convert_to_string(data, headers, **_): + """Convert all *data* and *headers* to strings.""" + return ([[to_string(v) for v in row] for row in data], + [to_string(h) for h in headers]) + + def override_missing_value(data, headers, missing_value='', **_): """Override missing values in the data with *missing_value*.""" return ([[missing_value if v is None else v for v in row] for row in data], headers) -def bytes_to_unicode(data, headers, **_): - """Convert all *data* and *headers* to unicode.""" - return ([[bytes_to_hex(v) for v in row] for row in data], - [bytes_to_hex(h) for h in headers]) +def bytes_to_string(data, headers, **_): + """Convert all *data* and *headers* to strings.""" + return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], + [encodingutils.bytes_to_string(h) for h in headers]) def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): @@ -53,16 +67,16 @@ def __init__(self, format_name=None): """Register the supported output formats.""" self._output_formats = { 'csv': (csv_wrapper, { - 'preprocessor': override_missing_value, + 'preprocessor': (override_missing_value, bytes_to_string), 'missing_value': '' }), 'tsv': (csv_wrapper, { - 'preprocessor': override_missing_value, + 'preprocessor': (override_missing_value, bytes_to_string), 'missing_value': '', 'delimiter': '\t' }), 'expanded': (expanded_table, { - 'preprocessor': override_missing_value, + 'preprocessor': (override_missing_value, convert_to_string), 'missing_value': '' }) } @@ -73,11 +87,11 @@ def __init__(self, format_name=None): 'moinmoin', 'html', 'latex', 'latex_booktabs', 'textile') for tabulate_format in tabulate_formats: - self._output_formats[tabulate_format] = ( - tabulate_wrapper, {'preprocessor': bytes_to_unicode, - 'table_format': tabulate_format, - 'missing_value': ''} - ) + self._output_formats[tabulate_format] = (tabulate_wrapper, { + 'preprocessor': (bytes_to_string, ), + 'table_format': tabulate_format, + 'missing_value': '' + }) if format_name: self.set_format_name(format_name) @@ -111,7 +125,8 @@ def format_output(self, data, headers, format_name=None, **kwargs): function, fkwargs = self._output_formats[format_name] fkwargs.update(kwargs) - preprocessor = fkwargs.pop('preprocessor', None) + preprocessor = fkwargs.get('preprocessor', None) if preprocessor: - data, headers = preprocessor(data, headers, **fkwargs) + for f in preprocessor: + data, headers = f(data, headers, **fkwargs) return function(data, headers, **fkwargs) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index d1108c2eb..c3a8e9a53 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,42 +1,37 @@ -import binascii +"""Format data into a vertical, expanded table layout.""" -from mycli.encodingutils import binary_type, text_type +from __future__ import unicode_literals -def pad(field, total, char=u" "): + +def pad(field, total, char=' '): return field + (char * (total - len(field))) -def get_separator(num, header_len, data_len): - sep = u"***************************[ %d. row ]***************************\n" % (num + 1) +def get_separator(num, header_len, data_len): + sep = "***************************[ %d. row ]***************************\n" % (num + 1) return sep -def format_field(value): - # Returns the field as a text type, otherwise will hexify the string - try: - if isinstance(value, binary_type): - return text_type(value, "ascii") - else: - return text_type(value) - except UnicodeDecodeError: - return text_type('0x' + binascii.hexlify(value).decode('ascii')) def expanded_table(rows, headers, **_): + """Format *rows* and *headers* as an expanded table. + + The values in *rows* and *headers* must be strings. + """ header_len = max([len(x) for x in headers]) max_row_len = 0 results = [] - padded_headers = [pad(x, header_len) + u" |" for x in headers] + padded_headers = [pad(x, header_len) + ' |' for x in headers] header_len += 2 for row in rows: - row = [format_field(x) for x in row] row_len = max([len(x) for x in row]) row_result = [] if row_len > max_row_len: max_row_len = row_len for header, value in zip(padded_headers, row): - row_result.append(u"%s %s" % (header, value)) + row_result.append('{0} {1}'.format(header, value)) results.append('\n'.join(row_result)) diff --git a/tests/test_expanded.py b/tests/test_expanded.py index 9b2a6c5cf..cec940696 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -1,7 +1,8 @@ from mycli.packages.expanded import expanded_table +from mycli.encodingutils import text_type def test_expanded_table_renders(): - input = [("hello", 123), ("world", 456)] + input = [("hello", text_type(123)), ("world", text_type(456))] expected = """***************************[ 1. row ]*************************** name | hello From 5fe9d68c9679fe7e40bbec719a05a8c4d9053dfb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:46:23 -0500 Subject: [PATCH 131/824] Refactor expanded table. --- mycli/packages/expanded.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index c3a8e9a53..0b9e6ef1d 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -3,13 +3,16 @@ from __future__ import unicode_literals -def pad(field, total, char=' '): - return field + (char * (total - len(field))) +def get_separator(num, header_len): + """Get a row separator.""" + sep = "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) + return sep -def get_separator(num, header_len, data_len): - sep = "***************************[ %d. row ]***************************\n" % (num + 1) - return sep +def format_row(headers, row): + """Format a row.""" + formatted_row = [' '.join(field) for field in zip(headers, row)] + return '\n'.join(formatted_row) def expanded_table(rows, headers, **_): @@ -18,26 +21,12 @@ def expanded_table(rows, headers, **_): The values in *rows* and *headers* must be strings. """ header_len = max([len(x) for x in headers]) - max_row_len = 0 - results = [] - - padded_headers = [pad(x, header_len) + ' |' for x in headers] - header_len += 2 - - for row in rows: - row_len = max([len(x) for x in row]) - row_result = [] - if row_len > max_row_len: - max_row_len = row_len - - for header, value in zip(padded_headers, row): - row_result.append('{0} {1}'.format(header, value)) - - results.append('\n'.join(row_result)) + padded_headers = ['{} |'.format(x.ljust(header_len)) for x in headers] + results = [format_row(padded_headers, row) for row in rows] output = [] for i, result in enumerate(results): - output.append(get_separator(i, header_len, max_row_len)) + output.append(get_separator(i, header_len + 2)) output.append(result) output.append('\n') From 2052ae45925ce10412b1b1cb26a8cab593d51e0e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:48:22 -0500 Subject: [PATCH 132/824] Remove extra variable. --- mycli/packages/expanded.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 0b9e6ef1d..e4283d73f 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -5,8 +5,7 @@ def get_separator(num, header_len): """Get a row separator.""" - sep = "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) - return sep + return "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) def format_row(headers, row): From 328a7673e13ee40f1518edd3231b8a93471ece32 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:50:12 -0500 Subject: [PATCH 133/824] Remove extra argument to get_separator. --- mycli/packages/expanded.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index e4283d73f..83ad4276c 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -3,8 +3,8 @@ from __future__ import unicode_literals -def get_separator(num, header_len): - """Get a row separator.""" +def get_separator(num): + """Get a row separator for row *num*.""" return "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) @@ -25,7 +25,7 @@ def expanded_table(rows, headers, **_): output = [] for i, result in enumerate(results): - output.append(get_separator(i, header_len + 2)) + output.append(get_separator(i)) output.append(result) output.append('\n') From 041eafbca08a1729698b59f6fee252ef96c686eb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:52:09 -0500 Subject: [PATCH 134/824] Use named format strings for clarity. --- mycli/packages/expanded.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 83ad4276c..3ec938291 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -5,7 +5,8 @@ def get_separator(num): """Get a row separator for row *num*.""" - return "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) + return "{divider}[ {n}. row ]{divider}\n".format( + divider='*' * 27, n=num + 1) def format_row(headers, row): From 3a10f5bb0c4ca5b4d099541591fd9618ce5e6d23 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:56:36 -0500 Subject: [PATCH 135/824] Move divider to format_row function. --- mycli/packages/expanded.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 3ec938291..2ab579944 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -11,7 +11,7 @@ def get_separator(num): def format_row(headers, row): """Format a row.""" - formatted_row = [' '.join(field) for field in zip(headers, row)] + formatted_row = [' | '.join(field) for field in zip(headers, row)] return '\n'.join(formatted_row) @@ -21,11 +21,11 @@ def expanded_table(rows, headers, **_): The values in *rows* and *headers* must be strings. """ header_len = max([len(x) for x in headers]) - padded_headers = ['{} |'.format(x.ljust(header_len)) for x in headers] - results = [format_row(padded_headers, row) for row in rows] + padded_headers = [x.ljust(header_len) for x in headers] + formatted_rows = [format_row(padded_headers, row) for row in rows] output = [] - for i, result in enumerate(results): + for i, result in enumerate(formatted_rows): output.append(get_separator(i)) output.append(result) output.append('\n') From 61bdbd98c5569b84a7b994bd67a457d4bfe62a2a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 21:03:17 -0500 Subject: [PATCH 136/824] Make the expanded format test more readable. --- tests/test_expanded.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/test_expanded.py b/tests/test_expanded.py index cec940696..db503cba8 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -1,14 +1,19 @@ +"""Test the vertical, expanded table formatter.""" +from textwrap import dedent + from mycli.packages.expanded import expanded_table from mycli.encodingutils import text_type + def test_expanded_table_renders(): - input = [("hello", text_type(123)), ("world", text_type(456))] - - expected = """***************************[ 1. row ]*************************** -name | hello -age | 123 -***************************[ 2. row ]*************************** -name | world -age | 456 -""" - assert expected == expanded_table(input, ["name", "age"]) + results = [('hello', text_type(123)), ('world', text_type(456))] + + expected = dedent("""\ + ***************************[ 1. row ]*************************** + name | hello + age | 123 + ***************************[ 2. row ]*************************** + name | world + age | 456 + """) + assert expected == expanded_table(results, ('name', 'age')) From 0f01a9d5767e1dc0d079633e806770099255af28 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 21:08:44 -0500 Subject: [PATCH 137/824] Update docstring to be clearer. --- mycli/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5474b41f6..86db809c0 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -37,7 +37,7 @@ def override_missing_value(data, headers, missing_value='', **_): def bytes_to_string(data, headers, **_): - """Convert all *data* and *headers* to strings.""" + """Convert all *data* and *headers* bytes to strings.""" return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], [encodingutils.bytes_to_string(h) for h in headers]) From 4b47e43eb8b7d332d64bd6c2f6575e34d3451b67 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Thu, 23 Mar 2017 12:58:37 +0100 Subject: [PATCH 138/824] Use pymysql default conversions (instead of only our conversions) This will convert a MySQL DECIMAL to the Python's decimal.Decimal type. Tabulate handles Decimal as a string (and not a number) and will not remove the padding (fixes Issue#375) --- changelog.md | 1 + mycli/sqlexecute.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 924d8efa4..f87407e34 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,7 @@ Bug Fixes: * Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) * Fix bug where mycli would not start due to the thanks/credit intro text. (Thanks: [Thomas Roten]). +* Use pymysql default conversions (issue #375). (Thanks: [Dick Marinus]). Internal Changes: ----------------- diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 786c8f0ec..08030af47 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -4,7 +4,7 @@ from .packages import special from pymysql.constants import FIELD_TYPE from pymysql.converters import (convert_mysql_timestamp, convert_datetime, - convert_timedelta, convert_date) + convert_timedelta, convert_date, conversions) _logger = logging.getLogger(__name__) @@ -65,12 +65,13 @@ def connect(self, database=None, user=None, password=None, host=None, '\tlocal_infile: %r' '\tssl: %r', database, user, host, port, socket, charset, local_infile, ssl) - conv = { + conv = conversions.copy() + conv.update({ FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), - } + }) conn = pymysql.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, From 5839c11d641f97660a6cded8e6a354ec54d72ce1 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sun, 2 Apr 2017 18:42:54 +0200 Subject: [PATCH 139/824] align floats on decimal point, quote columns with trailing/starting whitespace --- mycli/output_formatter.py | 97 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5474b41f6..35e266749 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -15,6 +15,8 @@ from . import encodingutils from .packages.expanded import expanded_table +from decimal import Decimal + def to_string(value): """Convert *value* to a string.""" @@ -42,6 +44,87 @@ def bytes_to_string(data, headers, **_): [encodingutils.bytes_to_string(h) for h in headers]) +def intlen(value): + """Find (character) length + >>> intlen('11.1') + 2 + >>> intlen('11') + 2 + >>> intlen('1.1') + 1 + """ + pos = value.find('.') + if pos < 0: + pos = len(value) + return pos + +def align_decimals(data, headers, **_): + """Align decimals to decimal point + >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) + 1 + 11.1 + 1.1 + """ + pointpos = len(data[0]) * [0] + for row in data: + i = 0 + for v in row: + if isinstance(v, Decimal): + v = str(v) + pointpos[i] = max(intlen(v), pointpos[i]) + i += 1 + results = [] + for row in data: + i = 0 + result = [] + for v in row: + if isinstance(v, Decimal): + v = str(v) + result.append((pointpos[i]-intlen(v))*" "+v) + else: + result.append(v) + i += 1 + results.append(result) + return results, headers + + +def quote_whitespaces(data, headers, quotestyle="'", **_): + """Quote whitespace + >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) + ' before' + 'after ' + ' both ' + 'none' + >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) + abc + def + ghi + jkl + """ + """Convert all *data* and *headers* to strings.""" + quote = len(data[0])*[False] + for row in data: + i = 0 + for v in row: + v = encodingutils.text_type(v) + if v[0] == ' ' or v[-1] == ' ': + quote[i] = True + i += 1 + + results = [] + for row in data: + result = [] + i = 0 + for v in row: + if quote[i]: + result.append('{quotestyle}{value}{quotestyle}'.format(quotestyle=quotestyle, value=v)) + else: + result.append(v) + i += 1 + results.append(result) + return results, headers + + def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, @@ -88,7 +171,7 @@ def __init__(self, format_name=None): 'textile') for tabulate_format in tabulate_formats: self._output_formats[tabulate_format] = (tabulate_wrapper, { - 'preprocessor': (bytes_to_string, ), + 'preprocessor': (bytes_to_string, align_decimals, quote_whitespaces), 'table_format': tabulate_format, 'missing_value': '' }) @@ -118,6 +201,18 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. + >>> print(OutputFormatter().format_output( \ + [["abc", Decimal(1)], ["defg", Decimal('11.1')], ["hi", Decimal('1.1')]], \ + ["text", "numeric"], \ + "psql" \ + )) + +--------+-----------+ + | text | numeric | + |--------+-----------| + | abc | ' 1' | + | defg | '11.1' | + | hi | ' 1.1' | + +--------+-----------+ """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): From 1db7f865b91f57baa19fd27234506d6974895801 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 13:24:47 -0500 Subject: [PATCH 140/824] Add basic output_formatter tests. --- tests/test_output_formatter.py | 82 ++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tests/test_output_formatter.py diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py new file mode 100644 index 000000000..f331262e8 --- /dev/null +++ b/tests/test_output_formatter.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +"""Test the generic output formatter interface.""" + +from __future__ import unicode_literals + +from textwrap import dedent + +from mycli.output_formatter import (bytes_to_string, convert_to_string, + csv_wrapper, OutputFormatter, + override_missing_value, tabulate_wrapper, + to_string) + + +def test_to_string(): + """Test the *output_formatter.to_string()* function.""" + assert 'a' == to_string('a') + assert 'a' == to_string(b'a') + assert '1' == to_string(1) + assert '1.23' == to_string(1.23) + + +def test_convert_to_string(): + """Test the *output_formatter.convert_to_string()* function.""" + data = [[1, 'John'], [2, 'Jill']] + headers = [0, 'name'] + expected = ([['1', 'John'], ['2', 'Jill']], ['0', 'name']) + + assert expected == convert_to_string(data, headers) + + +def test_override_missing_values(): + """Test the *output_formatter.override_missing_values()* function.""" + data = [[1, None], [2, 'Jill']] + headers = [0, 'name'] + expected = ([[1, ''], [2, 'Jill']], [0, 'name']) + + assert expected == override_missing_value(data, headers, + missing_value='') + + +def test_bytes_to_string(): + """Test the *output_formatter.bytes_to_string()* function.""" + data = [[1, 'John'], [2, b'Jill']] + headers = [0, 'name'] + expected = ([[1, 'John'], [2, 'Jill']], [0, 'name']) + + assert expected == bytes_to_string(data, headers) + + +def test_tabulate_wrapper(): + """Test the *output_formatter.tabulate_wrapper()* function.""" + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = tabulate_wrapper(data, headers, table_format='psql') + assert output == dedent('''\ + +-----------+----------+ + | letters | number | + |-----------+----------| + | abc | 1 | + | d | 456 | + +-----------+----------+''') + + +def test_csv_wrapper(): + """Test the *output_formatter.csv_wrapper()* function.""" + # Test comma-delimited output. + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = csv_wrapper(data, headers) + assert output == dedent('''\ + letters,number\r\n\ + abc,1\r\n\ + d,456\r\n''') + + # Test tab-delimited output. + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = csv_wrapper(data, headers, delimiter='\t') + assert output == dedent('''\ + letters\tnumber\r\n\ + abc\t1\r\n\ + d\t456\r\n''') From 86846167f1fd159a99b89f75d4e24d8290eff85e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 14:28:30 -0500 Subject: [PATCH 141/824] Add terminaltables. --- mycli/output_formatter.py | 33 ++++++++++++++++++++++++++++++--- setup.py | 1 + tests/test_output_formatter.py | 16 +++++++++++++++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index bc275984e..566a337f9 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -5,18 +5,18 @@ import contextlib import csv +from decimal import Decimal try: from cStringIO import StringIO except ImportError: from io import StringIO from tabulate import tabulate +import terminaltables from . import encodingutils from .packages.expanded import expanded_table -from decimal import Decimal - def to_string(value): """Convert *value* to a string.""" @@ -143,6 +143,23 @@ def csv_wrapper(data, headers, delimiter=',', **_): return content.getvalue() +def terminal_tables_wrapper(data, headers, table_format=None, **_): + """Wrap terminaltables inside a standard function for OutputFormatter.""" + if table_format == 'ascii': + table = terminaltables.AsciiTable + elif table_format == 'single': + table = terminaltables.SingleTable + elif table_format == 'double': + table = terminaltables.DoubleTable + elif table_format == 'github': + table = terminaltables.GithubFlavoredMarkdownTable + else: + raise ValueError('unrecognized table format: {}'.format(table_format)) + + t = table([headers] + data) + return t.table + + class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" @@ -171,11 +188,21 @@ def __init__(self, format_name=None): 'textile') for tabulate_format in tabulate_formats: self._output_formats[tabulate_format] = (tabulate_wrapper, { - 'preprocessor': (bytes_to_string, align_decimals, quote_whitespaces), + 'preprocessor': (bytes_to_string, align_decimals), 'table_format': tabulate_format, 'missing_value': '' }) + terminal_tables_formats = ('ascii', 'single', 'double', 'github') + for terminal_tables_format in terminal_tables_formats: + self._output_formats[terminal_tables_format] = ( + terminal_tables_wrapper, { + 'preprocessor': (bytes_to_string, override_missing_value, + align_decimals), + 'table_format': terminal_tables_format, + 'missing_value': '' + }) + if format_name: self.set_format_name(format_name) diff --git a/setup.py b/setup.py index 3ecc21c48..21e555086 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'configobj >= 5.0.5', 'pycryptodome >= 3', 'tabulate >= 0.7.6', + 'terminaltables >= 3.0.0', ] setup( diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index f331262e8..66ac42375 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -8,7 +8,7 @@ from mycli.output_formatter import (bytes_to_string, convert_to_string, csv_wrapper, OutputFormatter, override_missing_value, tabulate_wrapper, - to_string) + terminal_tables_wrapper, to_string) def test_to_string(): @@ -80,3 +80,17 @@ def test_csv_wrapper(): letters\tnumber\r\n\ abc\t1\r\n\ d\t456\r\n''') + + +def test_terminal_tables_wrapper(): + """Test the *output_formatter.terminal_tables_wrapper()* function.""" + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = terminal_tables_wrapper(data, headers, table_format='ascii') + assert output == dedent('''\ + +---------+--------+ + | letters | number | + +---------+--------+ + | abc | 1 | + | d | 456 | + +---------+--------+''') From 4c13163941b64dab71bad9a91906351f606c0e06 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 14:43:49 -0500 Subject: [PATCH 142/824] Default myclirc to ascii table. --- mycli/myclirc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mycli/myclirc b/mycli/myclirc index 36e9e409a..2f3599273 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -30,11 +30,9 @@ log_level = INFO # Timing of sql statments and table rendering. timing = True -# Table format. Possible values: psql, plain, simple, grid, fancy_grid, pipe, -# orgtbl, jira, rst, mediawiki, moinmoin, html, latex, latex_booktabs, -# textile, csv, tsv. -# Recommended: psql, fancy_grid and grid. -table_format = psql +# Table format. Possible values: ascii, single, double, or github. +# Recommended: ascii +table_format = ascii # Syntax coloring style. Possible values (many support the "-dark" suffix): # manni, igor, xcode, vim, autumn, vs, rrt, native, perldoc, borland, tango, emacs, From dd41cf394602523331d5c57152a6f223a7c52d06 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 14:43:58 -0500 Subject: [PATCH 143/824] Fix table format tests. --- mycli/output_formatter.py | 16 ++++++++-------- tests/test_main.py | 14 +++++++------- tests/test_sqlexecute.py | 32 ++++++++++++++++---------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 566a337f9..5e3305318 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -231,15 +231,15 @@ def format_output(self, data, headers, format_name=None, **kwargs): >>> print(OutputFormatter().format_output( \ [["abc", Decimal(1)], ["defg", Decimal('11.1')], ["hi", Decimal('1.1')]], \ ["text", "numeric"], \ - "psql" \ + "ascii" \ )) - +--------+-----------+ - | text | numeric | - |--------+-----------| - | abc | ' 1' | - | defg | '11.1' | - | hi | ' 1.1' | - +--------+-----------+ + +------+---------+ + | text | numeric | + +------+---------+ + | abc | 1 | + | defg | 11.1 | + | hi | 1.1 | + +------+---------+ """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): diff --git a/tests/test_main.py b/tests/test_main.py index 29501f137..1271f18da 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -48,7 +48,7 @@ def test_execute_arg_with_table(executor): sql = 'select * from test;' runner = CliRunner() result = runner.invoke(cli, args=CLI_ARGS + ['-e', sql] + ['--table']) - expected = '+-----+\n| a |\n|-----|\n| abc |\n+-----+\n' + expected = '+-----+\n| a |\n+-----+\n| abc |\n+-----+\n' assert result.exit_code == 0 assert expected in result.output @@ -98,14 +98,14 @@ def test_batch_mode_table(executor): result = runner.invoke(cli, args=CLI_ARGS + ['-t'], input=sql) expected = (dedent("""\ - +------------+ - | count(*) | - |------------| - | 3 | - +------------+ + +----------+ + | count(*) | + +----------+ + | 3 | + +----------+ +-----+ | a | - |-----| + +-----+ | abc | +-----+""")) diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index d964f43ba..a9b5fbec2 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -15,7 +15,7 @@ def test_conn(executor): assert results == dedent("""\ +-----+ | a | - |-----| + +-----+ | abc | +-----+ 1 row in set""") @@ -26,11 +26,11 @@ def test_bools(executor): run(executor, '''insert into test values(True)''') results = run(executor, '''select * from test''', join=True) assert results == dedent("""\ - +-----+ - | a | - |-----| - | 1 | - +-----+ + +---+ + | a | + +---+ + | 1 | + +---+ 1 row in set""") @dbtest @@ -41,7 +41,7 @@ def test_binary(executor): assert results == dedent("""\ +----------------------------------------------------------------------------------------------+ | geom | - |----------------------------------------------------------------------------------------------| + +----------------------------------------------------------------------------------------------+ | 0x00000000010200000002000000397f130a11185d4034f44f70b1de43400000000000185d40423ee8d9acde4340 | +----------------------------------------------------------------------------------------------+ 1 row in set""") @@ -140,7 +140,7 @@ def test_favorite_query(executor): > select * from test where a like 'a%' +-----+ | a | - |-----| + +-----+ | abc | +-----+""") @@ -163,13 +163,13 @@ def test_favorite_query_multiple_statement(executor): > select * from test where a like 'a%' +-----+ | a | - |-----| + +-----+ | abc | +-----+ > select * from test where a like 'd%' +-----+ | a | - |-----| + +-----+ | def | +-----+""") @@ -261,13 +261,13 @@ def test_favorite_query_multiline_statement(executor): > select * from test where a like 'a%' +-----+ | a | - |-----| + +-----+ | abc | +-----+ > select * from test where a like 'd%' +-----+ | a | - |-----| + +-----+ | def | +-----+""") @@ -282,7 +282,7 @@ def test_timestamp_null(executor): assert results == dedent("""\ +---------------------+ | a | - |---------------------| + +---------------------+ | 0000-00-00 00:00:00 | +---------------------+ 1 row in set""") @@ -295,7 +295,7 @@ def test_datetime_null(executor): assert results == dedent("""\ +---------------------+ | a | - |---------------------| + +---------------------+ | 0000-00-00 00:00:00 | +---------------------+ 1 row in set""") @@ -308,7 +308,7 @@ def test_date_null(executor): assert results == dedent("""\ +------------+ | a | - |------------| + +------------+ | 0000-00-00 | +------------+ 1 row in set""") @@ -321,7 +321,7 @@ def test_time_null(executor): assert results == dedent("""\ +----------+ | a | - |----------| + +----------+ | 00:00:00 | +----------+ 1 row in set""") From 145d5d2af0e5bf15fdaa3ffbf443eb2942244691 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 15:17:34 -0500 Subject: [PATCH 144/824] Add a multi-column test for align_decimals. --- mycli/output_formatter.py | 28 ++++++++++------------------ tests/test_output_formatter.py | 19 +++++++++++++++---- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5e3305318..068f0246b 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -67,23 +67,19 @@ def align_decimals(data, headers, **_): """ pointpos = len(data[0]) * [0] for row in data: - i = 0 - for v in row: + for i, v in enumerate(row): if isinstance(v, Decimal): - v = str(v) + v = encodingutils.text_type(v) pointpos[i] = max(intlen(v), pointpos[i]) - i += 1 results = [] for row in data: - i = 0 result = [] - for v in row: + for i, v in enumerate(row): if isinstance(v, Decimal): - v = str(v) - result.append((pointpos[i]-intlen(v))*" "+v) + v = encodingutils.text_type(v) + result.append((pointpos[i] - intlen(v)) * " " + v) else: result.append(v) - i += 1 results.append(result) return results, headers @@ -101,26 +97,22 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): ghi jkl """ - """Convert all *data* and *headers* to strings.""" - quote = len(data[0])*[False] + quote = len(data[0]) * [False] for row in data: - i = 0 - for v in row: + for i, v in enumerate(row): v = encodingutils.text_type(v) if v[0] == ' ' or v[-1] == ' ': quote[i] = True - i += 1 results = [] for row in data: result = [] - i = 0 - for v in row: + for i, v in enumerate(row): if quote[i]: - result.append('{quotestyle}{value}{quotestyle}'.format(quotestyle=quotestyle, value=v)) + result.append('{quotestyle}{value}{quotestyle}'.format( + quotestyle=quotestyle, value=v)) else: result.append(v) - i += 1 results.append(result) return results, headers diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 66ac42375..045bd730e 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -3,12 +3,14 @@ from __future__ import unicode_literals +from decimal import Decimal from textwrap import dedent -from mycli.output_formatter import (bytes_to_string, convert_to_string, - csv_wrapper, OutputFormatter, - override_missing_value, tabulate_wrapper, - terminal_tables_wrapper, to_string) +from mycli.output_formatter import (align_decimals, bytes_to_string, + convert_to_string, csv_wrapper, + OutputFormatter, override_missing_value, + tabulate_wrapper, terminal_tables_wrapper, + to_string) def test_to_string(): @@ -47,6 +49,15 @@ def test_bytes_to_string(): assert expected == bytes_to_string(data, headers) +def test_align_decimals(): + """Test the *output_formatter.align_decimals()* function.""" + data = [[Decimal('200'), Decimal('1')], [Decimal('1.00002'), Decimal('1.0')]] + headers = ['num1', 'num2'] + expected = ([['200', '1'], [' 1.00002', '1.0']], ['num1', 'num2']) + + assert expected == align_decimals(data, headers) + + def test_tabulate_wrapper(): """Test the *output_formatter.tabulate_wrapper()* function.""" data = [['abc', 1], ['d', 456]] From 6ea35bad05cee2df19e37fd19301b1129636e264 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 19:19:30 -0500 Subject: [PATCH 145/824] Format/idiomatic changes. --- mycli/output_formatter.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 068f0246b..549ee4e8b 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -101,18 +101,16 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): for row in data: for i, v in enumerate(row): v = encodingutils.text_type(v) - if v[0] == ' ' or v[-1] == ' ': + if v.startswith(' ') or v.endswith(' '): quote[i] = True results = [] for row in data: result = [] for i, v in enumerate(row): - if quote[i]: - result.append('{quotestyle}{value}{quotestyle}'.format( - quotestyle=quotestyle, value=v)) - else: - result.append(v) + quotation = quotestyle if quote[i] else '' + result.append('{quotestyle}{value}{quotestyle}'.format( + quotestyle=quotation, value=v)) results.append(result) return results, headers @@ -192,8 +190,8 @@ def __init__(self, format_name=None): 'preprocessor': (bytes_to_string, override_missing_value, align_decimals), 'table_format': terminal_tables_format, - 'missing_value': '' - }) + 'missing_value': ''} + ) if format_name: self.set_format_name(format_name) @@ -220,10 +218,12 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. - >>> print(OutputFormatter().format_output( \ - [["abc", Decimal(1)], ["defg", Decimal('11.1')], ["hi", Decimal('1.1')]], \ - ["text", "numeric"], \ - "ascii" \ + + >>> print(OutputFormatter().format_output(\ + [["abc", Decimal(1)], ["defg", Decimal('11.1')],\ + ["hi", Decimal('1.1')]],\ + ["text", "numeric"],\ + "ascii"\ )) +------+---------+ | text | numeric | From c51cfac3c1458a07a2f9b8acb59b2ef2f9321fbb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 19:19:53 -0500 Subject: [PATCH 146/824] Move format_output tests to tests module. --- mycli/output_formatter.py | 14 -------------- tests/test_output_formatter.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 549ee4e8b..6656ec8c1 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -218,20 +218,6 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. - - >>> print(OutputFormatter().format_output(\ - [["abc", Decimal(1)], ["defg", Decimal('11.1')],\ - ["hi", Decimal('1.1')]],\ - ["text", "numeric"],\ - "ascii"\ - )) - +------+---------+ - | text | numeric | - +------+---------+ - | abc | 1 | - | defg | 11.1 | - | hi | 1.1 | - +------+---------+ """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 045bd730e..8ddeb888c 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -105,3 +105,21 @@ def test_terminal_tables_wrapper(): | abc | 1 | | d | 456 | +---------+--------+''') + + +def test_output_formatter(): + """Test the *output_formatter.OutputFormatter* class.""" + data = [['abc', Decimal(1)], ['defg', Decimal('11.1')], + ['hi', Decimal('1.1')]] + headers = ['text', 'numeric'] + expected = dedent('''\ + +------+---------+ + | text | numeric | + +------+---------+ + | abc | 1 | + | defg | 11.1 | + | hi | 1.1 | + +------+---------+''') + + assert expected == OutputFormatter().format_output(data, headers, + format_name='ascii') From a36cf85b53f36221032acb6e2ee704e0900d3a0d Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 17:12:48 -0700 Subject: [PATCH 147/824] Refactor OutputFormatter to remove tight coupling. --- mycli/main.py | 7 +- mycli/output_formatter.py | 232 ------------------ mycli/output_formatter/__init__.py | 0 .../delimited_output_adapter.py | 22 ++ mycli/output_formatter/output_formatter.py | 91 +++++++ mycli/output_formatter/preprocessors.py | 98 ++++++++ mycli/output_formatter/tabulate_adapter.py | 14 ++ .../terminaltables_adapter.py | 24 ++ mycli/sqlcompleter.py | 6 +- 9 files changed, 255 insertions(+), 239 deletions(-) delete mode 100644 mycli/output_formatter.py create mode 100644 mycli/output_formatter/__init__.py create mode 100644 mycli/output_formatter/delimited_output_adapter.py create mode 100644 mycli/output_formatter/output_formatter.py create mode 100644 mycli/output_formatter/preprocessors.py create mode 100644 mycli/output_formatter/tabulate_adapter.py create mode 100644 mycli/output_formatter/terminaltables_adapter.py diff --git a/mycli/main.py b/mycli/main.py index faca8b37d..0b948a85e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -37,7 +37,7 @@ from .config import (write_default_config, get_mylogin_cnf_path, open_mylogin_cnf, read_config_files, str_to_bool) from .key_bindings import mycli_bindings -from .output_formatter import OutputFormatter +from .output_formatter import output_formatter from .encodingutils import utf8tounicode from .lexer import MyCliLexer from .__init__ import __version__ @@ -106,7 +106,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.formatter = OutputFormatter(format_name=c['main']['table_format']) + self.formatter = output_formatter.OutputFormatter(format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] @@ -145,7 +145,8 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. self.smart_completion = c['main'].as_bool('smart_completion') - self.completer = SQLCompleter(self.smart_completion) + self.completer = SQLCompleter(self.smart_completion, + supported_formats=self.formatter.supported_formats) self._completer_lock = threading.Lock() # Register custom special commands. diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py deleted file mode 100644 index 6656ec8c1..000000000 --- a/mycli/output_formatter.py +++ /dev/null @@ -1,232 +0,0 @@ -# -*- coding: utf-8 -*- -"""A generic output formatter interface.""" - -from __future__ import unicode_literals - -import contextlib -import csv -from decimal import Decimal -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -from tabulate import tabulate -import terminaltables - -from . import encodingutils -from .packages.expanded import expanded_table - - -def to_string(value): - """Convert *value* to a string.""" - if isinstance(value, encodingutils.binary_type): - return encodingutils.bytes_to_string(value) - else: - return encodingutils.text_type(value) - - -def convert_to_string(data, headers, **_): - """Convert all *data* and *headers* to strings.""" - return ([[to_string(v) for v in row] for row in data], - [to_string(h) for h in headers]) - - -def override_missing_value(data, headers, missing_value='', **_): - """Override missing values in the data with *missing_value*.""" - return ([[missing_value if v is None else v for v in row] for row in data], - headers) - - -def bytes_to_string(data, headers, **_): - """Convert all *data* and *headers* bytes to strings.""" - return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], - [encodingutils.bytes_to_string(h) for h in headers]) - - -def intlen(value): - """Find (character) length - >>> intlen('11.1') - 2 - >>> intlen('11') - 2 - >>> intlen('1.1') - 1 - """ - pos = value.find('.') - if pos < 0: - pos = len(value) - return pos - -def align_decimals(data, headers, **_): - """Align decimals to decimal point - >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) - 1 - 11.1 - 1.1 - """ - pointpos = len(data[0]) * [0] - for row in data: - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - pointpos[i] = max(intlen(v), pointpos[i]) - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - result.append((pointpos[i] - intlen(v)) * " " + v) - else: - result.append(v) - results.append(result) - return results, headers - - -def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote whitespace - >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) - ' before' - 'after ' - ' both ' - 'none' - >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) - abc - def - ghi - jkl - """ - quote = len(data[0]) * [False] - for row in data: - for i, v in enumerate(row): - v = encodingutils.text_type(v) - if v.startswith(' ') or v.endswith(' '): - quote[i] = True - - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - quotation = quotestyle if quote[i] else '' - result.append('{quotestyle}{value}{quotestyle}'.format( - quotestyle=quotation, value=v)) - results.append(result) - return results, headers - - -def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): - """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value, disable_numparse=True) - - -def csv_wrapper(data, headers, delimiter=',', **_): - """Wrap CSV formatting inside a standard function for OutputFormatter.""" - with contextlib.closing(StringIO()) as content: - writer = csv.writer(content, delimiter=str(delimiter)) - - writer.writerow(headers) - for row in data: - writer.writerow(row) - - return content.getvalue() - - -def terminal_tables_wrapper(data, headers, table_format=None, **_): - """Wrap terminaltables inside a standard function for OutputFormatter.""" - if table_format == 'ascii': - table = terminaltables.AsciiTable - elif table_format == 'single': - table = terminaltables.SingleTable - elif table_format == 'double': - table = terminaltables.DoubleTable - elif table_format == 'github': - table = terminaltables.GithubFlavoredMarkdownTable - else: - raise ValueError('unrecognized table format: {}'.format(table_format)) - - t = table([headers] + data) - return t.table - - -class OutputFormatter(object): - """A class with a standard interface for various formatting libraries.""" - - def __init__(self, format_name=None): - """Register the supported output formats.""" - self._output_formats = { - 'csv': (csv_wrapper, { - 'preprocessor': (override_missing_value, bytes_to_string), - 'missing_value': '' - }), - 'tsv': (csv_wrapper, { - 'preprocessor': (override_missing_value, bytes_to_string), - 'missing_value': '', - 'delimiter': '\t' - }), - 'expanded': (expanded_table, { - 'preprocessor': (override_missing_value, convert_to_string), - 'missing_value': '' - }) - } - self._format_name = None - - tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', - 'moinmoin', 'html', 'latex', 'latex_booktabs', - 'textile') - for tabulate_format in tabulate_formats: - self._output_formats[tabulate_format] = (tabulate_wrapper, { - 'preprocessor': (bytes_to_string, align_decimals), - 'table_format': tabulate_format, - 'missing_value': '' - }) - - terminal_tables_formats = ('ascii', 'single', 'double', 'github') - for terminal_tables_format in terminal_tables_formats: - self._output_formats[terminal_tables_format] = ( - terminal_tables_wrapper, { - 'preprocessor': (bytes_to_string, override_missing_value, - align_decimals), - 'table_format': terminal_tables_format, - 'missing_value': ''} - ) - - if format_name: - self.set_format_name(format_name) - - def set_format_name(self, format_name): - """Set the OutputFormatter's default format.""" - if format_name in self.supported_formats(): - self._format_name = format_name - else: - raise ValueError('unrecognized format_name: {}'.format( - format_name)) - - def get_format_name(self): - """Get the OutputFormatter's default format.""" - return self._format_name - - def supported_formats(self): - """Return the supported output format names.""" - return tuple(self._output_formats.keys()) - - def format_output(self, data, headers, format_name=None, **kwargs): - """Format the headers and data using a specific formatter. - - *format_name* must be a formatter available in `supported_formats()`. - - All keyword arguments are passed to the specified formatter. - """ - format_name = format_name or self._format_name - if format_name not in self.supported_formats(): - raise ValueError('unrecognized format: {}'.format(format_name)) - - function, fkwargs = self._output_formats[format_name] - fkwargs.update(kwargs) - preprocessor = fkwargs.get('preprocessor', None) - if preprocessor: - for f in preprocessor: - data, headers = f(data, headers, **fkwargs) - return function(data, headers, **fkwargs) diff --git a/mycli/output_formatter/__init__.py b/mycli/output_formatter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py new file mode 100644 index 000000000..04571e38f --- /dev/null +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -0,0 +1,22 @@ +import contextlib +import csv +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +from .preprocessors import (override_missing_value, bytes_to_string) + +supported_formats = ('csv',) +delimiter_preprocessors = (override_missing_value, bytes_to_string) + +def delimiter_adapter(data, headers, delimiter=',', **_): + """Wrap CSV formatting inside a standard function for OutputFormatter.""" + with contextlib.closing(StringIO()) as content: + writer = csv.writer(content, delimiter=str(delimiter)) + + writer.writerow(headers) + for row in data: + writer.writerow(row) + + return content.getvalue() diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py new file mode 100644 index 000000000..e4b280230 --- /dev/null +++ b/mycli/output_formatter/output_formatter.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from ..packages.expanded import expanded_table +from .preprocessors import (override_missing_value, convert_to_string) +from .delimited_output_adapter import delimiter_adapter, delimiter_preprocessors +from .tabulate_adapter import (tabulate_adapter, + supported_formats as tabulate_formats, + preprocessors as tabulate_preprocessors) +from .terminaltables_adapter import (terminaltables_adapter, + preprocessors as terminaltables_preprocessors, + supported_formats as terminaltables_formats) +from collections import namedtuple + +OutputFormatHandler = namedtuple('OutputFormatHandler', + 'format_name preprocessors formatter formatter_args') + +"""A generic output formatter interface.""" +class OutputFormatter(object): + """A class with a standard interface for various formatting libraries.""" + + _output_formats = {} + + def __init__(self, format_name=None): + """Register the supported output formats.""" + self._format_name = format_name + + def set_format_name(self, format_name): + """Set the OutputFormatter's default format.""" + if format_name in self.supported_formats(): + self._format_name = format_name + else: + raise ValueError('unrecognized format_name: {}'.format( + format_name)) + + def get_format_name(self): + """Get the OutputFormatter's default format.""" + return self._format_name + + def supported_formats(self): + """Return the supported output format names.""" + return tuple(self._output_formats.keys()) + + @classmethod + def register_new_formatter(cls, format_name, handler, preprocessors=None, + kwargs=None): + """Register a new fomatter to format the output""" + cls._output_formats[format_name] = OutputFormatHandler(format_name, + preprocessors, handler, kwargs) + + def format_output(self, data, headers, format_name=None, **kwargs): + """Format the headers and data using a specific formatter. + + *format_name* must be a formatter available in `supported_formats()`. + + All keyword arguments are passed to the specified formatter. + """ + format_name = format_name or self._format_name + if format_name not in self.supported_formats(): + raise ValueError('unrecognized format: {}'.format(format_name)) + + (_, preprocessors, formatter, fkwargs) = self._output_formats[format_name] + fkwargs.update(kwargs) + if preprocessors: + for f in preprocessors: + data, headers = f(data, headers, **fkwargs) + return formatter(data, headers, **fkwargs) + +OutputFormatter.register_new_formatter('csv', + delimiter_adapter, + delimiter_preprocessors, {'missing_value': ''}) +OutputFormatter.register_new_formatter('tsv', + delimiter_adapter, + delimiter_preprocessors, + {'missing_value': '', 'delimiter': '\t'}) +OutputFormatter.register_new_formatter('expanded', + expanded_table, + (override_missing_value, convert_to_string), + {'missing_value': '', 'delimiter': '\t'}) + +for tabulate_format in tabulate_formats: + OutputFormatter.register_new_formatter(tabulate_format, + tabulate_adapter, + tabulate_preprocessors, + {'table_format': tabulate_format, 'missing_value': ''}) + +for terminaltables_format in terminaltables_formats: + OutputFormatter.register_new_formatter(terminaltables_format, + terminaltables_adapter, + terminaltables_preprocessors, + {'table_format': terminaltables_format, 'missing_value': ''}) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py new file mode 100644 index 000000000..b3a9373af --- /dev/null +++ b/mycli/output_formatter/preprocessors.py @@ -0,0 +1,98 @@ +from decimal import Decimal +from .. import encodingutils + +def to_string(value): + """Convert *value* to a string.""" + if isinstance(value, encodingutils.binary_type): + return encodingutils.bytes_to_string(value) + else: + return encodingutils.text_type(value) + + +def convert_to_string(data, headers, **_): + """Convert all *data* and *headers* to strings.""" + return ([[to_string(v) for v in row] for row in data], + [to_string(h) for h in headers]) + + +def override_missing_value(data, headers, missing_value='', **_): + """Override missing values in the data with *missing_value*.""" + return ([[missing_value if v is None else v for v in row] for row in data], + headers) + + +def bytes_to_string(data, headers, **_): + """Convert all *data* and *headers* bytes to strings.""" + return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], + [encodingutils.bytes_to_string(h) for h in headers]) + + +def intlen(value): + """Find (character) length + >>> intlen('11.1') + 2 + >>> intlen('11') + 2 + >>> intlen('1.1') + 1 + """ + pos = value.find('.') + if pos < 0: + pos = len(value) + return pos + +def align_decimals(data, headers, **_): + """Align decimals to decimal point + >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) + 1 + 11.1 + 1.1 + """ + pointpos = len(data[0]) * [0] + for row in data: + for i, v in enumerate(row): + if isinstance(v, Decimal): + v = encodingutils.text_type(v) + pointpos[i] = max(intlen(v), pointpos[i]) + results = [] + for row in data: + result = [] + for i, v in enumerate(row): + if isinstance(v, Decimal): + v = encodingutils.text_type(v) + result.append((pointpos[i] - intlen(v)) * " " + v) + else: + result.append(v) + results.append(result) + return results, headers + + +def quote_whitespaces(data, headers, quotestyle="'", **_): + """Quote whitespace + >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) + ' before' + 'after ' + ' both ' + 'none' + >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) + abc + def + ghi + jkl + """ + quote = len(data[0]) * [False] + for row in data: + for i, v in enumerate(row): + v = encodingutils.text_type(v) + if v.startswith(' ') or v.endswith(' '): + quote[i] = True + + results = [] + for row in data: + result = [] + for i, v in enumerate(row): + quotation = quotestyle if quote[i] else '' + result.append('{quotestyle}{value}{quotestyle}'.format( + quotestyle=quotation, value=v)) + results.append(result) + return results, headers diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py new file mode 100644 index 000000000..8c4599c12 --- /dev/null +++ b/mycli/output_formatter/tabulate_adapter.py @@ -0,0 +1,14 @@ +from tabulate import tabulate +from .preprocessors import (bytes_to_string, align_decimals) + +supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', + 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', + 'moinmoin', 'html', 'latex', 'latex_booktabs', + 'textile') + +preprocessors = (bytes_to_string, align_decimals) + +def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): + """Wrap tabulate inside a standard function for OutputFormatter.""" + return tabulate(data, headers, tablefmt=table_format, + missingval=missing_value, disable_numparse=True) diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py new file mode 100644 index 000000000..159810783 --- /dev/null +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -0,0 +1,24 @@ +import terminaltables +from .preprocessors import (bytes_to_string, align_decimals, + override_missing_value) + +supported_formats = ('ascii', 'single', 'double', 'github') +preprocessors = (bytes_to_string, override_missing_value, align_decimals) + +def terminaltables_adapter(data, headers, table_format=None, **_): + """Wrap terminaltables inside a standard function for OutputFormatter.""" + + table_format_handler = { + 'ascii': terminaltables.AsciiTable, + 'single': terminaltables.SingleTable, + 'double': terminaltables.DoubleTable, + 'github': terminaltables.GithubFlavoredMarkdownTable, + } + + try: + table = table_format_handler[table_format] + except KeyError: + raise ValueError('unrecognized table format: {}'.format(table_format)) + + t = table([headers] + data) + return t.table diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 656e5d531..f01afb1da 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -6,7 +6,6 @@ from prompt_toolkit.completion import Completer, Completion -from .output_formatter import OutputFormatter from .packages.completion_engine import suggest_type from .packages.parseutils import last_word from .packages.special.favoritequeries import favoritequeries @@ -50,7 +49,7 @@ class SQLCompleter(Completer): users = [] - def __init__(self, smart_completion=True): + def __init__(self, smart_completion=True, supported_formats=()): super(self.__class__, self).__init__() self.smart_completion = smart_completion self.reserved_words = set() @@ -59,8 +58,7 @@ def __init__(self, smart_completion=True): self.name_pattern = compile("^[_a-z][_a-z0-9\$]*$") self.special_commands = [] - formatter = OutputFormatter() - self.table_formats = formatter.supported_formats() + self.table_formats = supported_formats self.reset_completions() def escape_name(self, name): From 7a81b9e6bc173fed75fd2653a67f606942bd04d6 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 19:52:35 -0700 Subject: [PATCH 148/824] Update the tests. --- tests/test_output_formatter.py | 18 ++++++++++++------ tests/utils.py | 1 - 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 8ddeb888c..484f52d81 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -6,12 +6,18 @@ from decimal import Decimal from textwrap import dedent -from mycli.output_formatter import (align_decimals, bytes_to_string, - convert_to_string, csv_wrapper, - OutputFormatter, override_missing_value, - tabulate_wrapper, terminal_tables_wrapper, - to_string) - +from mycli.output_formatter.preprocessors import (align_decimals, + bytes_to_string, + convert_to_string, + override_missing_value, + to_string) +from mycli.output_formatter.output_formatter import OutputFormatter +from mycli.output_formatter.delimited_output_adapter import (delimiter_adapter + as csv_wrapper) +from mycli.output_formatter.tabulate_adapter import (tabulate_adapter as + tabulate_wrapper) +from mycli.output_formatter.terminaltables_adapter import \ + (terminaltables_adapter as terminal_tables_wrapper) def test_to_string(): """Test the *output_formatter.to_string()* function.""" diff --git a/tests/utils.py b/tests/utils.py index 64006324f..bff31045d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,7 +4,6 @@ import pytest from mycli.main import MyCli, special -from mycli.output_formatter import OutputFormatter PASSWORD = getenv('PYTEST_PASSWORD') USER = getenv('PYTEST_USER', 'root') From a96867c6fa06049d34d309927535de2851ea18da Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 3 Apr 2017 23:01:12 -0500 Subject: [PATCH 149/824] PEP8 edits. --- .../delimited_output_adapter.py | 3 +- mycli/output_formatter/output_formatter.py | 74 ++++++++++--------- mycli/output_formatter/preprocessors.py | 5 +- mycli/output_formatter/tabulate_adapter.py | 11 +-- .../terminaltables_adapter.py | 14 ++-- tests/test_output_formatter.py | 22 +++--- 6 files changed, 71 insertions(+), 58 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 04571e38f..04e9f252a 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -5,11 +5,12 @@ except ImportError: from io import StringIO -from .preprocessors import (override_missing_value, bytes_to_string) +from .preprocessors import override_missing_value, bytes_to_string supported_formats = ('csv',) delimiter_preprocessors = (override_missing_value, bytes_to_string) + def delimiter_adapter(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index e4b280230..8f698fc0b 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -1,21 +1,25 @@ # -*- coding: utf-8 -*- +"""A generic output formatter interface.""" + from __future__ import unicode_literals +from collections import namedtuple -from ..packages.expanded import expanded_table +from mycli.packages.expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) -from .delimited_output_adapter import delimiter_adapter, delimiter_preprocessors +from .delimited_output_adapter import (delimiter_adapter, + delimiter_preprocessors) from .tabulate_adapter import (tabulate_adapter, - supported_formats as tabulate_formats, - preprocessors as tabulate_preprocessors) -from .terminaltables_adapter import (terminaltables_adapter, - preprocessors as terminaltables_preprocessors, - supported_formats as terminaltables_formats) -from collections import namedtuple + supported_formats as tabulate_formats, + preprocessors as tabulate_preprocessors) +from .terminaltables_adapter import ( + terminaltables_adapter, preprocessors as terminaltables_preprocessors, + supported_formats as terminaltables_formats) + +OutputFormatHandler = namedtuple( + 'OutputFormatHandler', + 'format_name preprocessors formatter formatter_args') -OutputFormatHandler = namedtuple('OutputFormatHandler', - 'format_name preprocessors formatter formatter_args') -"""A generic output formatter interface.""" class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" @@ -43,10 +47,10 @@ def supported_formats(self): @classmethod def register_new_formatter(cls, format_name, handler, preprocessors=None, - kwargs=None): - """Register a new fomatter to format the output""" - cls._output_formats[format_name] = OutputFormatHandler(format_name, - preprocessors, handler, kwargs) + kwargs=None): + """Register a new formatter to format the output.""" + cls._output_formats[format_name] = OutputFormatHandler( + format_name, preprocessors, handler, kwargs) def format_output(self, data, headers, format_name=None, **kwargs): """Format the headers and data using a specific formatter. @@ -66,26 +70,28 @@ def format_output(self, data, headers, format_name=None, **kwargs): data, headers = f(data, headers, **fkwargs) return formatter(data, headers, **fkwargs) -OutputFormatter.register_new_formatter('csv', - delimiter_adapter, - delimiter_preprocessors, {'missing_value': ''}) -OutputFormatter.register_new_formatter('tsv', - delimiter_adapter, - delimiter_preprocessors, - {'missing_value': '', 'delimiter': '\t'}) -OutputFormatter.register_new_formatter('expanded', - expanded_table, - (override_missing_value, convert_to_string), - {'missing_value': '', 'delimiter': '\t'}) + +OutputFormatter.register_new_formatter('csv', delimiter_adapter, + delimiter_preprocessors, + {'missing_value': ''}) +OutputFormatter.register_new_formatter('tsv', delimiter_adapter, + delimiter_preprocessors, + {'missing_value': '', + 'delimiter': '\t'}) +OutputFormatter.register_new_formatter('expanded', expanded_table, + (override_missing_value, + convert_to_string), + {'missing_value': '', + 'delimiter': '\t'}) for tabulate_format in tabulate_formats: - OutputFormatter.register_new_formatter(tabulate_format, - tabulate_adapter, - tabulate_preprocessors, - {'table_format': tabulate_format, 'missing_value': ''}) + OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, + tabulate_preprocessors, + {'table_format': tabulate_format, + 'missing_value': ''}) for terminaltables_format in terminaltables_formats: - OutputFormatter.register_new_formatter(terminaltables_format, - terminaltables_adapter, - terminaltables_preprocessors, - {'table_format': terminaltables_format, 'missing_value': ''}) + OutputFormatter.register_new_formatter( + terminaltables_format, terminaltables_adapter, + terminaltables_preprocessors, + {'table_format': terminaltables_format, 'missing_value': ''}) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index b3a9373af..d6db3fd44 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -1,5 +1,7 @@ from decimal import Decimal -from .. import encodingutils + +from mycli import encodingutils + def to_string(value): """Convert *value* to a string.""" @@ -41,6 +43,7 @@ def intlen(value): pos = len(value) return pos + def align_decimals(data, headers, **_): """Align decimals to decimal point >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 8c4599c12..0db31ff9f 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,13 +1,14 @@ from tabulate import tabulate -from .preprocessors import (bytes_to_string, align_decimals) -supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', - 'moinmoin', 'html', 'latex', 'latex_booktabs', - 'textile') +from .preprocessors import bytes_to_string, align_decimals + +supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', + 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', + 'html', 'latex', 'latex_booktabs', 'textile') preprocessors = (bytes_to_string, align_decimals) + def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py index 159810783..ac580517f 100644 --- a/mycli/output_formatter/terminaltables_adapter.py +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -1,19 +1,21 @@ import terminaltables + from .preprocessors import (bytes_to_string, align_decimals, - override_missing_value) + override_missing_value) supported_formats = ('ascii', 'single', 'double', 'github') preprocessors = (bytes_to_string, override_missing_value, align_decimals) + def terminaltables_adapter(data, headers, table_format=None, **_): """Wrap terminaltables inside a standard function for OutputFormatter.""" table_format_handler = { - 'ascii': terminaltables.AsciiTable, - 'single': terminaltables.SingleTable, - 'double': terminaltables.DoubleTable, - 'github': terminaltables.GithubFlavoredMarkdownTable, - } + 'ascii': terminaltables.AsciiTable, + 'single': terminaltables.SingleTable, + 'double': terminaltables.DoubleTable, + 'github': terminaltables.GithubFlavoredMarkdownTable, + } try: table = table_format_handler[table_format] diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 484f52d81..7c0890fde 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -2,22 +2,22 @@ """Test the generic output formatter interface.""" from __future__ import unicode_literals - from decimal import Decimal from textwrap import dedent from mycli.output_formatter.preprocessors import (align_decimals, - bytes_to_string, - convert_to_string, - override_missing_value, - to_string) + bytes_to_string, + convert_to_string, + override_missing_value, + to_string) from mycli.output_formatter.output_formatter import OutputFormatter -from mycli.output_formatter.delimited_output_adapter import (delimiter_adapter - as csv_wrapper) -from mycli.output_formatter.tabulate_adapter import (tabulate_adapter as - tabulate_wrapper) -from mycli.output_formatter.terminaltables_adapter import \ - (terminaltables_adapter as terminal_tables_wrapper) +from mycli.output_formatter.delimited_output_adapter import ( + delimiter_adapter as csv_wrapper) +from mycli.output_formatter.tabulate_adapter import ( + tabulate_adapter as tabulate_wrapper) +from mycli.output_formatter.terminaltables_adapter import ( + terminaltables_adapter as terminal_tables_wrapper) + def test_to_string(): """Test the *output_formatter.to_string()* function.""" From 3d9feb9241787eeecad974f7a5c1e5f9429ef604 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 21:14:52 -0700 Subject: [PATCH 150/824] Move expanded.py into output_formatter package. --- mycli/output_formatter/delimited_output_adapter.py | 2 +- mycli/{packages => output_formatter}/expanded.py | 0 mycli/output_formatter/output_formatter.py | 5 ++--- tests/test_expanded.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) rename mycli/{packages => output_formatter}/expanded.py (100%) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 04e9f252a..7111af6d3 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -7,7 +7,7 @@ from .preprocessors import override_missing_value, bytes_to_string -supported_formats = ('csv',) +supported_formats = ('csv', 'tsv') delimiter_preprocessors = (override_missing_value, bytes_to_string) diff --git a/mycli/packages/expanded.py b/mycli/output_formatter/expanded.py similarity index 100% rename from mycli/packages/expanded.py rename to mycli/output_formatter/expanded.py diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 8f698fc0b..1564026b3 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals from collections import namedtuple -from mycli.packages.expanded import expanded_table +from .expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) from .delimited_output_adapter import (delimiter_adapter, delimiter_preprocessors) @@ -81,8 +81,7 @@ def format_output(self, data, headers, format_name=None, **kwargs): OutputFormatter.register_new_formatter('expanded', expanded_table, (override_missing_value, convert_to_string), - {'missing_value': '', - 'delimiter': '\t'}) + {'missing_value': ''}) for tabulate_format in tabulate_formats: OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, diff --git a/tests/test_expanded.py b/tests/test_expanded.py index db503cba8..7233e91ce 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -1,7 +1,7 @@ """Test the vertical, expanded table formatter.""" from textwrap import dedent -from mycli.packages.expanded import expanded_table +from mycli.output_formatter.expanded import expanded_table from mycli.encodingutils import text_type From b5bda3f20f2cea84bc72aa6a28cbf1f31cfa67f7 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 21:26:33 -0700 Subject: [PATCH 151/824] Refactor the delimiter adapter to include tsv. --- mycli/output_formatter/delimited_output_adapter.py | 9 +++++++-- mycli/output_formatter/output_formatter.py | 14 +++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 7111af6d3..d3b5120b9 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -11,10 +11,15 @@ delimiter_preprocessors = (override_missing_value, bytes_to_string) -def delimiter_adapter(data, headers, delimiter=',', **_): +def delimiter_adapter(data, headers, table_format=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: - writer = csv.writer(content, delimiter=str(delimiter)) + if table_format == 'csv': + writer = csv.writer(content, delimiter=',') + elif table_format == 'tsv': + writer = csv.writer(content, delimiter='\t') + else: + raise ValueError('Invalid table_format specified.') writer.writerow(headers) for row in data: diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 1564026b3..a47029cca 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -7,6 +7,7 @@ from .expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) from .delimited_output_adapter import (delimiter_adapter, + supported_formats as delimiter_formats, delimiter_preprocessors) from .tabulate_adapter import (tabulate_adapter, supported_formats as tabulate_formats, @@ -71,18 +72,17 @@ def format_output(self, data, headers, format_name=None, **kwargs): return formatter(data, headers, **fkwargs) -OutputFormatter.register_new_formatter('csv', delimiter_adapter, - delimiter_preprocessors, - {'missing_value': ''}) -OutputFormatter.register_new_formatter('tsv', delimiter_adapter, - delimiter_preprocessors, - {'missing_value': '', - 'delimiter': '\t'}) OutputFormatter.register_new_formatter('expanded', expanded_table, (override_missing_value, convert_to_string), {'missing_value': ''}) +for delimiter_format in delimiter_formats: + OutputFormatter.register_new_formatter(delimiter_format, delimiter_adapter, + delimiter_preprocessors, + {'table_format': delimiter_format, + 'missing_value': ''}) + for tabulate_format in tabulate_formats: OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, tabulate_preprocessors, From 2120b3653e3f8027ebfa16703737b65aa7d31f81 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 21:33:41 -0700 Subject: [PATCH 152/824] Fix failing tests. --- mycli/output_formatter/delimited_output_adapter.py | 2 +- tests/test_output_formatter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index d3b5120b9..8036a5ff7 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -11,7 +11,7 @@ delimiter_preprocessors = (override_missing_value, bytes_to_string) -def delimiter_adapter(data, headers, table_format=',', **_): +def delimiter_adapter(data, headers, table_format='csv', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: if table_format == 'csv': diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 7c0890fde..7949032fd 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -92,7 +92,7 @@ def test_csv_wrapper(): # Test tab-delimited output. data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = csv_wrapper(data, headers, delimiter='\t') + output = csv_wrapper(data, headers, table_format='tsv') assert output == dedent('''\ letters\tnumber\r\n\ abc\t1\r\n\ From 98f827a7bba38d7e31692d0f435e158668f7c7b2 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 4 Apr 2017 20:22:21 +0200 Subject: [PATCH 153/824] --user-config-file parameter to only read user config file from the given file --- mycli/main.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index ba3443955..ca3cffc6f 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -92,12 +92,12 @@ class MyCli(object): ] default_config_file = os.path.join(PACKAGE_ROOT, 'myclirc') - user_config_file = '~/.myclirc' def __init__(self, sqlexecute=None, prompt=None, logfile=None, defaults_suffix=None, defaults_file=None, - login_path=None, auto_vertical_output=False, warn=None): + login_path=None, auto_vertical_output=False, warn=None, + myclirc="~/.myclirc"): self.sqlexecute = sqlexecute self.logfile = logfile self.defaults_suffix = defaults_suffix @@ -112,7 +112,7 @@ def __init__(self, sqlexecute=None, prompt=None, # Load config. config_files = ([self.default_config_file] + self.system_config_files + - [self.user_config_file]) + [myclirc]) c = self.config = read_config_files(config_files) self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] @@ -132,7 +132,7 @@ def __init__(self, sqlexecute=None, prompt=None, # Write user config if system config wasn't the last config loaded. if c.filename not in self.system_config_files: - write_default_config(self.default_config_file, self.user_config_file) + write_default_config(self.default_config_file, myclirc) # audit log if self.logfile is None and 'audit_log' in c['main']: @@ -759,6 +759,8 @@ def run_query(self, query, table_format=None, new_line=True): help='Read config group with the specified suffix.') @click.option('--defaults-file', type=click.Path(), help='Only read default options from the given file') +@click.option('--myclirc', type=click.Path(), default="~/.myclirc", + help='Location of myclirc file.') @click.option('--auto-vertical-output', is_flag=True, help='Automatically switch to vertical output mode if the result is wider than the terminal width.') @click.option('-t', '--table', is_flag=True, @@ -778,7 +780,7 @@ def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, login_path, auto_vertical_output, local_infile, ssl_ca, ssl_capath, ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, csv, - warn, execute): + warn, execute, myclirc): if version: print('Version:', __version__) @@ -787,7 +789,8 @@ def cli(database, user, host, port, socket, password, dbname, mycli = MyCli(prompt=prompt, logfile=logfile, defaults_suffix=defaults_group_suffix, defaults_file=defaults_file, login_path=login_path, - auto_vertical_output=auto_vertical_output, warn=warn) + auto_vertical_output=auto_vertical_output, warn=warn, + myclirc=myclirc) # Choose which ever one has a valid value. database = database or dbname From 252a257ace53f12e7631794e959f3ccac66524c1 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 20 Mar 2017 16:33:59 +0100 Subject: [PATCH 154/824] copy from pgcli 56af64585f4f033c25cd574072185c85c26e52a2 --- tests/behave.ini | 5 + tests/features/basic_commands.feature | 19 ++++ tests/features/crud_database.feature | 20 ++++ tests/features/crud_table.feature | 22 ++++ tests/features/db_utils.py | 82 ++++++++++++++ tests/features/environment.py | 96 ++++++++++++++++ tests/features/fixture_data/help.txt | 24 ++++ tests/features/fixture_data/help_commands.txt | 21 ++++ tests/features/fixture_utils.py | 34 ++++++ tests/features/iocommands.feature | 10 ++ tests/features/named_queries.feature | 12 ++ tests/features/specials.feature | 9 ++ tests/features/steps/basic_commands.py | 47 ++++++++ tests/features/steps/crud_database.py | 97 ++++++++++++++++ tests/features/steps/crud_table.py | 107 ++++++++++++++++++ tests/features/steps/iocommands.py | 44 +++++++ tests/features/steps/named_queries.py | 59 ++++++++++ tests/features/steps/specials.py | 26 +++++ tests/features/steps/wrappers.py | 15 +++ 19 files changed, 749 insertions(+) create mode 100644 tests/behave.ini create mode 100644 tests/features/basic_commands.feature create mode 100644 tests/features/crud_database.feature create mode 100644 tests/features/crud_table.feature create mode 100644 tests/features/db_utils.py create mode 100644 tests/features/environment.py create mode 100644 tests/features/fixture_data/help.txt create mode 100644 tests/features/fixture_data/help_commands.txt create mode 100644 tests/features/fixture_utils.py create mode 100644 tests/features/iocommands.feature create mode 100644 tests/features/named_queries.feature create mode 100644 tests/features/specials.feature create mode 100644 tests/features/steps/basic_commands.py create mode 100644 tests/features/steps/crud_database.py create mode 100644 tests/features/steps/crud_table.py create mode 100644 tests/features/steps/iocommands.py create mode 100644 tests/features/steps/named_queries.py create mode 100644 tests/features/steps/specials.py create mode 100644 tests/features/steps/wrappers.py diff --git a/tests/behave.ini b/tests/behave.ini new file mode 100644 index 000000000..c555d3c07 --- /dev/null +++ b/tests/behave.ini @@ -0,0 +1,5 @@ +[behave.userdata] +pg_test_user = postgres +pg_test_pass = +pg_test_host = localhost +pg_test_db = pgcli_behave_tests diff --git a/tests/features/basic_commands.feature b/tests/features/basic_commands.feature new file mode 100644 index 000000000..227fe769b --- /dev/null +++ b/tests/features/basic_commands.feature @@ -0,0 +1,19 @@ +Feature: run the cli, + call the help command, + exit the cli + + Scenario: run the cli + When we run dbcli + then we see dbcli prompt + + Scenario: run "\?" command + When we run dbcli + and we wait for prompt + and we send "\?" command + then we see help output + + Scenario: run the cli and exit + When we run dbcli + and we wait for prompt + and we send "ctrl + d" + then dbcli exits diff --git a/tests/features/crud_database.feature b/tests/features/crud_database.feature new file mode 100644 index 000000000..c72468c30 --- /dev/null +++ b/tests/features/crud_database.feature @@ -0,0 +1,20 @@ +Feature: manipulate databases: + create, drop, connect, disconnect + + Scenario: create and drop temporary database + When we run dbcli + and we wait for prompt + and we create database + then we see database created + when we drop database + then we see database dropped + when we connect to dbserver + then we see database connected + + Scenario: connect and disconnect from test database + When we run dbcli + and we wait for prompt + and we connect to test database + then we see database connected + when we connect to dbserver + then we see database connected diff --git a/tests/features/crud_table.feature b/tests/features/crud_table.feature new file mode 100644 index 000000000..d2209fd0e --- /dev/null +++ b/tests/features/crud_table.feature @@ -0,0 +1,22 @@ +Feature: manipulate tables: + create, insert, update, select, delete from, drop + + Scenario: create, insert, select from, update, drop table + When we run dbcli + and we wait for prompt + and we connect to test database + then we see database connected + when we create table + then we see table created + when we insert into table + then we see record inserted + when we update table + then we see record updated + when we select from table + then we see data selected + when we delete from table + then we see record deleted + when we drop table + then we see table dropped + when we connect to dbserver + then we see database connected diff --git a/tests/features/db_utils.py b/tests/features/db_utils.py new file mode 100644 index 000000000..63c6586b7 --- /dev/null +++ b/tests/features/db_utils.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from __future__ import print_function + +from psycopg2 import connect +from psycopg2.extensions import AsIs + + +def create_db(hostname='localhost', username=None, password=None, + dbname=None): + """ + Create test database. + :param hostname: string + :param username: string + :param password: string + :param dbname: string + :return: + """ + cn = create_cn(hostname, password, username, 'postgres') + + # ISOLATION_LEVEL_AUTOCOMMIT = 0 + # Needed for DB creation. + cn.set_isolation_level(0) + + with cn.cursor() as cr: + cr.execute('drop database if exists %s', (AsIs(dbname),)) + cr.execute('create database %s', (AsIs(dbname),)) + + cn.close() + + cn = create_cn(hostname, password, username, dbname) + return cn + + +def create_cn(hostname, password, username, dbname): + """ + Open connection to database. + :param hostname: + :param password: + :param username: + :param dbname: string + :return: psycopg2.connection + """ + if password: + cn = connect(host=hostname, user=username, database=dbname, + password=password) + else: + cn = connect(user=username, database=dbname) + + print('Created connection: {0}.'.format(cn.dsn)) + return cn + + +def drop_db(hostname='localhost', username=None, password=None, + dbname=None): + """ + Drop database. + :param hostname: string + :param username: string + :param password: string + :param dbname: string + """ + cn = create_cn(hostname, password, username, 'postgres') + + # ISOLATION_LEVEL_AUTOCOMMIT = 0 + # Needed for DB drop. + cn.set_isolation_level(0) + + with cn.cursor() as cr: + cr.execute('drop database if exists %s', (AsIs(dbname),)) + + close_cn(cn) + + +def close_cn(cn=None): + """ + Close connection. + :param connection: psycopg2.connection + """ + if cn: + cn.close() + print('Closed connection: {0}.'.format(cn.dsn)) diff --git a/tests/features/environment.py b/tests/features/environment.py new file mode 100644 index 000000000..51ae86b5a --- /dev/null +++ b/tests/features/environment.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from __future__ import print_function + +import os +import sys +import db_utils as dbutils +import fixture_utils as fixutils + + +def before_all(context): + """ + Set env parameters. + """ + os.environ['LINES'] = "100" + os.environ['COLUMNS'] = "100" + os.environ['PAGER'] = 'cat' + os.environ['EDITOR'] = 'ex' + os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../.coveragerc" + + context.exit_sent = False + + vi = '_'.join([str(x) for x in sys.version_info[:3]]) + db_name = context.config.userdata.get('pg_test_db', None) + db_name_full = '{0}_{1}'.format(db_name, vi) + + # Store get params from config. + context.conf = { + 'host': context.config.userdata.get('pg_test_host', 'localhost'), + 'user': context.config.userdata.get('pg_test_user', 'postgres'), + 'pass': context.config.userdata.get('pg_test_pass', None), + 'dbname': db_name_full, + 'dbname_tmp': db_name_full + '_tmp', + 'vi': vi, + 'cli_command': context.config.userdata.get('pg_cli_command', None) or + sys.executable + + ' -c "import coverage; coverage.process_startup(); import pgcli.main; pgcli.main.cli()"' + } + + # Store old env vars. + context.pgenv = { + 'PGDATABASE': os.environ.get('PGDATABASE', None), + 'PGUSER': os.environ.get('PGUSER', None), + 'PGHOST': os.environ.get('PGHOST', None), + 'PGPASSWORD': os.environ.get('PGPASSWORD', None), + } + + # Set new env vars. + os.environ['PGDATABASE'] = context.conf['dbname'] + os.environ['PGUSER'] = context.conf['user'] + os.environ['PGHOST'] = context.conf['host'] + + if context.conf['pass']: + os.environ['PGPASSWORD'] = context.conf['pass'] + else: + if 'PGPASSWORD' in os.environ: + del os.environ['PGPASSWORD'] + if 'PGHOST' in os.environ: + del os.environ['PGHOST'] + + context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], + context.conf['pass'], + context.conf['dbname']) + + context.fixture_data = fixutils.read_fixture_files() + + +def after_all(context): + """ + Unset env parameters. + """ + dbutils.close_cn(context.cn) + dbutils.drop_db(context.conf['host'], context.conf['user'], + context.conf['pass'], context.conf['dbname']) + + # Restore env vars. + for k, v in context.pgenv.items(): + if k in os.environ and v is None: + del os.environ[k] + elif v: + os.environ[k] = v + + +def after_scenario(context, _): + """ + Cleans up after each test complete. + """ + + if hasattr(context, 'cli') and not context.exit_sent: + # Terminate nicely. + context.cli.terminate() + +# TODO: uncomment to debug a failure +# def after_step(context, step): +# if step.status == "failed": +# import ipdb; ipdb.set_trace() diff --git a/tests/features/fixture_data/help.txt b/tests/features/fixture_data/help.txt new file mode 100644 index 000000000..deb499a4e --- /dev/null +++ b/tests/features/fixture_data/help.txt @@ -0,0 +1,24 @@ ++--------------------------+-----------------------------------------------+ +| Command | Description | +|--------------------------+-----------------------------------------------| +| \# | Refresh auto-completions. | +| \? | Show Help. | +| \c[onnect] database_name | Change to a new database. | +| \d [pattern] | List or describe tables, views and sequences. | +| \dT[S+] [pattern] | List data types | +| \df[+] [pattern] | List functions. | +| \di[+] [pattern] | List indexes. | +| \dn[+] [pattern] | List schemas. | +| \ds[+] [pattern] | List sequences. | +| \dt[+] [pattern] | List tables. | +| \du[+] [pattern] | List roles. | +| \dv[+] [pattern] | List views. | +| \e [file] | Edit the query with external editor. | +| \l | List databases. | +| \n[+] [name] | List or execute named queries. | +| \nd [name [query]] | Delete a named query. | +| \ns name query | Save a named query. | +| \refresh | Refresh auto-completions. | +| \timing | Toggle timing of commands. | +| \x | Toggle expanded output. | ++--------------------------+-----------------------------------------------+ diff --git a/tests/features/fixture_data/help_commands.txt b/tests/features/fixture_data/help_commands.txt new file mode 100644 index 000000000..3f3dd0b95 --- /dev/null +++ b/tests/features/fixture_data/help_commands.txt @@ -0,0 +1,21 @@ +Command +\# +\? +\c[onnect] database_name +\d [pattern] +\dT[S+] [pattern] +\df[+] [pattern] +\di[+] [pattern] +\dn[+] [pattern] +\ds[+] [pattern] +\dt[+] [pattern] +\du[+] [pattern] +\dv[+] [pattern] +\e [file] +\l +\n[+] [name] +\nd [name] +\ns name query +\refresh +\timing +\x \ No newline at end of file diff --git a/tests/features/fixture_utils.py b/tests/features/fixture_utils.py new file mode 100644 index 000000000..917baeb32 --- /dev/null +++ b/tests/features/fixture_utils.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from __future__ import print_function + +import os +import codecs + + +def read_fixture_lines(filename): + """ + Read lines of text from file. + :param filename: string name + :return: list of strings + """ + lines = [] + for line in codecs.open(filename, 'rb', encoding='utf-8'): + lines.append(line.strip()) + return lines + + +def read_fixture_files(): + """ + Read all files inside fixture_data directory. + """ + fixture_dict = {} + + current_dir = os.path.dirname(__file__) + fixture_dir = os.path.join(current_dir, 'fixture_data/') + for filename in os.listdir(fixture_dir): + if filename not in ['.', '..']: + fullname = os.path.join(fixture_dir, filename) + fixture_dict[filename] = read_fixture_lines(fullname) + + return fixture_dict diff --git a/tests/features/iocommands.feature b/tests/features/iocommands.feature new file mode 100644 index 000000000..d043dc2ea --- /dev/null +++ b/tests/features/iocommands.feature @@ -0,0 +1,10 @@ +Feature: I/O commands + + Scenario: edit sql in file with external editor + When we run dbcli + and we wait for prompt + and we start external editor providing a file name + and we type sql in the editor + and we exit the editor + then we see dbcli prompt + and we see the sql in prompt diff --git a/tests/features/named_queries.feature b/tests/features/named_queries.feature new file mode 100644 index 000000000..79f31ac3a --- /dev/null +++ b/tests/features/named_queries.feature @@ -0,0 +1,12 @@ +Feature: named queries: + save, use and delete named queries + + Scenario: save, use and delete named queries + When we run dbcli + and we wait for prompt + and we connect to test database + then we see database connected + when we save a named query + then we see the named query saved + when we delete a named query + then we see the named query deleted diff --git a/tests/features/specials.feature b/tests/features/specials.feature new file mode 100644 index 000000000..9bacec45a --- /dev/null +++ b/tests/features/specials.feature @@ -0,0 +1,9 @@ +Feature: Special commands + + @wip + Scenario: run refresh command + When we run dbcli + and we wait for prompt + and we refresh completions + and we wait for prompt + then we see completions refresh started diff --git a/tests/features/steps/basic_commands.py b/tests/features/steps/basic_commands.py new file mode 100644 index 000000000..568c793e4 --- /dev/null +++ b/tests/features/steps/basic_commands.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import pexpect + +from behave import when +import wrappers + + +@when('we run dbcli') +def step_run_cli(context): + """ + Run the process using pexpect. + """ + cli_cmd = context.conf.get('cli_command') + context.cli = pexpect.spawnu(cli_cmd, cwd='..') + context.exit_sent = False + + +@when('we wait for prompt') +def step_wait_prompt(context): + """ + Make sure prompt is displayed. + """ + wrappers.expect_exact(context, '{0}> '.format(context.conf['dbname']), timeout=5) + + +@when('we send "ctrl + d"') +def step_ctrl_d(context): + """ + Send Ctrl + D to hopefully exit. + """ + context.cli.sendcontrol('d') + context.exit_sent = True + + +@when('we send "\?" command') +def step_send_help(context): + """ + Send \? to see help. + """ + context.cli.sendline('\?') diff --git a/tests/features/steps/crud_database.py b/tests/features/steps/crud_database.py new file mode 100644 index 000000000..97bdfa7e2 --- /dev/null +++ b/tests/features/steps/crud_database.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import pexpect + +import wrappers +from behave import when, then + + +@when('we create database') +def step_db_create(context): + """ + Send create database. + """ + context.cli.sendline('create database {0};'.format( + context.conf['dbname_tmp'])) + + context.response = { + 'database_name': context.conf['dbname_tmp'] + } + + +@when('we drop database') +def step_db_drop(context): + """ + Send drop database. + """ + context.cli.sendline('drop database {0};'.format( + context.conf['dbname_tmp'])) + + +@when('we connect to test database') +def step_db_connect_test(context): + """ + Send connect to database. + """ + db_name = context.conf['dbname'] + context.cli.sendline('\\connect {0}'.format(db_name)) + + +@when('we connect to dbserver') +def step_db_connect_dbserver(context): + """ + Send connect to database. + """ + context.cli.sendline('\\connect postgres') + + +@then('dbcli exits') +def step_wait_exit(context): + """ + Make sure the cli exits. + """ + wrappers.expect_exact(context, pexpect.EOF, timeout=5) + + +@then('we see dbcli prompt') +def step_see_prompt(context): + """ + Wait to see the prompt. + """ + wrappers.expect_exact(context, '{0}> '.format(context.conf['dbname']), timeout=5) + + +@then('we see help output') +def step_see_help(context): + for expected_line in context.fixture_data['help_commands.txt']: + wrappers.expect_exact(context, expected_line, timeout=1) + + +@then('we see database created') +def step_see_db_created(context): + """ + Wait to see create database output. + """ + wrappers.expect_exact(context, 'CREATE DATABASE', timeout=2) + + +@then('we see database dropped') +def step_see_db_dropped(context): + """ + Wait to see drop database output. + """ + wrappers.expect_exact(context, 'DROP DATABASE', timeout=2) + + +@then('we see database connected') +def step_see_db_connected(context): + """ + Wait to see drop database output. + """ + wrappers.expect_exact(context, 'You are now connected to database', timeout=2) diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py new file mode 100644 index 000000000..0863a224f --- /dev/null +++ b/tests/features/steps/crud_table.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import wrappers +from behave import when, then + + +@when('we create table') +def step_create_table(context): + """ + Send create table. + """ + context.cli.sendline('create table a(x text);') + + +@when('we insert into table') +def step_insert_into_table(context): + """ + Send insert into table. + """ + context.cli.sendline('''insert into a(x) values('xxx');''') + + +@when('we update table') +def step_update_table(context): + """ + Send insert into table. + """ + context.cli.sendline('''update a set x = 'yyy' where x = 'xxx';''') + + +@when('we select from table') +def step_select_from_table(context): + """ + Send select from table. + """ + context.cli.sendline('select * from a;') + + +@when('we delete from table') +def step_delete_from_table(context): + """ + Send deete from table. + """ + context.cli.sendline('''delete from a where x = 'yyy';''') + + +@when('we drop table') +def step_drop_table(context): + """ + Send drop table. + """ + context.cli.sendline('drop table a;') + + +@then('we see table created') +def step_see_table_created(context): + """ + Wait to see create table output. + """ + wrappers.expect_exact(context, 'CREATE TABLE', timeout=2) + + +@then('we see record inserted') +def step_see_record_inserted(context): + """ + Wait to see insert output. + """ + wrappers.expect_exact(context, 'INSERT 0 1', timeout=2) + + +@then('we see record updated') +def step_see_record_updated(context): + """ + Wait to see update output. + """ + wrappers.expect_exact(context, 'UPDATE 1', timeout=2) + + +@then('we see data selected') +def step_see_data_selected(context): + """ + Wait to see select output. + """ + wrappers.expect_exact(context, 'yyy', timeout=1) + wrappers.expect_exact(context, 'SELECT 1', timeout=1) + + +@then('we see record deleted') +def step_see_data_deleted(context): + """ + Wait to see delete output. + """ + wrappers.expect_exact(context, 'DELETE 1', timeout=2) + + +@then('we see table dropped') +def step_see_table_dropped(context): + """ + Wait to see drop output. + """ + wrappers.expect_exact(context, 'DROP TABLE', timeout=2) diff --git a/tests/features/steps/iocommands.py b/tests/features/steps/iocommands.py new file mode 100644 index 000000000..885200469 --- /dev/null +++ b/tests/features/steps/iocommands.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 +from __future__ import unicode_literals +import os +import wrappers + +from behave import when, then + + +@when('we start external editor providing a file name') +def step_edit_file(context): + """ + Edit file with external editor. + """ + context.editor_file_name = 'test_file_{0}.sql'.format(context.conf['vi']) + if os.path.exists(context.editor_file_name): + os.remove(context.editor_file_name) + context.cli.sendline('\e {0}'.format(context.editor_file_name)) + wrappers.expect_exact(context, 'Entering Ex mode. Type "visual" to go to Normal mode.', timeout=2) + wrappers.expect_exact(context, '\r\n:', timeout=2) + + +@when('we type sql in the editor') +def step_edit_type_sql(context): + context.cli.sendline('i') + context.cli.sendline('select * from abc') + context.cli.sendline('.') + wrappers.expect_exact(context, ':', timeout=2) + + +@when('we exit the editor') +def step_edit_quit(context): + context.cli.sendline('x') + wrappers.expect_exact(context, "written", timeout=2) + + +@then('we see the sql in prompt') +def step_edit_done_sql(context): + for match in 'select * from abc'.split(' '): + wrappers.expect_exact(context, match, timeout=1) + # Cleanup the command line. + context.cli.sendcontrol('u') + # Cleanup the edited file. + if context.editor_file_name and os.path.exists(context.editor_file_name): + os.remove(context.editor_file_name) diff --git a/tests/features/steps/named_queries.py b/tests/features/steps/named_queries.py new file mode 100644 index 000000000..cd1273c2a --- /dev/null +++ b/tests/features/steps/named_queries.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import wrappers +from behave import when, then + + +@when('we save a named query') +def step_save_named_query(context): + """ + Send \ns command + """ + context.cli.sendline('\\ns foo SELECT 12345') + + +@when('we use a named query') +def step_use_named_query(context): + """ + Send \n command + """ + context.cli.sendline('\\n foo') + + +@when('we delete a named query') +def step_delete_named_query(context): + """ + Send \nd command + """ + context.cli.sendline('\\nd foo') + + +@then('we see the named query saved') +def step_see_named_query_saved(context): + """ + Wait to see query saved. + """ + wrappers.expect_exact(context, 'Saved.', timeout=1) + + +@then('we see the named query executed') +def step_see_named_query_executed(context): + """ + Wait to see select output. + """ + wrappers.expect_exact(context, '12345', timeout=1) + wrappers.expect_exact(context, 'SELECT 1', timeout=1) + + +@then('we see the named query deleted') +def step_see_named_query_deleted(context): + """ + Wait to see query deleted. + """ + wrappers.expect_exact(context, 'foo: Deleted', timeout=1) diff --git a/tests/features/steps/specials.py b/tests/features/steps/specials.py new file mode 100644 index 000000000..a5122c22e --- /dev/null +++ b/tests/features/steps/specials.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import wrappers +from behave import when, then + + +@when('we refresh completions') +def step_refresh_completions(context): + """ + Send refresh command. + """ + context.cli.sendline('\\refresh') + + +@then('we see completions refresh started') +def step_see_refresh_started(context): + """ + Wait to see refresh output. + """ + wrappers.expect_exact(context, 'refresh started in the background', timeout=2) diff --git a/tests/features/steps/wrappers.py b/tests/features/steps/wrappers.py new file mode 100644 index 000000000..eac7c8304 --- /dev/null +++ b/tests/features/steps/wrappers.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 +from __future__ import unicode_literals + +import re + + +def expect_exact(context, expected, timeout): + try: + context.cli.expect_exact(expected, timeout=timeout) + except: + # Strip color codes out of the output. + actual = re.sub(r'\x1b\[([0-9A-Za-z;?])+[m|K]?', '', context.cli.before) + raise Exception('Expected:\n---\n{0!r}\n---\n\nActual:\n---\n{1!r}\n---'.format( + expected, + actual)) From 48a9ab73eb2f2099d8b297d39684040173f6f0ed Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sun, 2 Apr 2017 20:40:10 +0200 Subject: [PATCH 155/824] Changes for differences between pgcli and mycli --- .coveragerc | 3 + .gitignore | 3 + .travis.yml | 6 +- changelog.md | 1 + requirements-dev.txt | 3 + tests/behave.ini | 5 -- tests/features/db_utils.py | 52 +++++++++-------- tests/features/environment.py | 58 ++++++++----------- tests/features/fixture_data/help_commands.txt | 48 ++++++++------- tests/features/fixture_utils.py | 5 +- tests/features/steps/basic_commands.py | 23 ++++++-- tests/features/steps/crud_database.py | 21 ++++--- tests/features/steps/crud_table.py | 17 +++--- tests/features/steps/named_queries.py | 6 +- tests/features/steps/specials.py | 4 +- 15 files changed, 143 insertions(+), 112 deletions(-) create mode 100644 .coveragerc delete mode 100644 tests/behave.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..ae818eefb --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +parallel=True +source=mycli diff --git a/.gitignore b/.gitignore index 12588dc82..59fa76be2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,12 @@ /dist /mycli.egg-info /src +/tests/behave.ini .vagrant *.pyc *.deb *.swp .cache/ +.coverage +.coverage.* diff --git a/.travis.yml b/.travis.yml index 1a7991836..c0473c48f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,16 @@ python: - "3.6" install: - - pip install PyMySQL . pytest mock codecov + - pip install PyMySQL . pytest mock codecov pexpect behave script: - coverage run --source mycli -m py.test + - cd tests + - behave + - cd .. after_success: + - coverage combine - codecov notifications: diff --git a/changelog.md b/changelog.md index 924d8efa4..7fe1b0243 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,7 @@ Bug Fixes: * Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) * Fix bug where mycli would not start due to the thanks/credit intro text. (Thanks: [Thomas Roten]). +* Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). Internal Changes: ----------------- diff --git a/requirements-dev.txt b/requirements-dev.txt index e54eabb9c..7511ac52e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,6 @@ mock pytest tox twine==1.8.1 +behave +pexpect +coverage==4.3.4 diff --git a/tests/behave.ini b/tests/behave.ini deleted file mode 100644 index c555d3c07..000000000 --- a/tests/behave.ini +++ /dev/null @@ -1,5 +0,0 @@ -[behave.userdata] -pg_test_user = postgres -pg_test_pass = -pg_test_host = localhost -pg_test_db = pgcli_behave_tests diff --git a/tests/features/db_utils.py b/tests/features/db_utils.py index 63c6586b7..f604c608c 100644 --- a/tests/features/db_utils.py +++ b/tests/features/db_utils.py @@ -2,9 +2,7 @@ from __future__ import unicode_literals from __future__ import print_function -from psycopg2 import connect -from psycopg2.extensions import AsIs - +import pymysql def create_db(hostname='localhost', username=None, password=None, dbname=None): @@ -16,15 +14,17 @@ def create_db(hostname='localhost', username=None, password=None, :param dbname: string :return: """ - cn = create_cn(hostname, password, username, 'postgres') - - # ISOLATION_LEVEL_AUTOCOMMIT = 0 - # Needed for DB creation. - cn.set_isolation_level(0) + cn = pymysql.connect( + host=hostname, + user=username, + password=password, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) with cn.cursor() as cr: - cr.execute('drop database if exists %s', (AsIs(dbname),)) - cr.execute('create database %s', (AsIs(dbname),)) + cr.execute('drop database if exists '+dbname) + cr.execute('create database '+dbname) cn.close() @@ -41,13 +41,15 @@ def create_cn(hostname, password, username, dbname): :param dbname: string :return: psycopg2.connection """ - if password: - cn = connect(host=hostname, user=username, database=dbname, - password=password) - else: - cn = connect(user=username, database=dbname) + cn = pymysql.connect( + host=hostname, + user=username, + password=password, + db=dbname, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) - print('Created connection: {0}.'.format(cn.dsn)) return cn @@ -60,14 +62,17 @@ def drop_db(hostname='localhost', username=None, password=None, :param password: string :param dbname: string """ - cn = create_cn(hostname, password, username, 'postgres') - - # ISOLATION_LEVEL_AUTOCOMMIT = 0 - # Needed for DB drop. - cn.set_isolation_level(0) + cn = pymysql.connect( + host=hostname, + user=username, + password=password, + db=dbname, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) with cn.cursor() as cr: - cr.execute('drop database if exists %s', (AsIs(dbname),)) + cr.execute('drop database if exists '+dbname) close_cn(cn) @@ -75,8 +80,7 @@ def drop_db(hostname='localhost', username=None, password=None, def close_cn(cn=None): """ Close connection. - :param connection: psycopg2.connection + :param connection: pymysql.connection """ if cn: cn.close() - print('Closed connection: {0}.'.format(cn.dsn)) diff --git a/tests/features/environment.py b/tests/features/environment.py index 51ae86b5a..3f55757b8 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -21,43 +21,31 @@ def before_all(context): context.exit_sent = False vi = '_'.join([str(x) for x in sys.version_info[:3]]) - db_name = context.config.userdata.get('pg_test_db', None) + db_name = context.config.userdata.get('my_test_db', None) or "mycli_behave_tests" db_name_full = '{0}_{1}'.format(db_name, vi) - # Store get params from config. + # Store get params from config/environment variables context.conf = { - 'host': context.config.userdata.get('pg_test_host', 'localhost'), - 'user': context.config.userdata.get('pg_test_user', 'postgres'), - 'pass': context.config.userdata.get('pg_test_pass', None), - 'dbname': db_name_full, + 'host': context.config.userdata.get( + 'my_test_host', + os.getenv('PYTEST_HOST', 'localhost') + ), + 'user': context.config.userdata.get( + 'my_test_user', + os.getenv('PYTEST_USER', 'root') + ), + 'pass': context.config.userdata.get( + 'my_test_pass', + os.getenv('PYTEST_PASSWORD', None) + ), + 'cli_command': context.config.userdata.get( + 'my_cli_command', None) or + sys.executable+' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"', + 'dbname': db_name, 'dbname_tmp': db_name_full + '_tmp', 'vi': vi, - 'cli_command': context.config.userdata.get('pg_cli_command', None) or - sys.executable + - ' -c "import coverage; coverage.process_startup(); import pgcli.main; pgcli.main.cli()"' } - # Store old env vars. - context.pgenv = { - 'PGDATABASE': os.environ.get('PGDATABASE', None), - 'PGUSER': os.environ.get('PGUSER', None), - 'PGHOST': os.environ.get('PGHOST', None), - 'PGPASSWORD': os.environ.get('PGPASSWORD', None), - } - - # Set new env vars. - os.environ['PGDATABASE'] = context.conf['dbname'] - os.environ['PGUSER'] = context.conf['user'] - os.environ['PGHOST'] = context.conf['host'] - - if context.conf['pass']: - os.environ['PGPASSWORD'] = context.conf['pass'] - else: - if 'PGPASSWORD' in os.environ: - del os.environ['PGPASSWORD'] - if 'PGHOST' in os.environ: - del os.environ['PGHOST'] - context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], context.conf['pass'], context.conf['dbname']) @@ -74,11 +62,11 @@ def after_all(context): context.conf['pass'], context.conf['dbname']) # Restore env vars. - for k, v in context.pgenv.items(): - if k in os.environ and v is None: - del os.environ[k] - elif v: - os.environ[k] = v + #for k, v in context.pgenv.items(): + # if k in os.environ and v is None: + # del os.environ[k] + # elif v: + # os.environ[k] = v def after_scenario(context, _): diff --git a/tests/features/fixture_data/help_commands.txt b/tests/features/fixture_data/help_commands.txt index 3f3dd0b95..1c5a08bf4 100644 --- a/tests/features/fixture_data/help_commands.txt +++ b/tests/features/fixture_data/help_commands.txt @@ -1,21 +1,27 @@ -Command -\# -\? -\c[onnect] database_name -\d [pattern] -\dT[S+] [pattern] -\df[+] [pattern] -\di[+] [pattern] -\dn[+] [pattern] -\ds[+] [pattern] -\dt[+] [pattern] -\du[+] [pattern] -\dv[+] [pattern] -\e [file] -\l -\n[+] [name] -\nd [name] -\ns name query -\refresh -\timing -\x \ No newline at end of file ++-------------+-------------------+---------------------------------------------------------+ +| Command | Shortcut | Description | +|-------------+-------------------+---------------------------------------------------------| +| \G | \G | Display results vertically. | +| \dt | \dt [table] | List or describe tables. | +| \e | \e | Edit command with editor. (uses $EDITOR) | +| \f | \f [name] | List or execute favorite queries. | +| \fd | \fd [name] | Delete a favorite query. | +| \fs | \fs name query | Save a favorite query. | +| \l | \l | List databases. | +| \timing | \t | Toggle timing of commands. | +| connect | \r | Reconnect to the database. Optional database argument. | +| exit | \q | Exit. | +| help | \? | Show this help. | +| nopager | \n | Disable pager, print to stdout. | +| notee | notee | stop writing to an output file | +| pager | \P [command] | Set PAGER. Print the query results via PAGER | +| prompt | \R | Change prompt format. | +| quit | \q | Quit. | +| rehash | \# | Refresh auto-completions. | +| source | \. filename | Execute commands from file. | +| status | \s | Get status information from the server. | +| system | system [command] | Execute a system commmand. | +| tableformat | \T | Change Table Type. | +| tee | tee [-o] filename | write to an output file (optionally overwrite using -o) | +| use | \u | Change to a new database. | ++-------------+-------------------+---------------------------------------------------------+ diff --git a/tests/features/fixture_utils.py b/tests/features/fixture_utils.py index 917baeb32..f3b490c40 100644 --- a/tests/features/fixture_utils.py +++ b/tests/features/fixture_utils.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from __future__ import print_function import os -import codecs +import io def read_fixture_lines(filename): @@ -13,7 +12,7 @@ def read_fixture_lines(filename): :return: list of strings """ lines = [] - for line in codecs.open(filename, 'rb', encoding='utf-8'): + for line in io.open(filename, 'r', encoding='utf8'): lines.append(line.strip()) return lines diff --git a/tests/features/steps/basic_commands.py b/tests/features/steps/basic_commands.py index 568c793e4..7aa476640 100644 --- a/tests/features/steps/basic_commands.py +++ b/tests/features/steps/basic_commands.py @@ -17,8 +17,20 @@ def step_run_cli(context): """ Run the process using pexpect. """ - cli_cmd = context.conf.get('cli_command') - context.cli = pexpect.spawnu(cli_cmd, cwd='..') + run_args = [] + if context.conf.get('host', None): + run_args.extend(('-h', context.conf['host'])) + if context.conf.get('user', None): + run_args.extend(('-u', context.conf['user'])) + if context.conf.get('pass', None): + run_args.extend(('-p', context.conf['pass'])) + if context.conf.get('dbname', None): + run_args.extend(('-D', context.conf['dbname'])) + cli_cmd = context.conf.get('cli_command', None) or sys.executable+' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' + + cmd_parts = [cli_cmd] + run_args + cmd = ' '.join(cmd_parts) + context.cli = pexpect.spawnu(cmd, cwd='..') context.exit_sent = False @@ -27,7 +39,10 @@ def step_wait_prompt(context): """ Make sure prompt is displayed. """ - wrappers.expect_exact(context, '{0}> '.format(context.conf['dbname']), timeout=5) + user = context.conf['user'] + host = context.conf['host'] + dbname = context.conf['dbname'] + wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format(user, host, dbname), timeout=5) @when('we send "ctrl + d"') @@ -44,4 +59,4 @@ def step_send_help(context): """ Send \? to see help. """ - context.cli.sendline('\?') + context.cli.sendline('\\?') diff --git a/tests/features/steps/crud_database.py b/tests/features/steps/crud_database.py index 97bdfa7e2..3eab34d9d 100644 --- a/tests/features/steps/crud_database.py +++ b/tests/features/steps/crud_database.py @@ -33,6 +33,8 @@ def step_db_drop(context): context.cli.sendline('drop database {0};'.format( context.conf['dbname_tmp'])) + wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + context.cli.sendline('y') @when('we connect to test database') def step_db_connect_test(context): @@ -40,7 +42,7 @@ def step_db_connect_test(context): Send connect to database. """ db_name = context.conf['dbname'] - context.cli.sendline('\\connect {0}'.format(db_name)) + context.cli.sendline('use {0}'.format(db_name)) @when('we connect to dbserver') @@ -48,7 +50,7 @@ def step_db_connect_dbserver(context): """ Send connect to database. """ - context.cli.sendline('\\connect postgres') + context.cli.sendline('use mysql') @then('dbcli exits') @@ -64,13 +66,16 @@ def step_see_prompt(context): """ Wait to see the prompt. """ - wrappers.expect_exact(context, '{0}> '.format(context.conf['dbname']), timeout=5) + user = context.conf['user'] + host = context.conf['host'] + dbname = context.conf['dbname'] + wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format(user, host, dbname), timeout=5) @then('we see help output') def step_see_help(context): for expected_line in context.fixture_data['help_commands.txt']: - wrappers.expect_exact(context, expected_line, timeout=1) + wrappers.expect_exact(context, expected_line+'\r\n', timeout=1) @then('we see database created') @@ -78,7 +83,7 @@ def step_see_db_created(context): """ Wait to see create database output. """ - wrappers.expect_exact(context, 'CREATE DATABASE', timeout=2) + wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see database dropped') @@ -86,7 +91,7 @@ def step_see_db_dropped(context): """ Wait to see drop database output. """ - wrappers.expect_exact(context, 'DROP DATABASE', timeout=2) + wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see database connected') @@ -94,4 +99,6 @@ def step_see_db_connected(context): """ Wait to see drop database output. """ - wrappers.expect_exact(context, 'You are now connected to database', timeout=2) + wrappers.expect_exact(context, 'You are now connected to database "', timeout=2) + wrappers.expect_exact(context, '"', timeout=2) + wrappers.expect_exact(context, ' as user "{0}"\r\n'.format(context.conf['user']), timeout=2) diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py index 0863a224f..72b4e0800 100644 --- a/tests/features/steps/crud_table.py +++ b/tests/features/steps/crud_table.py @@ -48,6 +48,8 @@ def step_delete_from_table(context): Send deete from table. """ context.cli.sendline('''delete from a where x = 'yyy';''') + wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + context.cli.sendline('y') @when('we drop table') @@ -56,6 +58,8 @@ def step_drop_table(context): Send drop table. """ context.cli.sendline('drop table a;') + wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + context.cli.sendline('y') @then('we see table created') @@ -63,7 +67,7 @@ def step_see_table_created(context): """ Wait to see create table output. """ - wrappers.expect_exact(context, 'CREATE TABLE', timeout=2) + wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see record inserted') @@ -71,7 +75,7 @@ def step_see_record_inserted(context): """ Wait to see insert output. """ - wrappers.expect_exact(context, 'INSERT 0 1', timeout=2) + wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see record updated') @@ -79,7 +83,7 @@ def step_see_record_updated(context): """ Wait to see update output. """ - wrappers.expect_exact(context, 'UPDATE 1', timeout=2) + wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see data selected') @@ -87,8 +91,7 @@ def step_see_data_selected(context): """ Wait to see select output. """ - wrappers.expect_exact(context, 'yyy', timeout=1) - wrappers.expect_exact(context, 'SELECT 1', timeout=1) + wrappers.expect_exact(context, '+-----+\r\n| x |\r\n|-----|\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') @@ -96,7 +99,7 @@ def step_see_data_deleted(context): """ Wait to see delete output. """ - wrappers.expect_exact(context, 'DELETE 1', timeout=2) + wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see table dropped') @@ -104,4 +107,4 @@ def step_see_table_dropped(context): """ Wait to see drop output. """ - wrappers.expect_exact(context, 'DROP TABLE', timeout=2) + wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) diff --git a/tests/features/steps/named_queries.py b/tests/features/steps/named_queries.py index cd1273c2a..b53ad47db 100644 --- a/tests/features/steps/named_queries.py +++ b/tests/features/steps/named_queries.py @@ -15,7 +15,7 @@ def step_save_named_query(context): """ Send \ns command """ - context.cli.sendline('\\ns foo SELECT 12345') + context.cli.sendline('\\fs foo SELECT 12345') @when('we use a named query') @@ -23,7 +23,7 @@ def step_use_named_query(context): """ Send \n command """ - context.cli.sendline('\\n foo') + context.cli.sendline('\\f foo') @when('we delete a named query') @@ -31,7 +31,7 @@ def step_delete_named_query(context): """ Send \nd command """ - context.cli.sendline('\\nd foo') + context.cli.sendline('\\fd foo') @then('we see the named query saved') diff --git a/tests/features/steps/specials.py b/tests/features/steps/specials.py index a5122c22e..790b2476f 100644 --- a/tests/features/steps/specials.py +++ b/tests/features/steps/specials.py @@ -15,7 +15,7 @@ def step_refresh_completions(context): """ Send refresh command. """ - context.cli.sendline('\\refresh') + context.cli.sendline('rehash') @then('we see completions refresh started') @@ -23,4 +23,4 @@ def step_see_refresh_started(context): """ Wait to see refresh output. """ - wrappers.expect_exact(context, 'refresh started in the background', timeout=2) + wrappers.expect_exact(context, 'Auto-completion refresh started in the background', timeout=2) From d1f624780ab5217dd06f0b872d2bed0d82e54dfa Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 4 Apr 2017 18:48:23 -0500 Subject: [PATCH 156/824] Add --myclirc option to changelog. --- changelog.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 7fe1b0243..06cd58880 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Features: +--------- + +* Add ability to specify alternative myclirc file. (Thanks: [Dick Marinus]). + Bug Fixes: ---------- @@ -10,13 +15,13 @@ Bug Fixes: * Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) * Fix bug where mycli would not start due to the thanks/credit intro text. (Thanks: [Thomas Roten]). -* Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). Internal Changes: ----------------- * Upload mycli distributions in a safer manner (using twine). (Thanks: [Thomas Roten]). +* Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). 1.9.0: ====== From 549d9ad683a67ae16ef08fc92ba62c02252cc737 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 4 Apr 2017 18:57:56 -0500 Subject: [PATCH 157/824] Fix behave tests table formatting. --- tests/features/fixture_data/help_commands.txt | 2 +- tests/features/steps/crud_table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/features/fixture_data/help_commands.txt b/tests/features/fixture_data/help_commands.txt index 1c5a08bf4..ee3d4ca47 100644 --- a/tests/features/fixture_data/help_commands.txt +++ b/tests/features/fixture_data/help_commands.txt @@ -1,6 +1,6 @@ +-------------+-------------------+---------------------------------------------------------+ | Command | Shortcut | Description | -|-------------+-------------------+---------------------------------------------------------| ++-------------+-------------------+---------------------------------------------------------+ | \G | \G | Display results vertically. | | \dt | \dt [table] | List or describe tables. | | \e | \e | Edit command with editor. (uses $EDITOR) | diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py index 72b4e0800..a4ed0fe53 100644 --- a/tests/features/steps/crud_table.py +++ b/tests/features/steps/crud_table.py @@ -91,7 +91,7 @@ def step_see_data_selected(context): """ Wait to see select output. """ - wrappers.expect_exact(context, '+-----+\r\n| x |\r\n|-----|\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) + wrappers.expect_exact(context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') From 7b72b05dcafc1567a473fc5ea542b33dcdfe8e23 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 4 Apr 2017 19:08:55 -0500 Subject: [PATCH 158/824] Update README app usage. --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6ecca526c..c12a8eb5e 100644 --- a/README.md +++ b/README.md @@ -43,11 +43,11 @@ $ sudo apt-get install mycli # Only on debian or ubuntu Options: -h, --host TEXT Host address of the database. - -P, --port TEXT Port number to use for connection. Honors + -P, --port INTEGER Port number to use for connection. Honors $MYSQL_TCP_PORT -u, --user TEXT User name to connect to the database. -S, --socket TEXT The socket file to use for connection. - -p, --password Force password prompt. + -p, --password TEXT Password to connect to the database --pass TEXT Password to connect to the database --ssl-ca PATH CA file in PEM format --ssl-capath TEXT CA directory @@ -58,18 +58,21 @@ $ sudo apt-get install mycli # Only on debian or ubuntu against hostname used when connecting. This option is disabled by default -v, --version Version of mycli. - -D, --database TEXT Database to use. + -D, --database TEXT Database to use. -R, --prompt TEXT Prompt format (Default: "\t \u@\h:\d> ") -l, --logfile FILENAME Log every query and its results to a file. --defaults-group-suffix TEXT Read config group with the specified suffix. --defaults-file PATH Only read default options from the given file + --myclirc PATH Location of myclirc file. --auto-vertical-output Automatically switch to vertical output mode if the result is wider than the terminal width. -t, --table Display batch output in table format. + --csv Display batch output in CSV format. --warn / --no-warn Warn before running a destructive query. --local-infile BOOLEAN Enable/disable LOAD DATA LOCAL INFILE. --login-path TEXT Read this path from the login file. + -e, --execute TEXT Execute query to the database. --help Show this message and exit. ### Examples From 26400cb895bfaee690d12815381a72a578e087f6 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 4 Apr 2017 19:12:24 -0500 Subject: [PATCH 159/824] Add README check to release script. --- release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/release.py b/release.py index 9a5d399f9..817695838 100755 --- a/release.py +++ b/release.py @@ -101,6 +101,7 @@ def checklist(questions): checks = ['Have you created the debian package?', 'Have you updated the AUTHORS file?', + 'Have you updated the `Usage` section of the README?', ] checklist(checks) From 03e4050128f9948c7ebf2862d3b854a1b9c91344 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Wed, 5 Apr 2017 18:20:39 -0700 Subject: [PATCH 160/824] Added pep8radius. --- .travis.yml | 3 +++ requirements-dev.txt | 1 + 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index c0473c48f..f78d94b00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,12 +8,15 @@ python: install: - pip install PyMySQL . pytest mock codecov pexpect behave + - pip install git+https://github.com/hayd/pep8radius.git script: - coverage run --source mycli -m py.test - cd tests - behave - cd .. + # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. + - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) after_success: - coverage combine diff --git a/requirements-dev.txt b/requirements-dev.txt index 7511ac52e..b7e6e2dca 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,3 +5,4 @@ twine==1.8.1 behave pexpect coverage==4.3.4 +pep8radius From 74a739fe9e35af3421e3ffc59620e7af031a0e81 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Wed, 5 Apr 2017 18:38:26 -0700 Subject: [PATCH 161/824] Added change to changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 06cd58880..fcf9c198d 100644 --- a/changelog.md +++ b/changelog.md @@ -22,6 +22,7 @@ Internal Changes: * Upload mycli distributions in a safer manner (using twine). (Thanks: [Thomas Roten]). * Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). +* Run pep8 checks in travis (Thanks: [Irina Truong]). 1.9.0: ====== From 079d9f870debfed03a8ec0351859ddd0a0670df0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 5 Apr 2017 22:04:04 -0500 Subject: [PATCH 162/824] PEP8 changes per pep8radius request. --- mycli/encodingutils.py | 15 +++++++++------ mycli/main.py | 15 +++++++++------ mycli/output_formatter/expanded.py | 1 + mycli/output_formatter/output_formatter.py | 4 +++- mycli/output_formatter/preprocessors.py | 10 ++++++++-- mycli/sqlexecute.py | 12 ++++++------ tests/features/steps/crud_table.py | 3 ++- tests/test_output_formatter.py | 3 ++- tests/utils.py | 3 ++- 9 files changed, 42 insertions(+), 24 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index aac4aa07d..1a8b5bbb1 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -16,9 +16,10 @@ def unicode2utf8(arg): - """ - Only in Python 2. Psycopg2 expects the args as bytes not unicode. - In Python 3 the args are expected as unicode. + """Convert strings to UTF8-encoded bytes. + + Only in Python 2. In Python 3 the args are expected as unicode. + """ if PY2 and isinstance(arg, text_type): @@ -27,9 +28,10 @@ def unicode2utf8(arg): def utf8tounicode(arg): - """ - Only in Python 2. Psycopg2 returns the error message as utf-8. - In Python 3 the errors are returned as unicode. + """Convert UTF8-encoded bytes to strings. + + Only in Python 2. In Python 3 the errors are returned as strings. + """ if PY2 and isinstance(arg, binary_type): @@ -46,6 +48,7 @@ def bytes_to_string(b): abc >>> print(bytes_to_string('✌')) ✌ + """ if isinstance(b, binary_type): try: diff --git a/mycli/main.py b/mycli/main.py index aea9c7301..64c587057 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -106,7 +106,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.formatter = output_formatter.OutputFormatter(format_name=c['main']['table_format']) + self.formatter = output_formatter.OutputFormatter( + format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] @@ -146,7 +147,7 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. self.smart_completion = c['main'].as_bool('smart_completion') self.completer = SQLCompleter(self.smart_completion, - supported_formats=self.formatter.supported_formats) + supported_formats=self.formatter.supported_formats) self._completer_lock = threading.Lock() # Register custom special commands. @@ -186,7 +187,8 @@ def change_table_format(self, arg, **_): yield (None, None, None, 'Changed table type to {}'.format(arg)) except ValueError: - msg = 'Table type {} not yet implemented. Allowed types:'.format(arg) + msg = 'Table type {} not yet implemented. Allowed types:'.format( + arg) for table_type in self.formatter.supported_formats(): msg += "\n\t{}".format(table_type) yield (None, None, None, msg) @@ -524,8 +526,9 @@ def one_iteration(document=None): else: max_width = None - formatted = self.format_output(title, cur, headers, - status, special.is_expanded_output(), max_width) + formatted = self.format_output(title, cur, headers, status, + special.is_expanded_output(), + max_width) output.extend(formatted) end = time() @@ -706,7 +709,7 @@ def get_prompt(self, string): return string def run_query(self, query, new_line=True): - """Runs query""" + """Runs *query*.""" results = self.sqlexecute.run(query) for result in results: title, cur, headers, status = result diff --git a/mycli/output_formatter/expanded.py b/mycli/output_formatter/expanded.py index 2ab579944..f77c1ee38 100644 --- a/mycli/output_formatter/expanded.py +++ b/mycli/output_formatter/expanded.py @@ -19,6 +19,7 @@ def expanded_table(rows, headers, **_): """Format *rows* and *headers* as an expanded table. The values in *rows* and *headers* must be strings. + """ header_len = max([len(x) for x in headers]) padded_headers = [x.ljust(header_len) for x in headers] diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index a47029cca..759fb879a 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -59,12 +59,14 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. + """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): raise ValueError('unrecognized format: {}'.format(format_name)) - (_, preprocessors, formatter, fkwargs) = self._output_formats[format_name] + (_, preprocessors, formatter, + fkwargs) = self._output_formats[format_name] fkwargs.update(kwargs) if preprocessors: for f in preprocessors: diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index d6db3fd44..010c1b14c 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -30,13 +30,15 @@ def bytes_to_string(data, headers, **_): def intlen(value): - """Find (character) length + """Find (character) length. + >>> intlen('11.1') 2 >>> intlen('11') 2 >>> intlen('1.1') 1 + """ pos = value.find('.') if pos < 0: @@ -45,11 +47,13 @@ def intlen(value): def align_decimals(data, headers, **_): - """Align decimals to decimal point + """Align decimals to decimal point. + >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) 1 11.1 1.1 + """ pointpos = len(data[0]) * [0] for row in data: @@ -72,6 +76,7 @@ def align_decimals(data, headers, **_): def quote_whitespaces(data, headers, quotestyle="'", **_): """Quote whitespace + >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) ' before' 'after ' @@ -82,6 +87,7 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): def ghi jkl + """ quote = len(data[0]) * [False] for row in data: diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 08030af47..4d5c0a091 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -4,7 +4,7 @@ from .packages import special from pymysql.constants import FIELD_TYPE from pymysql.converters import (convert_mysql_timestamp, convert_datetime, - convert_timedelta, convert_date, conversions) + convert_timedelta, convert_date, conversions) _logger = logging.getLogger(__name__) @@ -67,11 +67,11 @@ def connect(self, database=None, user=None, password=None, host=None, database, user, host, port, socket, charset, local_infile, ssl) conv = conversions.copy() conv.update({ - FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), - FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), - FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), - FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), - }) + FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), + FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), + FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), + FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), + }) conn = pymysql.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py index a4ed0fe53..b73ff0d6f 100644 --- a/tests/features/steps/crud_table.py +++ b/tests/features/steps/crud_table.py @@ -91,7 +91,8 @@ def step_see_data_selected(context): """ Wait to see select output. """ - wrappers.expect_exact(context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) + wrappers.expect_exact( + context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 7949032fd..ac5cfc073 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -57,7 +57,8 @@ def test_bytes_to_string(): def test_align_decimals(): """Test the *output_formatter.align_decimals()* function.""" - data = [[Decimal('200'), Decimal('1')], [Decimal('1.00002'), Decimal('1.0')]] + data = [[Decimal('200'), Decimal('1')], [ + Decimal('1.00002'), Decimal('1.0')]] headers = ['num1', 'num2'] expected = ([['200', '1'], [' 1.00002', '1.0']], ['num1', 'num2']) diff --git a/tests/utils.py b/tests/utils.py index bff31045d..b29e5e072 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,7 +44,8 @@ def run(executor, sql, join=False): # It should test raw results. mycli = MyCli() for title, rows, headers, status in executor.run(sql): - result.extend(mycli.format_output(title, rows, headers, status, special.is_expanded_output())) + result.extend(mycli.format_output(title, rows, headers, status, + special.is_expanded_output())) if join: result = '\n'.join(result) From 8bc0fc41db40bb86c35d159fbbab00bc34d5b092 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 5 Apr 2017 22:05:59 -0500 Subject: [PATCH 163/824] Add period for pep8radius. --- mycli/output_formatter/preprocessors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index 010c1b14c..c2e221f8f 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -75,7 +75,7 @@ def align_decimals(data, headers, **_): def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote whitespace + """Quote whitespace. >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) ' before' From 5c4c675d8019d36f59f2aedb55f02adb50b151a1 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Wed, 5 Apr 2017 21:09:42 -0700 Subject: [PATCH 164/824] Documented pep8radius usage. --- DEVELOP.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/DEVELOP.rst b/DEVELOP.rst index 5169abeb8..0d53e6ad7 100644 --- a/DEVELOP.rst +++ b/DEVELOP.rst @@ -70,3 +70,17 @@ after asking a few questions like maintainer name, email etc. $ vagrant up +PEP8 checks +----------- + +When you submit a PR, the changeset is checked for pep8 compliance using +`pep8radius `_. If you see a build failing because +of these checks, install pep8radius and apply style fixes: + +:: + + $ pip install pep8radius + $ pep8radius --docformatter --diff # view a diff of proposed fixes + $ pep8radius --docformatter --in-place # apply the fixes + +Then commit and push the fixes. From 935fc568f2b33f641c37cbd743ec2a213b82dc81 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 10 Apr 2017 08:06:18 -0500 Subject: [PATCH 165/824] Do not reference results rows by index. --- mycli/output_formatter/preprocessors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index c2e221f8f..8fe73bb73 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -55,7 +55,7 @@ def align_decimals(data, headers, **_): 1.1 """ - pointpos = len(data[0]) * [0] + pointpos = len(headers) * [0] for row in data: for i, v in enumerate(row): if isinstance(v, Decimal): @@ -89,7 +89,7 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): jkl """ - quote = len(data[0]) * [False] + quote = len(headers) * [False] for row in data: for i, v in enumerate(row): v = encodingutils.text_type(v) From 5849ac9261a9c5c68a10bd5b3fceee88c2daf4b2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 10 Apr 2017 08:15:21 -0500 Subject: [PATCH 166/824] Add empty result tests. --- mycli/output_formatter/preprocessors.py | 24 ++----------------- tests/test_output_formatter.py | 31 ++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index 8fe73bb73..6f2e459c0 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -47,14 +47,7 @@ def intlen(value): def align_decimals(data, headers, **_): - """Align decimals to decimal point. - - >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) - 1 - 11.1 - 1.1 - - """ + """Align decimals to decimal point.""" pointpos = len(headers) * [0] for row in data: for i, v in enumerate(row): @@ -75,20 +68,7 @@ def align_decimals(data, headers, **_): def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote whitespace. - - >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) - ' before' - 'after ' - ' both ' - 'none' - >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) - abc - def - ghi - jkl - - """ + """Quote leading/trailing whitespace.""" quote = len(headers) * [False] for row in data: for i, v in enumerate(row): diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index ac5cfc073..dcc26e62c 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -8,6 +8,7 @@ from mycli.output_formatter.preprocessors import (align_decimals, bytes_to_string, convert_to_string, + quote_whitespaces, override_missing_value, to_string) from mycli.output_formatter.output_formatter import OutputFormatter @@ -56,7 +57,7 @@ def test_bytes_to_string(): def test_align_decimals(): - """Test the *output_formatter.align_decimals()* function.""" + """Test the *align_decimals()* function.""" data = [[Decimal('200'), Decimal('1')], [ Decimal('1.00002'), Decimal('1.0')]] headers = ['num1', 'num2'] @@ -65,6 +66,34 @@ def test_align_decimals(): assert expected == align_decimals(data, headers) +def test_align_decimals_empty_result(): + """Test *align_decimals()* with no results.""" + data = [] + headers = ['num1', 'num2'] + expected = ([], ['num1', 'num2']) + + assert expected == align_decimals(data, headers) + + +def test_quote_whitespaces(): + """Test the *quote_whitespaces()* function.""" + data = [[" before", "after "], [" both ", "none"]] + headers = ['h1', 'h2'] + expected = ([["' before'", "'after '"], ["' both '", "'none'"]], + ['h1', 'h2']) + + assert expected == quote_whitespaces(data, headers) + + +def test_quote_whitespaces_empty_result(): + """Test the *quote_whitespaces()* function with no results.""" + data = [] + headers = ['h1', 'h2'] + expected = ([], ['h1', 'h2']) + + assert expected == quote_whitespaces(data, headers) + + def test_tabulate_wrapper(): """Test the *output_formatter.tabulate_wrapper()* function.""" data = [['abc', 1], ['d', 456]] From 385f47892fbf2e8c8b0fdf3bde174a2128a3b255 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 14:49:03 -0500 Subject: [PATCH 167/824] Revert "Remove tabulate license note." This reverts commit d0f1dbf441ae823d56cadca456b1ff35a35abd88. --- LICENSE.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/LICENSE.txt b/LICENSE.txt index 8afaa2657..9a41a67d4 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -27,3 +27,9 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------- + +This program also bundles with it python-tabulate +(https://pypi.python.org/pypi/tabulate) library. This library is licensed under +MIT License. + +------------------------------------------------------------------------------- From 16738b1dbcb6b146175e3308352c113006b5b3ca Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:13:03 -0500 Subject: [PATCH 168/824] Add vendored tabulate version 0.8.0. --- mycli/output_formatter/tabulate_adapter.py | 11 +- mycli/packages/tabulate.py | 1423 ++++++++++++++++++++ setup.py | 1 - 3 files changed, 1428 insertions(+), 7 deletions(-) create mode 100644 mycli/packages/tabulate.py diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 0db31ff9f..14e6c770f 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,15 +1,14 @@ -from tabulate import tabulate - -from .preprocessors import bytes_to_string, align_decimals +from mycli.packages import tabulate +from .preprocessors import bytes_to_string, align_decimals, quote_whitespaces supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', 'html', 'latex', 'latex_booktabs', 'textile') -preprocessors = (bytes_to_string, align_decimals) +preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value, disable_numparse=True) + return tabulate.tabulate(data, headers, tablefmt=table_format, + missingval=missing_value, disable_numparse=True) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py new file mode 100644 index 000000000..a25032995 --- /dev/null +++ b/mycli/packages/tabulate.py @@ -0,0 +1,1423 @@ +# -*- coding: utf-8 -*- + +"""Pretty-print tabular data.""" + +from __future__ import print_function +from __future__ import unicode_literals +from collections import namedtuple, Iterable +from platform import python_version_tuple +import re + + +if python_version_tuple()[0] < "3": + from itertools import izip_longest + from functools import partial + _none_type = type(None) + _bool_type = bool + _int_type = int + _long_type = long + _float_type = float + _text_type = unicode + _binary_type = str + + def _is_file(f): + return isinstance(f, file) + +else: + from itertools import zip_longest as izip_longest + from functools import reduce, partial + _none_type = type(None) + _bool_type = bool + _int_type = int + _long_type = int + _float_type = float + _text_type = str + _binary_type = bytes + basestring = str + + import io + + def _is_file(f): + return isinstance(f, io.IOBase) + +try: + import wcwidth # optional wide-character (CJK) support +except ImportError: + wcwidth = None + + +__all__ = ["tabulate", "tabulate_formats"] +__version__ = "0.8.0" + + +# minimum extra space in headers +MIN_PADDING = 2 + + +_DEFAULT_FLOATFMT = "g" +_DEFAULT_MISSINGVAL = "" + + +# if True, enable wide-character (CJK) support +WIDE_CHARS_MODE = wcwidth is not None + + +Line = namedtuple("Line", ["begin", "hline", "sep", "end"]) + + +DataRow = namedtuple("DataRow", ["begin", "sep", "end"]) + + +# A table structure is suppposed to be: +# +# --- lineabove --------- +# headerrow +# --- linebelowheader --- +# datarow +# --- linebewteenrows --- +# ... (more datarows) ... +# --- linebewteenrows --- +# last datarow +# --- linebelow --------- +# +# TableFormat's line* elements can be +# +# - either None, if the element is not used, +# - or a Line tuple, +# - or a function: [col_widths], [col_alignments] -> string. +# +# TableFormat's *row elements can be +# +# - either None, if the element is not used, +# - or a DataRow tuple, +# - or a function: [cell_values], [col_widths], [col_alignments] -> string. +# +# padding (an integer) is the amount of white space around data values. +# +# with_header_hide: +# +# - either None, to display all table elements unconditionally, +# - or a list of elements not to be displayed if the table has column +# headers. +# +TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", + "linebetweenrows", "linebelow", + "headerrow", "datarow", + "padding", "with_header_hide"]) + + +def _pipe_segment_with_colons(align, colwidth): + """Return a segment of a horizontal line with optional colons which + indicate column's alignment (as in `pipe` output format).""" + w = colwidth + if align in ["right", "decimal"]: + return ('-' * (w - 1)) + ":" + elif align == "center": + return ":" + ('-' * (w - 2)) + ":" + elif align == "left": + return ":" + ('-' * (w - 1)) + else: + return '-' * w + + +def _pipe_line_with_colons(colwidths, colaligns): + """Return a horizontal line with optional colons to indicate column's + alignment (as in `pipe` output format).""" + segments = [_pipe_segment_with_colons(a, w) for a, w in + zip(colaligns, colwidths)] + return "|" + "|".join(segments) + "|" + + +def _mediawiki_row_with_attrs(separator, cell_values, colwidths, colaligns): + alignment = {"left": '', + "right": 'align="right"| ', + "center": 'align="center"| ', + "decimal": 'align="right"| '} + # hard-coded padding _around_ align attribute and value together + # rather than padding parameter which affects only the value + values_with_attrs = [' ' + alignment.get(a, '') + c + ' ' + for c, a in zip(cell_values, colaligns)] + colsep = separator*2 + return (separator + colsep.join(values_with_attrs)).rstrip() + + +def _textile_row_with_attrs(cell_values, colwidths, colaligns): + cell_values[0] += ' ' + alignment = {"left": "<.", "right": ">.", "center": "=.", "decimal": ">."} + values = (alignment.get(a, '') + v for a, v in zip(colaligns, cell_values)) + return '|' + '|'.join(values) + '|' + + +def _html_begin_table_without_header(colwidths_ignore, colaligns_ignore): + # this table header will be suppressed if there is a header row + return "\n".join(["", ""]) + + +def _html_row_with_attrs(celltag, cell_values, colwidths, colaligns): + alignment = {"left": '', + "right": ' style="text-align: right;"', + "center": ' style="text-align: center;"', + "decimal": ' style="text-align: right;"'} + values_with_attrs = ["<{0}{1}>{2}".format( + celltag, alignment.get(a, ''), c) for c, a in + zip(cell_values, colaligns)] + rowhtml = "" + "".join(values_with_attrs).rstrip() + "" + if celltag == "th": # it's a header row, create a new table header + rowhtml = "\n".join(["
", + "", + rowhtml, + "", + ""]) + return rowhtml + + +def _moin_row_with_attrs(celltag, cell_values, colwidths, colaligns, + header=''): + alignment = {"left": '', + "right": '', + "center": '', + "decimal": ''} + values_with_attrs = ["{0}{1} {2} ".format(celltag, + alignment.get(a, ''), + header+c+header) + for c, a in zip(cell_values, colaligns)] + return "".join(values_with_attrs)+"||" + + +def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): + alignment = {"left": "l", "right": "r", "center": "c", "decimal": "r"} + tabular_columns_fmt = "".join([alignment.get(a, "l") for a in colaligns]) + return "\n".join(["\\begin{tabular}{" + tabular_columns_fmt + "}", + "\\toprule" if booktabs else "\hline"]) + + +LATEX_ESCAPE_RULES = {r"&": r"\&", r"%": r"\%", r"$": r"\$", r"#": r"\#", + r"_": r"\_", r"^": r"\^{}", r"{": r"\{", r"}": r"\}", + r"~": r"\textasciitilde{}", "\\": r"\textbackslash{}", + r"<": r"\ensuremath{<}", r">": r"\ensuremath{>}"} + + +def _latex_row(cell_values, colwidths, colaligns, escrules=LATEX_ESCAPE_RULES): + def escape_char(c): + return escrules.get(c, c) + escaped_values = ["".join(map(escape_char, cell)) for cell in cell_values] + rowfmt = DataRow("", "&", "\\\\") + return _build_simple_row(escaped_values, rowfmt) + + +def _rst_escape_first_column(rows, headers): + def escape_empty(val): + if isinstance(val, (_text_type, _binary_type)) and val.strip() is "": + return ".." + else: + return val + new_headers = list(headers) + new_rows = [] + if headers: + new_headers[0] = escape_empty(headers[0]) + for row in rows: + new_row = list(row) + if new_row: + new_row[0] = escape_empty(row[0]) + new_rows.append(new_row) + return new_rows, new_headers + + +_table_formats = {"simple": + TableFormat( + lineabove=Line("", "-", " ", ""), + linebelowheader=Line("", "-", " ", ""), + linebetweenrows=None, + linebelow=Line("", "-", " ", ""), + headerrow=DataRow("", " ", ""), + datarow=DataRow("", " ", ""), + padding=0, + with_header_hide=["lineabove", "linebelow"]), + "plain": + TableFormat( + lineabove=None, linebelowheader=None, + linebetweenrows=None, linebelow=None, + headerrow=DataRow("", " ", ""), + datarow=DataRow("", " ", ""), + padding=0, with_header_hide=None), + "grid": + TableFormat( + lineabove=Line("+", "-", "+", "+"), + linebelowheader=Line("+", "=", "+", "+"), + linebetweenrows=Line("+", "-", "+", "+"), + linebelow=Line("+", "-", "+", "+"), + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "fancy_grid": + TableFormat( + lineabove=Line("╒", "═", "╤", "╕"), + linebelowheader=Line("╞", "═", "╪", "╡"), + linebetweenrows=Line("├", "─", "┼", "┤"), + linebelow=Line("╘", "═", "╧", "╛"), + headerrow=DataRow("│", "│", "│"), + datarow=DataRow("│", "│", "│"), + padding=1, with_header_hide=None), + "pipe": + TableFormat( + lineabove=_pipe_line_with_colons, + linebelowheader=_pipe_line_with_colons, + linebetweenrows=None, + linebelow=None, + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, + with_header_hide=["lineabove"]), + "orgtbl": + TableFormat( + lineabove=None, + linebelowheader=Line("|", "-", "+", "|"), + linebetweenrows=None, + linebelow=None, + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "jira": + TableFormat( + lineabove=None, + linebelowheader=None, + linebetweenrows=None, + linebelow=None, + headerrow=DataRow("||", "||", "||"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "psql": + TableFormat( + lineabove=Line("+", "-", "+", "+"), + linebelowheader=Line("|", "-", "+", "|"), + linebetweenrows=None, + linebelow=Line("+", "-", "+", "+"), + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "rst": + TableFormat( + lineabove=Line("", "=", " ", ""), + linebelowheader=Line("", "=", " ", ""), + linebetweenrows=None, + linebelow=Line("", "=", " ", ""), + headerrow=DataRow("", " ", ""), + datarow=DataRow("", " ", ""), + padding=0, with_header_hide=None), + "mediawiki": + TableFormat(lineabove=Line( + "{| class=\"wikitable\" style=\"text-align: left;\"", + "", "", "\n|+ \n|-"), + linebelowheader=Line("|-", "", "", ""), + linebetweenrows=Line("|-", "", "", ""), + linebelow=Line("|}", "", "", ""), + headerrow=partial(_mediawiki_row_with_attrs, "!"), + datarow=partial(_mediawiki_row_with_attrs, "|"), + padding=0, with_header_hide=None), + "moinmoin": + TableFormat( + lineabove=None, + linebelowheader=None, + linebetweenrows=None, + linebelow=None, + headerrow=partial(_moin_row_with_attrs, "||", + header="'''"), + datarow=partial(_moin_row_with_attrs, "||"), + padding=1, with_header_hide=None), + "html": + TableFormat( + lineabove=_html_begin_table_without_header, + linebelowheader="", + linebetweenrows=None, + linebelow=Line("\n
", "", "", ""), + headerrow=partial(_html_row_with_attrs, "th"), + datarow=partial(_html_row_with_attrs, "td"), + padding=0, with_header_hide=["lineabove"]), + "latex": + TableFormat( + lineabove=_latex_line_begin_tabular, + linebelowheader=Line("\\hline", "", "", ""), + linebetweenrows=None, + linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), + headerrow=_latex_row, + datarow=_latex_row, + padding=1, with_header_hide=None), + "latex_raw": + TableFormat( + lineabove=_latex_line_begin_tabular, + linebelowheader=Line("\\hline", "", "", ""), + linebetweenrows=None, + linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), + headerrow=partial(_latex_row, escrules={}), + datarow=partial(_latex_row, escrules={}), + padding=1, with_header_hide=None), + "latex_booktabs": + TableFormat( + lineabove=partial(_latex_line_begin_tabular, + booktabs=True), + linebelowheader=Line("\\midrule", "", "", ""), + linebetweenrows=None, + linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", + ""), + headerrow=_latex_row, + datarow=_latex_row, + padding=1, with_header_hide=None), + "textile": + TableFormat( + lineabove=None, linebelowheader=None, + linebetweenrows=None, linebelow=None, + headerrow=DataRow("|_. ", "|_.", "|"), + datarow=_textile_row_with_attrs, + padding=1, with_header_hide=None)} + + +tabulate_formats = list(sorted(_table_formats.keys())) + + +# ANSI color codes +_invisible_codes = re.compile(r"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") +_invisible_codes_bytes = re.compile(b"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") + + +def _isconvertible(conv, string): + try: + n = conv(string) + return True + except (ValueError, TypeError): + return False + + +def _isnumber(string): + """ + >>> _isnumber("123.45") + True + >>> _isnumber("123") + True + >>> _isnumber("spam") + False + """ + return _isconvertible(float, string) + + +def _isint(string, inttype=int): + """ + >>> _isint("123") + True + >>> _isint("123.45") + False + """ + return type(string) is inttype or\ + (isinstance(string, _binary_type) or isinstance(string, _text_type))\ + and\ + _isconvertible(inttype, string) + + +def _isbool(string): + """ + >>> _isbool(True) + True + >>> _isbool("False") + True + >>> _isbool(1) + False + """ + return type(string) is _bool_type or\ + (isinstance(string, (_binary_type, _text_type)) and + string in ("True", "False")) + + +def _type(string, has_invisible=True, numparse=True): + """The least generic type (type(None), int, float, str, unicode). + + >>> _type(None) is type(None) + True + >>> _type("foo") is type("") + True + >>> _type("1") is type(1) + True + >>> _type('\x1b[31m42\x1b[0m') is type(42) + True + >>> _type('\x1b[31m42\x1b[0m') is type(42) + True + + """ + + if has_invisible and \ + (isinstance(string, _text_type) or isinstance(string, _binary_type)): + string = _strip_invisible(string) + + if string is None: + return _none_type + elif hasattr(string, "isoformat"): # datetime.datetime, date, and time + return _text_type + elif _isbool(string): + return _bool_type + elif _isint(string) and numparse: + return int + elif _isint(string, _long_type) and numparse: + return int + elif _isnumber(string) and numparse: + return float + elif isinstance(string, _binary_type): + return _binary_type + else: + return _text_type + + +def _afterpoint(string): + """Symbols after a decimal point, -1 if the string lacks the decimal point. + + >>> _afterpoint("123.45") + 2 + >>> _afterpoint("1001") + -1 + >>> _afterpoint("eggs") + -1 + >>> _afterpoint("123e45") + 2 + + """ + if _isnumber(string): + if _isint(string): + return -1 + else: + pos = string.rfind(".") + pos = string.lower().rfind("e") if pos < 0 else pos + if pos >= 0: + return len(string) - pos - 1 + else: + return -1 # no point + else: + return -1 # not a number + + +def _padleft(width, s): + """Flush right. + + >>> _padleft(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430' + True + + """ + fmt = "{0:>%ds}" % width + return fmt.format(s) + + +def _padright(width, s): + """Flush left. + + >>> _padright(6, '\u044f\u0439\u0446\u0430') == '\u044f\u0439\u0446\u0430 ' + True + + """ + fmt = "{0:<%ds}" % width + return fmt.format(s) + + +def _padboth(width, s): + """Center string. + + >>> _padboth(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430 ' + True + + """ + fmt = "{0:^%ds}" % width + return fmt.format(s) + + +def _strip_invisible(s): + "Remove invisible ANSI color codes." + if isinstance(s, _text_type): + return re.sub(_invisible_codes, "", s) + else: # a bytestring + return re.sub(_invisible_codes_bytes, "", s) + + +def _visible_width(s): + """Visible width of a printed string. ANSI color codes are removed. + + >>> _visible_width('\x1b[31mhello\x1b[0m'), _visible_width("world") + (5, 5) + + """ + # optional wide-character support + if wcwidth is not None and WIDE_CHARS_MODE: + len_fn = wcwidth.wcswidth + else: + len_fn = len + if isinstance(s, _text_type) or isinstance(s, _binary_type): + return len_fn(_strip_invisible(s)) + else: + return len_fn(_text_type(s)) + + +def _align_column(strings, alignment, minwidth=0, has_invisible=True): + """[string] -> [padded_string] + + >>> list(map(str,_align_column( + ... ["12.345", "-1234.5", "1.23", "1234.5", "1e+234", "1.0e234"], + ... "decimal"))) + [' 12.345 ', '-1234.5 ', ' 1.23 ', ' 1234.5 ', ' 1e+234 ', ' 1.0e234'] + + >>> list(map(str,_align_column(['123.4', '56.7890'], None))) + ['123.4', '56.7890'] + + """ + if alignment == "right": + strings = [s.strip() for s in strings] + padfn = _padleft + elif alignment == "center": + strings = [s.strip() for s in strings] + padfn = _padboth + elif alignment == "decimal": + if has_invisible: + decimals = [_afterpoint(_strip_invisible(s)) for s in strings] + else: + decimals = [_afterpoint(s) for s in strings] + maxdecimals = max(decimals) + strings = [s + (maxdecimals - decs) * " " + for s, decs in zip(strings, decimals)] + padfn = _padleft + elif not alignment: + return strings + else: + strings = [s.strip() for s in strings] + padfn = _padright + + enable_widechars = wcwidth is not None and WIDE_CHARS_MODE + if has_invisible: + width_fn = _visible_width + elif enable_widechars: # optional wide-character support if available + width_fn = wcwidth.wcswidth + else: + width_fn = len + + s_lens = list(map(len, strings)) + s_widths = list(map(width_fn, strings)) + maxwidth = max(max(s_widths), minwidth) + if not enable_widechars and not has_invisible: + padded_strings = [padfn(maxwidth, s) for s in strings] + else: + # enable wide-character width corrections + visible_widths = [maxwidth - (w - l) for w, l in zip(s_widths, s_lens)] + # wcswidth and _visible_width don't count invisible characters; + # padfn doesn't need to apply another correction + padded_strings = [padfn(w, s) for s, w in zip(strings, visible_widths)] + return padded_strings + + +def _more_generic(type1, type2): + types = {_none_type: 0, _bool_type: 1, int: 2, float: 3, _binary_type: 4, + _text_type: 5} + invtypes = {5: _text_type, 4: _binary_type, 3: float, 2: int, + 1: _bool_type, 0: _none_type} + moregeneric = max(types.get(type1, 5), types.get(type2, 5)) + return invtypes[moregeneric] + + +def _column_type(strings, has_invisible=True, numparse=True): + """The least generic type all column values are convertible to. + + >>> _column_type([True, False]) is _bool_type + True + >>> _column_type(["1", "2"]) is _int_type + True + >>> _column_type(["1", "2.3"]) is _float_type + True + >>> _column_type(["1", "2.3", "four"]) is _text_type + True + >>> _column_type(["four", '\u043f\u044f\u0442\u044c']) is _text_type + True + >>> _column_type([None, "brux"]) is _text_type + True + >>> _column_type([1, 2, None]) is _int_type + True + >>> import datetime as dt + >>> _column_type([dt.datetime(1991,2,19), dt.time(17,35)]) is _text_type + True + + """ + types = [_type(s, has_invisible, numparse) for s in strings] + return reduce(_more_generic, types, _bool_type) + + +def _format(val, valtype, floatfmt, missingval="", has_invisible=True): + """Format a value accoding to its type. + + Unicode is supported: + + >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', + ... '\u0446\u0438\u0444\u0440\u0430'] + >>> tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] + >>> good_result = ('\\u0431\\u0443\\u043a\\u0432\\u0430 ' + ... '\\u0446\\u0438\\u0444\\u0440\\u0430\\n------- ' + ... '-------\\n\\u0430\\u0437 ' + ... '2\\n\\u0431\\u0443\\u043a\\u0438 4') + >>> tabulate(tbl, headers=hrow) == good_result + True + + """ + if val is None: + return missingval + + if valtype in [int, _text_type]: + return "{0}".format(val) + elif valtype is _binary_type: + try: + return _text_type(val, "ascii") + except TypeError: + return _text_type(val) + elif valtype is float: + is_a_colored_number = (has_invisible and + isinstance(val, (_text_type, _binary_type))) + if is_a_colored_number: + raw_val = _strip_invisible(val) + formatted_val = format(float(raw_val), floatfmt) + return val.replace(raw_val, formatted_val) + else: + return format(float(val), floatfmt) + else: + return "{0}".format(val) + + +def _align_header(header, alignment, width, visible_width): + "Pad string header to width chars given known visible_width of the header." + width += len(header) - visible_width + if alignment == "left": + return _padright(width, header) + elif alignment == "center": + return _padboth(width, header) + elif not alignment: + return "{0}".format(header) + else: + return _padleft(width, header) + + +def _prepend_row_index(rows, index): + """Add a left-most index column.""" + if index is None or index is False: + return rows + if len(index) != len(rows): + print('index=', index) + print('rows=', rows) + raise ValueError('index must be as long as the number of data rows') + rows = [[v]+list(row) for v, row in zip(index, rows)] + return rows + + +def _bool(val): + "A wrapper around standard bool() which doesn't throw on NumPy arrays" + try: + return bool(val) + except ValueError: # val is likely to be a numpy array with many elements + return False + + +def _normalize_tabular_data(tabular_data, headers, showindex="default"): + """Transform a supported data type to a list of lists, and a list of headers. + + Supported tabular data types: + + * list-of-lists or another iterable of iterables + + * list of named tuples (usually used with headers="keys") + + * list of dicts (usually used with headers="keys") + + * list of OrderedDicts (usually used with headers="keys") + + * 2D NumPy arrays + + * NumPy record arrays (usually used with headers="keys") + + * dict of iterables (usually used with headers="keys") + + * pandas.DataFrame (usually used with headers="keys") + + The first row can be used as headers if headers="firstrow", + column indices can be used as headers if headers="keys". + + If showindex="default", show row indices of the pandas.DataFrame. + If showindex="always", show row indices for all types of data. + If showindex="never", don't show row indices for all types of data. + If showindex is an iterable, show its values as row indices. + + """ + + try: + bool(headers) + is_headers2bool_broken = False + except ValueError: # numpy.ndarray, pandas.core.index.Index, ... + is_headers2bool_broken = True + headers = list(headers) + + index = None + if hasattr(tabular_data, "keys") and hasattr(tabular_data, "values"): + # dict-like and pandas.DataFrame? + if hasattr(tabular_data.values, "__call__"): + # likely a conventional dict + keys = tabular_data.keys() + # columns have to be transposed + rows = list(izip_longest(*tabular_data.values())) + elif hasattr(tabular_data, "index"): + # values is a property, has .index => it's likely a + # pandas.DataFrame (pandas 0.11.0) + keys = list(tabular_data) + if tabular_data.index.name is not None: + if isinstance(tabular_data.index.name, list): + keys[:0] = tabular_data.index.name + else: + keys[:0] = [tabular_data.index.name] + # values matrix doesn't need to be transposed + vals = tabular_data.values + # for DataFrames add an index per default + index = list(tabular_data.index) + rows = [list(row) for row in vals] + else: + raise ValueError( + "tabular data doesn't appear to be a dict or a DataFrame") + + if headers == "keys": + headers = list(map(_text_type, keys)) # headers should be strings + + else: # it's a usual an iterable of iterables, or a NumPy array + rows = list(tabular_data) + + if (headers == "keys" and not rows): + # an empty table (issue #81) + headers = [] + elif (headers == "keys" and + hasattr(tabular_data, "dtype") and + getattr(tabular_data.dtype, "names")): + # numpy record array + headers = tabular_data.dtype.names + elif (headers == "keys" + and len(rows) > 0 + and isinstance(rows[0], tuple) + and hasattr(rows[0], "_fields")): + # namedtuple + headers = list(map(_text_type, rows[0]._fields)) + elif (len(rows) > 0 + and isinstance(rows[0], dict)): + # dict or OrderedDict + uniq_keys = set() # implements hashed lookup + keys = [] # storage for set + if headers == "firstrow": + firstdict = rows[0] if len(rows) > 0 else {} + keys.extend(firstdict.keys()) + uniq_keys.update(keys) + rows = rows[1:] + for row in rows: + for k in row.keys(): + # Save unique items in input order + if k not in uniq_keys: + keys.append(k) + uniq_keys.add(k) + if headers == 'keys': + headers = keys + elif isinstance(headers, dict): + # a dict of headers for a list of dicts + headers = [headers.get(k, k) for k in keys] + headers = list(map(_text_type, headers)) + elif headers == "firstrow": + if len(rows) > 0: + headers = [firstdict.get(k, k) for k in keys] + headers = list(map(_text_type, headers)) + else: + headers = [] + elif headers: + raise ValueError( + 'headers for a list of dicts is not a dict or a keyword') + rows = [[row.get(k) for k in keys] for row in rows] + + elif (headers == "keys" + and hasattr(tabular_data, "description") + and hasattr(tabular_data, "fetchone") + and hasattr(tabular_data, "rowcount")): + # Python Database API cursor object (PEP 0249) + # print tabulate(cursor, headers='keys') + headers = [column[0] for column in tabular_data.description] + + elif headers == "keys" and len(rows) > 0: + # keys are column indices + headers = list(map(_text_type, range(len(rows[0])))) + + # take headers from the first row if necessary + if headers == "firstrow" and len(rows) > 0: + if index is not None: + headers = [index[0]] + list(rows[0]) + index = index[1:] + else: + headers = rows[0] + headers = list(map(_text_type, headers)) # headers should be strings + rows = rows[1:] + + headers = list(map(_text_type, headers)) + rows = list(map(list, rows)) + + # add or remove an index column + showindex_is_a_str = type(showindex) in [_text_type, _binary_type] + if showindex == "default" and index is not None: + rows = _prepend_row_index(rows, index) + elif isinstance(showindex, Iterable) and not showindex_is_a_str: + rows = _prepend_row_index(rows, list(showindex)) + elif (showindex == "always" or + (_bool(showindex) and not showindex_is_a_str)): + if index is None: + index = list(range(len(rows))) + rows = _prepend_row_index(rows, index) + elif (showindex == "never" or + (not _bool(showindex) and not showindex_is_a_str)): + pass + + # pad with empty headers for initial columns if necessary + if headers and len(rows) > 0: + nhs = len(headers) + ncols = len(rows[0]) + if nhs < ncols: + headers = [""]*(ncols - nhs) + headers + + return rows, headers + + +def tabulate(tabular_data, headers=(), tablefmt="simple", + floatfmt=_DEFAULT_FLOATFMT, numalign="decimal", stralign="left", + missingval=_DEFAULT_MISSINGVAL, showindex="default", + disable_numparse=False): + """Format a fixed width table for pretty printing. + + >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])) + --- --------- + 1 2.34 + -56 8.999 + 2 10001 + --- --------- + + The first required argument (`tabular_data`) can be a + list-of-lists (or another iterable of iterables), a list of named + tuples, a dictionary of iterables, an iterable of dictionaries, + a two-dimensional NumPy array, NumPy record array, or a Pandas' + dataframe. + + + Table headers + ------------- + + To print nice column headers, supply the second argument (`headers`): + + - `headers` can be an explicit list of column headers + - if `headers="firstrow"`, then the first row of data is used + - if `headers="keys"`, then dictionary keys or column indices are used + + Otherwise a headerless table is produced. + + If the number of headers is less than the number of columns, they + are supposed to be names of the last columns. This is consistent + with the plain-text format of R and Pandas' dataframes. + + >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], + ... headers="firstrow")) + sex age + ----- ----- ----- + Alice F 24 + Bob M 19 + + By default, pandas.DataFrame data have an additional column called + row index. To add a similar column to all other types of data, + use `showindex="always"` or `showindex=True`. To suppress row indices + for all types of data, pass `showindex="never" or `showindex=False`. + To add a custom row index column, pass `showindex=some_iterable`. + + >>> print(tabulate([["F",24],["M",19]], showindex="always")) + - - -- + 0 F 24 + 1 M 19 + - - -- + + + Column alignment + ---------------- + + `tabulate` tries to detect column types automatically, and aligns + the values properly. By default it aligns decimal points of the + numbers (or flushes integer numbers to the right), and flushes + everything else to the left. Possible column alignments + (`numalign`, `stralign`) are: "right", "center", "left", "decimal" + (only for `numalign`), and None (to disable alignment). + + + Table formats + ------------- + + `floatfmt` is a format specification used for columns which + contain numeric data with a decimal point. This can also be + a list or tuple of format strings, one per column. + + `None` values are replaced with a `missingval` string (like + `floatfmt`, this can also be a list of values for different + columns): + + >>> print(tabulate([["spam", 1, None], + ... ["eggs", 42, 3.14], + ... ["other", None, 2.7]], missingval="?")) + ----- -- ---- + spam 1 ? + eggs 42 3.14 + other ? 2.7 + ----- -- ---- + + Various plain-text table formats (`tablefmt`) are supported: + 'plain', 'simple', 'grid', 'pipe', 'orgtbl', 'rst', 'mediawiki', + 'latex', 'latex_raw' and 'latex_booktabs'. Variable `tabulate_formats` + contains the list of currently supported formats. + + "plain" format doesn't use any pseudographics to draw tables, + it separates columns with a double space: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "plain")) + strings numbers + spam 41.9999 + eggs 451 + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="plain")) + spam 41.9999 + eggs 451 + + "simple" format is like Pandoc simple_tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "simple")) + strings numbers + --------- --------- + spam 41.9999 + eggs 451 + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="simple")) + ---- -------- + spam 41.9999 + eggs 451 + ---- -------- + + "grid" is similar to tables produced by Emacs table.el package or + Pandoc grid_tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "grid")) + +-----------+-----------+ + | strings | numbers | + +===========+===========+ + | spam | 41.9999 | + +-----------+-----------+ + | eggs | 451 | + +-----------+-----------+ + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="grid")) + +------+----------+ + | spam | 41.9999 | + +------+----------+ + | eggs | 451 | + +------+----------+ + + "fancy_grid" draws a grid using box-drawing characters: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "fancy_grid")) + ╒═══════════╤═══════════╕ + │ strings │ numbers │ + ╞═══════════╪═══════════╡ + │ spam │ 41.9999 │ + ├───────────┼───────────┤ + │ eggs │ 451 │ + ╘═══════════╧═══════════╛ + + "pipe" is like tables in PHP Markdown Extra extension or Pandoc + pipe_tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "pipe")) + | strings | numbers | + |:----------|----------:| + | spam | 41.9999 | + | eggs | 451 | + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="pipe")) + |:-----|---------:| + | spam | 41.9999 | + | eggs | 451 | + + "orgtbl" is like tables in Emacs org-mode and orgtbl-mode. They + are slightly different from "pipe" format by not using colons to + define column alignment, and using a "+" sign to indicate line + intersections: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "orgtbl")) + | strings | numbers | + |-----------+-----------| + | spam | 41.9999 | + | eggs | 451 | + + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="orgtbl")) + | spam | 41.9999 | + | eggs | 451 | + + "rst" is like a simple table format from reStructuredText; please + note that reStructuredText accepts also "grid" tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "rst")) + ========= ========= + strings numbers + ========= ========= + spam 41.9999 + eggs 451 + ========= ========= + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")) + ==== ======== + spam 41.9999 + eggs 451 + ==== ======== + + "mediawiki" produces a table markup used in Wikipedia and on other + MediaWiki-based sites: + + >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], + ... ["eggs", "451.0"]], headers="firstrow", + ... tablefmt="mediawiki")) + {| class="wikitable" style="text-align: left;" + |+ + |- + ! strings !! align="right"| numbers + |- + | spam || align="right"| 41.9999 + |- + | eggs || align="right"| 451 + |} + + "html" produces HTML markup: + + >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], + ... ["eggs", "451.0"]], headers="firstrow", + ... tablefmt="html")) + + + + + + + + +
strings numbers
spam 41.9999
eggs 451
+ + "latex" produces a tabular environment of LaTeX document markup: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="latex")) + \\begin{tabular}{lr} + \\hline + spam & 41.9999 \\\\ + eggs & 451 \\\\ + \\hline + \\end{tabular} + + "latex_raw" is similar to "latex", but doesn't escape special characters, + such as backslash and underscore, so LaTeX commands may embedded into + cells' values: + + >>> print(tabulate([["spam$_9$", 41.9999], ["\\\\emph{eggs}", "451.0"]], + ... tablefmt="latex_raw")) + \\begin{tabular}{lr} + \\hline + spam$_9$ & 41.9999 \\\\ + \\emph{eggs} & 451 \\\\ + \\hline + \\end{tabular} + + "latex_booktabs" produces a tabular environment of LaTeX document markup + using the booktabs.sty package: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="latex_booktabs")) + \\begin{tabular}{lr} + \\toprule + spam & 41.9999 \\\\ + eggs & 451 \\\\ + \\bottomrule + \end{tabular} + + Number parsing + -------------- + By default, anything which can be parsed as a number is a number. + This ensures numbers represented as strings are aligned properly. + This can lead to weird results for particular strings such as + specific git SHAs e.g. "42992e1" will be parsed into the number + 429920 and aligned as such. + + To completely disable number parsing (and alignment), use + `disable_numparse=True`. For more fine grained control, a list column + indices is used to disable number parsing only on those columns + e.g. `disable_numparse=[0, 2]` would disable number parsing only on the + first and third columns. + """ + if tabular_data is None: + tabular_data = [] + list_of_lists, headers = _normalize_tabular_data( + tabular_data, headers, showindex=showindex) + + # empty values in the first column of RST tables should be escaped + # (issue #82). "" should be escaped as "\\ " or ".." + if tablefmt == 'rst': + list_of_lists, headers = _rst_escape_first_column(list_of_lists, + headers) + + # optimization: look for ANSI control codes once, + # enable smart width functions only if a control code is found + plain_text = '\n'.join(['\t'.join(map(_text_type, headers))] + + ['\t'.join(map(_text_type, row)) + for row in list_of_lists]) + + has_invisible = re.search(_invisible_codes, plain_text) + enable_widechars = wcwidth is not None and WIDE_CHARS_MODE + if has_invisible: + width_fn = _visible_width + elif enable_widechars: # optional wide-character support if available + width_fn = wcwidth.wcswidth + else: + width_fn = len + + # format rows and columns, convert numeric values to strings + cols = list(izip_longest(*list_of_lists)) + numparses = _expand_numparse(disable_numparse, len(cols)) + coltypes = [_column_type(col, numparse=np) for col, np in + zip(cols, numparses)] + if isinstance(floatfmt, basestring): # old version + # just duplicate the string to use in each column + float_formats = len(cols) * [floatfmt] + else: # if floatfmt is list, tuple etc we have one per column + float_formats = list(floatfmt) + if len(float_formats) < len(cols): + float_formats.extend((len(cols)-len(float_formats)) * + [_DEFAULT_FLOATFMT]) + if isinstance(missingval, basestring): + missing_vals = len(cols) * [missingval] + else: + missing_vals = list(missingval) + if len(missing_vals) < len(cols): + missing_vals.extend((len(cols)-len(missing_vals)) * + [_DEFAULT_MISSINGVAL]) + cols = [[_format(v, ct, fl_fmt, miss_v, has_invisible) for v in c] + for c, ct, fl_fmt, miss_v in zip(cols, coltypes, float_formats, + missing_vals)] + + # align columns + aligns = [numalign if ct in [int, float] else stralign for ct in coltypes] + minwidths = [width_fn(h) + MIN_PADDING + for h in headers] if headers else [0]*len(cols) + cols = [_align_column(c, a, minw, has_invisible) + for c, a, minw in zip(cols, aligns, minwidths)] + + if headers: + # align headers and add headers + t_cols = cols or [['']] * len(headers) + t_aligns = aligns or [stralign] * len(headers) + minwidths = [max(minw, width_fn(c[0])) + for minw, c in zip(minwidths, t_cols)] + headers = [_align_header(h, a, minw, width_fn(h)) + for h, a, minw in zip(headers, t_aligns, minwidths)] + rows = list(zip(*cols)) + else: + minwidths = [width_fn(c[0]) for c in cols] + rows = list(zip(*cols)) + + if not isinstance(tablefmt, TableFormat): + tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) + + return _format_table(tablefmt, headers, rows, minwidths, aligns) + + +def _expand_numparse(disable_numparse, column_count): + """ + Return a list of bools of length `column_count` which indicates whether + number parsing should be used on each column. + If `disable_numparse` is a list of indices, each of those indices are + False, and everything else is True. + If `disable_numparse` is a bool, then the returned list is all the same. + """ + if isinstance(disable_numparse, Iterable): + numparses = [True] * column_count + for index in disable_numparse: + numparses[index] = False + return numparses + else: + return [not disable_numparse] * column_count + + +def _build_simple_row(padded_cells, rowfmt): + "Format row according to DataRow format without padding." + begin, sep, end = rowfmt + return (begin + sep.join(padded_cells) + end).rstrip() + + +def _build_row(padded_cells, colwidths, colaligns, rowfmt): + "Return a string which represents a row of data cells." + if not rowfmt: + return None + if hasattr(rowfmt, "__call__"): + return rowfmt(padded_cells, colwidths, colaligns) + else: + return _build_simple_row(padded_cells, rowfmt) + + +def _build_line(colwidths, colaligns, linefmt): + "Return a string which represents a horizontal line." + if not linefmt: + return None + if hasattr(linefmt, "__call__"): + return linefmt(colwidths, colaligns) + else: + begin, fill, sep, end = linefmt + cells = [fill*w for w in colwidths] + return _build_simple_row(cells, (begin, sep, end)) + + +def _pad_row(cells, padding): + if cells: + pad = " "*padding + padded_cells = [pad + cell + pad for cell in cells] + return padded_cells + else: + return cells + + +def _format_table(fmt, headers, rows, colwidths, colaligns): + """Produce a plain-text representation of the table.""" + lines = [] + hidden = fmt.with_header_hide if (headers and fmt.with_header_hide) else [] + pad = fmt.padding + headerrow = fmt.headerrow + + padded_widths = [(w + 2*pad) for w in colwidths] + padded_headers = _pad_row(headers, pad) + padded_rows = [_pad_row(row, pad) for row in rows] + + if fmt.lineabove and "lineabove" not in hidden: + lines.append(_build_line(padded_widths, colaligns, fmt.lineabove)) + + if padded_headers: + lines.append(_build_row(padded_headers, padded_widths, colaligns, + headerrow)) + if fmt.linebelowheader and "linebelowheader" not in hidden: + lines.append(_build_line(padded_widths, colaligns, + fmt.linebelowheader)) + + if padded_rows and fmt.linebetweenrows and "linebetweenrows" not in hidden: + # initial rows with a line below + for row in padded_rows[:-1]: + lines.append(_build_row(row, padded_widths, colaligns, + fmt.datarow)) + lines.append(_build_line(padded_widths, colaligns, + fmt.linebetweenrows)) + # the last row without a line below + lines.append(_build_row(padded_rows[-1], padded_widths, colaligns, + fmt.datarow)) + else: + for row in padded_rows: + lines.append(_build_row(row, padded_widths, colaligns, + fmt.datarow)) + + if fmt.linebelow and "linebelow" not in hidden: + lines.append(_build_line(padded_widths, colaligns, fmt.linebelow)) + + if headers or rows: + return "\n".join(lines) + else: # a completely empty table + return "" + + +def _main(): + """\ + Usage: tabulate [options] [FILE ...] + + Pretty-print tabular data. + See also https://bitbucket.org/astanin/python-tabulate + + FILE a filename of the file with tabular data; + if "-" or missing, read data from stdin. + + Options: + + -h, --help show this message + -1, --header use the first row of data as a table header + -o FILE, --output FILE print table to FILE (default: stdout) + -s REGEXP, --sep REGEXP use a custom column separator (default: whitespace) + -F FPFMT, --float FPFMT floating point number format (default: g) + -f FMT, --format FMT set output table format; supported formats: + plain, simple, grid, fancy_grid, pipe, orgtbl, + rst, mediawiki, html, latex, latex_raw, + latex_booktabs, tsv + (default: simple) + """ + import getopt + import sys + import textwrap + usage = textwrap.dedent(_main.__doc__) + try: + opts, args = getopt.getopt( + sys.argv[1:], "h1o:s:F:f:", + ["help", "header", "output", "sep=", "float=", "format="]) + except getopt.GetoptError as e: + print(e) + print(usage) + sys.exit(2) + headers = [] + floatfmt = _DEFAULT_FLOATFMT + tablefmt = "simple" + sep = r"\s+" + outfile = "-" + for opt, value in opts: + if opt in ["-1", "--header"]: + headers = "firstrow" + elif opt in ["-o", "--output"]: + outfile = value + elif opt in ["-F", "--float"]: + floatfmt = value + elif opt in ["-f", "--format"]: + if value not in tabulate_formats: + print("%s is not a supported table format" % value) + print(usage) + sys.exit(3) + tablefmt = value + elif opt in ["-s", "--sep"]: + sep = value + elif opt in ["-h", "--help"]: + print(usage) + sys.exit(0) + files = [sys.stdin] if not args else args + with (sys.stdout if outfile == "-" else open(outfile, "w")) as out: + for f in files: + if f == "-": + f = sys.stdin + if _is_file(f): + _pprint_file(f, headers=headers, tablefmt=tablefmt, + sep=sep, floatfmt=floatfmt, file=out) + else: + with open(f) as fobj: + _pprint_file(fobj, headers=headers, tablefmt=tablefmt, + sep=sep, floatfmt=floatfmt, file=out) + + +def _pprint_file(fobject, headers, tablefmt, sep, floatfmt, file): + rows = fobject.readlines() + table = [re.split(sep, r.rstrip()) for r in rows if r.strip()] + print(tabulate(table, headers, tablefmt, floatfmt=floatfmt), file=file) + + +if __name__ == "__main__": + _main() diff --git a/setup.py b/setup.py index 21e555086..417929c63 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', 'pycryptodome >= 3', - 'tabulate >= 0.7.6', 'terminaltables >= 3.0.0', ] From 4c9060569d23fe177324387733a2a1c17a9532e2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:13:30 -0500 Subject: [PATCH 169/824] Add preserve whitespace option to tabulate. --- mycli/output_formatter/tabulate_adapter.py | 2 ++ mycli/packages/tabulate.py | 10 +++++++--- tests/test_tabulate.py | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 tests/test_tabulate.py diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 14e6c770f..c9b1acd2a 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,6 +1,8 @@ from mycli.packages import tabulate from .preprocessors import bytes_to_string, align_decimals, quote_whitespaces +tabulate.PRESERVE_WHITESPACE = True + supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', 'html', 'latex', 'latex_booktabs', 'textile') diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index a25032995..292f40bb0 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -53,6 +53,7 @@ def _is_file(f): # minimum extra space in headers MIN_PADDING = 2 +PRESERVE_WHITESPACE = False _DEFAULT_FLOATFMT = "g" _DEFAULT_MISSINGVAL = "" @@ -563,10 +564,12 @@ def _align_column(strings, alignment, minwidth=0, has_invisible=True): """ if alignment == "right": - strings = [s.strip() for s in strings] + if not PRESERVE_WHITESPACE: + strings = [s.strip() for s in strings] padfn = _padleft elif alignment == "center": - strings = [s.strip() for s in strings] + if not PRESERVE_WHITESPACE: + strings = [s.strip() for s in strings] padfn = _padboth elif alignment == "decimal": if has_invisible: @@ -580,7 +583,8 @@ def _align_column(strings, alignment, minwidth=0, has_invisible=True): elif not alignment: return strings else: - strings = [s.strip() for s in strings] + if not PRESERVE_WHITESPACE: + strings = [s.strip() for s in strings] padfn = _padright enable_widechars = wcwidth is not None and WIDE_CHARS_MODE diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py new file mode 100644 index 000000000..ae7c25ce1 --- /dev/null +++ b/tests/test_tabulate.py @@ -0,0 +1,17 @@ +from textwrap import dedent + +from mycli.packages import tabulate + +tabulate.PRESERVE_WHITESPACE = True + + +def test_dont_strip_leading_whitespace(): + data = [[' abc']] + headers = ['xyz'] + tbl = tabulate.tabulate(data, headers, tablefmt='psql') + assert tbl == dedent(''' + +---------+ + | xyz | + |---------| + | abc | + +---------+ ''').strip() From 694919eae6b1a4489e61c8f8adfc390b05ec6f55 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:17:12 -0500 Subject: [PATCH 170/824] Remove extra html format. --- mycli/output_formatter/tabulate_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index c9b1acd2a..fc6a7d8d5 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -5,7 +5,7 @@ supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', - 'html', 'latex', 'latex_booktabs', 'textile') + 'latex', 'latex_booktabs', 'textile') preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) From 85d0e026278b7f0f2b044b483412832beae205d8 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:29:49 -0500 Subject: [PATCH 171/824] Do not align columns for markup tables. --- mycli/output_formatter/tabulate_adapter.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index fc6a7d8d5..7b4dfac3f 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -3,14 +3,20 @@ tabulate.PRESERVE_WHITESPACE = True -supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', - 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', - 'latex', 'latex_booktabs', 'textile') +supported_markup_formats = ('mediawiki', 'html', 'latex', 'latex_booktabs', + 'textile', 'moinmoin', 'jira') +supported_table_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', + 'orgtbl', 'psql', 'rst') +supported_formats = supported_markup_formats + supported_table_formats preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate.tabulate(data, headers, tablefmt=table_format, - missingval=missing_value, disable_numparse=True) + kwargs = {'tablefmt': table_format, 'missingval': missing_value, + 'disable_numparse': True} + if table_format in supported_markup_formats: + kwargs.update(numalign=None, stralign=None) + + return tabulate.tabulate(data, headers, **kwargs) From ae8311c5424a9dc4632364d671e8615d38d44d67 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:30:51 -0500 Subject: [PATCH 172/824] Do not quote whitespaces for tabulate formats. --- mycli/output_formatter/tabulate_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 7b4dfac3f..26ef40aa2 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -9,7 +9,7 @@ 'orgtbl', 'psql', 'rst') supported_formats = supported_markup_formats + supported_table_formats -preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) +preprocessors = (bytes_to_string, align_decimals) def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): From 359044c8e87ed0e8394eac2dacb3e0d53cc498c5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:47:42 -0500 Subject: [PATCH 173/824] Pep8radius changes. --- mycli/packages/tabulate.py | 41 +++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index 292f40bb0..ab3796f66 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -180,9 +180,9 @@ def _moin_row_with_attrs(celltag, cell_values, colwidths, colaligns, "decimal": ''} values_with_attrs = ["{0}{1} {2} ".format(celltag, alignment.get(a, ''), - header+c+header) + header + c + header) for c, a in zip(cell_values, colaligns)] - return "".join(values_with_attrs)+"||" + return "".join(values_with_attrs) + "||" def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): @@ -684,7 +684,8 @@ def _format(val, valtype, floatfmt, missingval="", has_invisible=True): def _align_header(header, alignment, width, visible_width): - "Pad string header to width chars given known visible_width of the header." + """Pad string header to width chars given known visible_width of the + header.""" width += len(header) - visible_width if alignment == "left": return _padright(width, header) @@ -704,12 +705,13 @@ def _prepend_row_index(rows, index): print('index=', index) print('rows=', rows) raise ValueError('index must be as long as the number of data rows') - rows = [[v]+list(row) for v, row in zip(index, rows)] + rows = [[v] + list(row) for v, row in zip(index, rows)] return rows def _bool(val): - "A wrapper around standard bool() which doesn't throw on NumPy arrays" + """A wrapper around standard bool() which doesn't throw on NumPy + arrays.""" try: return bool(val) except ValueError: # val is likely to be a numpy array with many elements @@ -717,7 +719,8 @@ def _bool(val): def _normalize_tabular_data(tabular_data, headers, showindex="default"): - """Transform a supported data type to a list of lists, and a list of headers. + """Transform a supported data type to a list of lists, and a list of + headers. Supported tabular data types: @@ -878,7 +881,7 @@ def _normalize_tabular_data(tabular_data, headers, showindex="default"): nhs = len(headers) ncols = len(rows[0]) if nhs < ncols: - headers = [""]*(ncols - nhs) + headers + headers = [""] * (ncols - nhs) + headers return rows, headers @@ -1169,11 +1172,12 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", indices is used to disable number parsing only on those columns e.g. `disable_numparse=[0, 2]` would disable number parsing only on the first and third columns. + """ if tabular_data is None: tabular_data = [] list_of_lists, headers = _normalize_tabular_data( - tabular_data, headers, showindex=showindex) + tabular_data, headers, showindex=showindex) # empty values in the first column of RST tables should be escaped # (issue #82). "" should be escaped as "\\ " or ".." @@ -1207,14 +1211,14 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", else: # if floatfmt is list, tuple etc we have one per column float_formats = list(floatfmt) if len(float_formats) < len(cols): - float_formats.extend((len(cols)-len(float_formats)) * + float_formats.extend((len(cols) - len(float_formats)) * [_DEFAULT_FLOATFMT]) if isinstance(missingval, basestring): missing_vals = len(cols) * [missingval] else: missing_vals = list(missingval) if len(missing_vals) < len(cols): - missing_vals.extend((len(cols)-len(missing_vals)) * + missing_vals.extend((len(cols) - len(missing_vals)) * [_DEFAULT_MISSINGVAL]) cols = [[_format(v, ct, fl_fmt, miss_v, has_invisible) for v in c] for c, ct, fl_fmt, miss_v in zip(cols, coltypes, float_formats, @@ -1223,7 +1227,7 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", # align columns aligns = [numalign if ct in [int, float] else stralign for ct in coltypes] minwidths = [width_fn(h) + MIN_PADDING - for h in headers] if headers else [0]*len(cols) + for h in headers] if headers else [0] * len(cols) cols = [_align_column(c, a, minw, has_invisible) for c, a, minw in zip(cols, aligns, minwidths)] @@ -1247,12 +1251,13 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", def _expand_numparse(disable_numparse, column_count): - """ - Return a list of bools of length `column_count` which indicates whether + """Return a list of bools of length `column_count` which indicates whether number parsing should be used on each column. - If `disable_numparse` is a list of indices, each of those indices are - False, and everything else is True. - If `disable_numparse` is a bool, then the returned list is all the same. + + If `disable_numparse` is a list of indices, each of those indices + are False, and everything else is True. If `disable_numparse` is a + bool, then the returned list is all the same. + """ if isinstance(disable_numparse, Iterable): numparses = [True] * column_count @@ -1346,8 +1351,7 @@ def _format_table(fmt, headers, rows, colwidths, colaligns): def _main(): - """\ - Usage: tabulate [options] [FILE ...] + """\ Usage: tabulate [options] [FILE ...] Pretty-print tabular data. See also https://bitbucket.org/astanin/python-tabulate @@ -1367,6 +1371,7 @@ def _main(): rst, mediawiki, html, latex, latex_raw, latex_booktabs, tsv (default: simple) + """ import getopt import sys From 13991b193a2d764a7baa16937193f37a5bdc3c6e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 16:06:23 -0500 Subject: [PATCH 174/824] Add output format changes to changelog. --- changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index eb36b6bab..5bda9ae72 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,8 @@ Features: --------- * Add ability to specify alternative myclirc file. (Thanks: [Dick Marinus]). +* Add new display formats for pretty printing query results. (Thanks: [Amjith + Ramanujam], [Dick Marinus], [Thomas Roten]). Bug Fixes: ---------- From 33e5c26eba1f3261b66d603fcbcb66b8de8f76e2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 16:11:03 -0500 Subject: [PATCH 175/824] Add additional table formats to myclirc. --- mycli/myclirc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mycli/myclirc b/mycli/myclirc index 2f3599273..febd7e591 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -30,7 +30,9 @@ log_level = INFO # Timing of sql statments and table rendering. timing = True -# Table format. Possible values: ascii, single, double, or github. +# Table format. Possible values: ascii, single, double, github, +# psql, plain, simple, grid, fancy_grid, pipe, orgtbl, rst, mediawiki, html, +# latex, latex_booktabs, textile, moinmoin, jira, expanded, tsv, csv. # Recommended: ascii table_format = ascii From f06451cb4c12baf30f32d8fd0be3da0042d2f23c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 16:16:13 -0500 Subject: [PATCH 176/824] Don't import quote_whitespaces since it's not used. --- mycli/output_formatter/tabulate_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 26ef40aa2..ee63b5b1d 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,5 +1,5 @@ from mycli.packages import tabulate -from .preprocessors import bytes_to_string, align_decimals, quote_whitespaces +from .preprocessors import bytes_to_string, align_decimals tabulate.PRESERVE_WHITESPACE = True From 8f70b12ef9db79e2610eba5c8c45a0b92d07fc93 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 21:11:04 -0500 Subject: [PATCH 177/824] Fix outdated docstring. --- mycli/output_formatter/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 759fb879a..9f9b6a8e5 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -27,7 +27,7 @@ class OutputFormatter(object): _output_formats = {} def __init__(self, format_name=None): - """Register the supported output formats.""" + """Set the default *format_name*.""" self._format_name = format_name def set_format_name(self, format_name): From fd861d41e041b08da83dface8b6d8a0843f3b2fa Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 21:30:21 -0500 Subject: [PATCH 178/824] Make SQLCompleter options pass through refresher. --- mycli/completion_refresher.py | 14 ++++++++------ mycli/main.py | 11 +++++++---- tests/test_completion_refresher.py | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index 33afa009c..d24e03a80 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -13,7 +13,7 @@ def __init__(self): self._completer_thread = None self._restart_refresh = threading.Event() - def refresh(self, executor, callbacks): + def refresh(self, executor, callbacks, completer_options={}): """ Creates a SQLCompleter object and populates it with the relevant completion suggestions in a background thread. @@ -23,14 +23,16 @@ def refresh(self, executor, callbacks): callbacks - A function or a list of functions to call after the thread has completed the refresh. The newly created completion object will be passed in as an argument to each callback. + completer_options - dict of options to pass to SQLCompleter. """ if self.is_refreshing(): self._restart_refresh.set() return [(None, None, None, 'Auto-completion refresh restarted.')] else: - self._completer_thread = threading.Thread(target=self._bg_refresh, - args=(executor, callbacks), - name='completion_refresh') + self._completer_thread = threading.Thread( + target=self._bg_refresh, + args=(executor, callbacks, completer_options), + name='completion_refresh') self._completer_thread.setDaemon(True) self._completer_thread.start() return [(None, None, None, @@ -39,8 +41,8 @@ def refresh(self, executor, callbacks): def is_refreshing(self): return self._completer_thread and self._completer_thread.is_alive() - def _bg_refresh(self, sqlexecute, callbacks): - completer = SQLCompleter(smart_completion=True) + def _bg_refresh(self, sqlexecute, callbacks, completer_options): + completer = SQLCompleter(**completer_options) # Create a new pgexecute method to popoulate the completions. e = sqlexecute diff --git a/mycli/main.py b/mycli/main.py index 64c587057..2a9b04364 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -146,8 +146,9 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. self.smart_completion = c['main'].as_bool('smart_completion') - self.completer = SQLCompleter(self.smart_completion, - supported_formats=self.formatter.supported_formats) + self.completer = SQLCompleter( + self.smart_completion, + supported_formats=self.formatter.supported_formats()) self._completer_lock = threading.Lock() # Register custom special commands. @@ -665,8 +666,10 @@ def refresh_completions(self, reset=False): if reset: with self._completer_lock: self.completer.reset_completions() - self.completion_refresher.refresh(self.sqlexecute, - self._on_completions_refreshed) + self.completion_refresher.refresh( + self.sqlexecute, self._on_completions_refreshed, + {'smart_completion': self.smart_completion, + 'supported_formats': self.formatter.supported_formats()}) return [(None, None, None, 'Auto-completion refresh started in the background.')] diff --git a/tests/test_completion_refresher.py b/tests/test_completion_refresher.py index a50ad7492..8851eae65 100644 --- a/tests/test_completion_refresher.py +++ b/tests/test_completion_refresher.py @@ -37,7 +37,7 @@ def test_refresh_called_once(refresher): assert len(actual) == 1 assert len(actual[0]) == 4 assert actual[0][3] == 'Auto-completion refresh started in the background.' - bg_refresh.assert_called_with(sqlexecute, callbacks) + bg_refresh.assert_called_with(sqlexecute, callbacks, {}) def test_refresh_called_twice(refresher): From 81c54962c530e1656e0d6575829a67d913daa958 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 21:33:02 -0500 Subject: [PATCH 179/824] Pep8radius fixes. --- mycli/completion_refresher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index d24e03a80..2bbe32d0e 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -14,8 +14,7 @@ def __init__(self): self._restart_refresh = threading.Event() def refresh(self, executor, callbacks, completer_options={}): - """ - Creates a SQLCompleter object and populates it with the relevant + """Creates a SQLCompleter object and populates it with the relevant completion suggestions in a background thread. executor - SQLExecute object, used to extract the credentials to connect @@ -24,6 +23,7 @@ def refresh(self, executor, callbacks, completer_options={}): has completed the refresh. The newly created completion object will be passed in as an argument to each callback. completer_options - dict of options to pass to SQLCompleter. + """ if self.is_refreshing(): self._restart_refresh.set() From 9a23584a03a4e29eaff755768e04cc2589ac90ad Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 08:47:16 -0500 Subject: [PATCH 180/824] Fix fancy_grid table format. --- mycli/packages/tabulate.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index ab3796f66..1e67cea7d 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -252,12 +252,12 @@ def escape_empty(val): padding=1, with_header_hide=None), "fancy_grid": TableFormat( - lineabove=Line("â•’", "═", "╤", "â••"), - linebelowheader=Line("â•ž", "═", "╪", "â•¡"), - linebetweenrows=Line("├", "─", "┼", "┤"), - linebelow=Line("╘", "═", "╧", "â•›"), - headerrow=DataRow("│", "│", "│"), - datarow=DataRow("│", "│", "│"), + lineabove=Line("╒", "═", "╤", "╕"), + linebelowheader=Line("╞", "═", "╪", "╡"), + linebetweenrows=Line("├", "─", "┼", "┤"), + linebelow=Line("╘", "═", "╧", "╛"), + headerrow=DataRow("│", "│", "│"), + datarow=DataRow("│", "│", "│"), padding=1, with_header_hide=None), "pipe": TableFormat( @@ -1032,13 +1032,13 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], ... ["strings", "numbers"], "fancy_grid")) - ╒═══════════╤═══════════╕ - │ strings │ numbers │ - ╞═══════════╪═══════════╡ - │ spam │ 41.9999 │ - ├───────────┼───────────┤ - │ eggs │ 451 │ - ╘═══════════╧═══════════╛ + ╒═══════════╤═══════════╕ + │ strings │ numbers │ + ╞═══════════╪═══════════╡ + │ spam │ 41.9999 │ + ├───────────┼───────────┤ + │ eggs │ 451 │ + ╘═══════════╧═══════════╛ "pipe" is like tables in PHP Markdown Extra extension or Pandoc pipe_tables: From 4b0de47ebe3e9c91cf6136d8ee28fb59fbf92ee2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 13:40:53 -0500 Subject: [PATCH 181/824] Remove single terminaltable format since it's causing problems. --- mycli/myclirc | 2 +- mycli/output_formatter/terminaltables_adapter.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mycli/myclirc b/mycli/myclirc index febd7e591..01a114265 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -30,7 +30,7 @@ log_level = INFO # Timing of sql statments and table rendering. timing = True -# Table format. Possible values: ascii, single, double, github, +# Table format. Possible values: ascii, double, github, # psql, plain, simple, grid, fancy_grid, pipe, orgtbl, rst, mediawiki, html, # latex, latex_booktabs, textile, moinmoin, jira, expanded, tsv, csv. # Recommended: ascii diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py index ac580517f..0c702abe0 100644 --- a/mycli/output_formatter/terminaltables_adapter.py +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -3,7 +3,7 @@ from .preprocessors import (bytes_to_string, align_decimals, override_missing_value) -supported_formats = ('ascii', 'single', 'double', 'github') +supported_formats = ('ascii', 'double', 'github') preprocessors = (bytes_to_string, override_missing_value, align_decimals) @@ -12,7 +12,6 @@ def terminaltables_adapter(data, headers, table_format=None, **_): table_format_handler = { 'ascii': terminaltables.AsciiTable, - 'single': terminaltables.SingleTable, 'double': terminaltables.DoubleTable, 'github': terminaltables.GithubFlavoredMarkdownTable, } From cc76dee6198b759fd0219bff9cafc383a450456b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 14:27:19 -0500 Subject: [PATCH 182/824] Use constant for missing value default. --- mycli/output_formatter/output_formatter.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 9f9b6a8e5..3e53d826d 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -16,6 +16,8 @@ terminaltables_adapter, preprocessors as terminaltables_preprocessors, supported_formats as terminaltables_formats) +MISSING_VALUE = '' + OutputFormatHandler = namedtuple( 'OutputFormatHandler', 'format_name preprocessors formatter formatter_args') @@ -77,22 +79,22 @@ def format_output(self, data, headers, format_name=None, **kwargs): OutputFormatter.register_new_formatter('expanded', expanded_table, (override_missing_value, convert_to_string), - {'missing_value': ''}) + {'missing_value': MISSING_VALUE}) for delimiter_format in delimiter_formats: OutputFormatter.register_new_formatter(delimiter_format, delimiter_adapter, delimiter_preprocessors, {'table_format': delimiter_format, - 'missing_value': ''}) + 'missing_value': MISSING_VALUE}) for tabulate_format in tabulate_formats: OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, tabulate_preprocessors, {'table_format': tabulate_format, - 'missing_value': ''}) + 'missing_value': MISSING_VALUE}) for terminaltables_format in terminaltables_formats: OutputFormatter.register_new_formatter( terminaltables_format, terminaltables_adapter, terminaltables_preprocessors, - {'table_format': terminaltables_format, 'missing_value': ''}) + {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) From 39c4aec92645306a3c80ad11ec9213c918437284 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 14:38:38 -0500 Subject: [PATCH 183/824] Make registering a format not require kwargs. --- mycli/output_formatter/output_formatter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 3e53d826d..c87da5edd 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -49,8 +49,8 @@ def supported_formats(self): return tuple(self._output_formats.keys()) @classmethod - def register_new_formatter(cls, format_name, handler, preprocessors=None, - kwargs=None): + def register_new_formatter(cls, format_name, handler, preprocessors=(), + kwargs={}): """Register a new formatter to format the output.""" cls._output_formats[format_name] = OutputFormatHandler( format_name, preprocessors, handler, kwargs) From 21c600256de4c01fb4c4aa97b84ebd28f3f81c9f Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Fri, 14 Apr 2017 21:22:22 +0200 Subject: [PATCH 184/824] remove temporary hack --- changelog.md | 1 + mycli/packages/completion_engine.py | 43 +++++++++++++---------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/changelog.md b/changelog.md index fcf9c198d..8ac468380 100644 --- a/changelog.md +++ b/changelog.md @@ -23,6 +23,7 @@ Internal Changes: Roten]). * Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). * Run pep8 checks in travis (Thanks: [Irina Truong]). +* Remove temporary hack for sqlparse (Thanks: [Dick Marinus]). 1.9.0: ====== diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 6e2165dca..bd6a7b89c 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -28,32 +28,27 @@ def suggest_type(full_text, text_before_cursor): identifier = None - # This is a temporary hack; the exception handling - # here should be removed once sqlparse has been fixed - try: - # If we've partially typed a word then word_before_cursor won't be an empty - # string. In that case we want to remove the partially typed string before - # sending it to the sqlparser. Otherwise the last token will always be the - # partially typed string which renders the smart completion useless because - # it will always return the list of keywords as completion. - if word_before_cursor: - if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': - parsed = sqlparse.parse(text_before_cursor) - else: - parsed = sqlparse.parse( - text_before_cursor[:-len(word_before_cursor)]) + # If we've partially typed a word then word_before_cursor won't be an empty + # string. In that case we want to remove the partially typed string before + # sending it to the sqlparser. Otherwise the last token will always be the + # partially typed string which renders the smart completion useless because + # it will always return the list of keywords as completion. + if word_before_cursor: + if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': + parsed = sqlparse.parse(text_before_cursor) + else: + parsed = sqlparse.parse( + text_before_cursor[:-len(word_before_cursor)]) - # word_before_cursor may include a schema qualification, like - # "schema_name.partial_name" or "schema_name.", so parse it - # separately - p = sqlparse.parse(word_before_cursor)[0] + # word_before_cursor may include a schema qualification, like + # "schema_name.partial_name" or "schema_name.", so parse it + # separately + p = sqlparse.parse(word_before_cursor)[0] - if p.tokens and isinstance(p.tokens[0], Identifier): - identifier = p.tokens[0] - else: - parsed = sqlparse.parse(text_before_cursor) - except (TypeError, AttributeError): - return [] + if p.tokens and isinstance(p.tokens[0], Identifier): + identifier = p.tokens[0] + else: + parsed = sqlparse.parse(text_before_cursor) if len(parsed) > 1: # Multiple statements being edited -- isolate the current one by From 5d5cf7a46a391a81bdd62042bad5dddfdf594755 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 08:26:56 +0200 Subject: [PATCH 185/824] make the code a bit more pythonic --- mycli/packages/completion_engine.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index bd6a7b89c..b97cadf71 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -34,7 +34,8 @@ def suggest_type(full_text, text_before_cursor): # partially typed string which renders the smart completion useless because # it will always return the list of keywords as completion. if word_before_cursor: - if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': + if word_before_cursor.endswith( + '(') or word_before_cursor.startswith('\\'): parsed = sqlparse.parse(text_before_cursor) else: parsed = sqlparse.parse( From 034c7ae23766abbdb4c4944ca58b10a8684f5b8c Mon Sep 17 00:00:00 2001 From: "John K. Sterling" Date: Sat, 15 Apr 2017 09:50:09 -0400 Subject: [PATCH 186/824] #402 use abbreviated prompt if the default causes it to be huge --- mycli/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index ca3cffc6f..259bdf431 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -77,9 +77,9 @@ def emit(self, record): class MyCli(object): default_prompt = '\\t \\u@\\h:\\d> ' + max_len_prompt = 45 defaults_suffix = None - # In order of being loaded. Files lower in list override earlier ones. cnf_files = [ '/etc/my.cnf', '/etc/mysql/my.cnf', @@ -458,7 +458,10 @@ def run_cli(self): print('Thanks to the contributor -', thanks_picker([author_file, sponsor_file])) def prompt_tokens(cli): - return [(Token.Prompt, self.get_prompt(self.prompt_format))] + prompt = self.get_prompt(self.prompt_format) + if self.prompt_format == self.default_prompt and len(prompt) > self.max_len_prompt: + prompt = self.get_prompt('\\d> ') + return [(Token.Prompt, prompt)] def get_continuation_tokens(cli, width): continuation_prompt = self.get_prompt(self.prompt_continuation_format) From 5c7c544f7fc34cf13a4719c2798e8ce6ae61aac5 Mon Sep 17 00:00:00 2001 From: "John K. Sterling" Date: Sat, 15 Apr 2017 09:52:43 -0400 Subject: [PATCH 187/824] #402 keep comment in place --- mycli/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/main.py b/mycli/main.py index 259bdf431..0b209b731 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -80,6 +80,7 @@ class MyCli(object): max_len_prompt = 45 defaults_suffix = None + # In order of being loaded. Files lower in list override earlier ones. cnf_files = [ '/etc/my.cnf', '/etc/mysql/my.cnf', From a06675b6cc4801f9b979bbeec0b09722338cc67e Mon Sep 17 00:00:00 2001 From: "John K. Sterling" Date: Sat, 15 Apr 2017 09:59:01 -0400 Subject: [PATCH 188/824] #402 update change log --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index fcf9c198d..b393128cb 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,7 @@ Features: --------- * Add ability to specify alternative myclirc file. (Thanks: [Dick Marinus]). +* Add logic to shorten the default prompt if it becomes too long once generated. (Thanks: [John Sterling]) Bug Fixes: ---------- From 7de06479b7d63814bf10c72d205818b31467f06f Mon Sep 17 00:00:00 2001 From: "John K. Sterling" Date: Sat, 15 Apr 2017 09:59:56 -0400 Subject: [PATCH 189/824] #402 update change log --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index b393128cb..b9763e4bc 100644 --- a/changelog.md +++ b/changelog.md @@ -5,7 +5,7 @@ Features: --------- * Add ability to specify alternative myclirc file. (Thanks: [Dick Marinus]). -* Add logic to shorten the default prompt if it becomes too long once generated. (Thanks: [John Sterling]) +* Add logic to shorten the default prompt if it becomes too long once generated. (Thanks: [John Sterling]). Bug Fixes: ---------- From 9c70ce12098fc4ebf1567999cd29745cbd767d3a Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 17:10:07 +0200 Subject: [PATCH 190/824] don't import as --- mycli/output_formatter/output_formatter.py | 39 ++++++++++------------ 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index c87da5edd..ca8fb4eb4 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -6,15 +6,10 @@ from .expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) -from .delimited_output_adapter import (delimiter_adapter, - supported_formats as delimiter_formats, - delimiter_preprocessors) -from .tabulate_adapter import (tabulate_adapter, - supported_formats as tabulate_formats, - preprocessors as tabulate_preprocessors) -from .terminaltables_adapter import ( - terminaltables_adapter, preprocessors as terminaltables_preprocessors, - supported_formats as terminaltables_formats) + +from . import delimited_output_adapter +from . import tabulate_adapter +from . import terminaltables_adapter MISSING_VALUE = '' @@ -81,20 +76,20 @@ def format_output(self, data, headers, format_name=None, **kwargs): convert_to_string), {'missing_value': MISSING_VALUE}) -for delimiter_format in delimiter_formats: - OutputFormatter.register_new_formatter(delimiter_format, delimiter_adapter, - delimiter_preprocessors, - {'table_format': delimiter_format, - 'missing_value': MISSING_VALUE}) +for delimiter_format in delimited_output_adapter.supported_formats: + OutputFormatter.register_new_formatter( + delimiter_format, delimited_output_adapter.delimiter_adapter, + delimited_output_adapter.delimiter_preprocessors, + {'table_format': delimiter_format, 'missing_value': MISSING_VALUE}) -for tabulate_format in tabulate_formats: - OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, - tabulate_preprocessors, - {'table_format': tabulate_format, - 'missing_value': MISSING_VALUE}) +for tabulate_format in tabulate_adapter.supported_formats: + OutputFormatter.register_new_formatter( + tabulate_format, tabulate_adapter.tabulate_adapter, + tabulate_adapter.preprocessors, + {'table_format': tabulate_format, 'missing_value': MISSING_VALUE}) -for terminaltables_format in terminaltables_formats: +for terminaltables_format in terminaltables_adapter.supported_formats: OutputFormatter.register_new_formatter( - terminaltables_format, terminaltables_adapter, - terminaltables_preprocessors, + terminaltables_format, terminaltables_adapter.terminaltables_adapter, + terminaltables_adapter.preprocessors, {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) From a5f11e6e023c0bccdc2d93cf8e13bf8c7aced016 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 18:33:38 +0200 Subject: [PATCH 191/824] rename all adapter functions to adapter --- mycli/output_formatter/delimited_output_adapter.py | 4 ++-- mycli/output_formatter/output_formatter.py | 8 ++++---- mycli/output_formatter/tabulate_adapter.py | 2 +- mycli/output_formatter/terminaltables_adapter.py | 2 +- tests/test_output_formatter.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 8036a5ff7..a01a28433 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -8,10 +8,10 @@ from .preprocessors import override_missing_value, bytes_to_string supported_formats = ('csv', 'tsv') -delimiter_preprocessors = (override_missing_value, bytes_to_string) +preprocessors = (override_missing_value, bytes_to_string) -def delimiter_adapter(data, headers, table_format='csv', **_): +def adapter(data, headers, table_format='csv', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: if table_format == 'csv': diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index ca8fb4eb4..61e3c8d52 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -78,18 +78,18 @@ def format_output(self, data, headers, format_name=None, **kwargs): for delimiter_format in delimited_output_adapter.supported_formats: OutputFormatter.register_new_formatter( - delimiter_format, delimited_output_adapter.delimiter_adapter, - delimited_output_adapter.delimiter_preprocessors, + delimiter_format, delimited_output_adapter.adapter, + delimited_output_adapter.preprocessors, {'table_format': delimiter_format, 'missing_value': MISSING_VALUE}) for tabulate_format in tabulate_adapter.supported_formats: OutputFormatter.register_new_formatter( - tabulate_format, tabulate_adapter.tabulate_adapter, + tabulate_format, tabulate_adapter.adapter, tabulate_adapter.preprocessors, {'table_format': tabulate_format, 'missing_value': MISSING_VALUE}) for terminaltables_format in terminaltables_adapter.supported_formats: OutputFormatter.register_new_formatter( - terminaltables_format, terminaltables_adapter.terminaltables_adapter, + terminaltables_format, terminaltables_adapter.adapter, terminaltables_adapter.preprocessors, {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index ee63b5b1d..b89dcc0bd 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -12,7 +12,7 @@ preprocessors = (bytes_to_string, align_decimals) -def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): +def adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" kwargs = {'tablefmt': table_format, 'missingval': missing_value, 'disable_numparse': True} diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py index 0c702abe0..a8f50f985 100644 --- a/mycli/output_formatter/terminaltables_adapter.py +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -7,7 +7,7 @@ preprocessors = (bytes_to_string, override_missing_value, align_decimals) -def terminaltables_adapter(data, headers, table_format=None, **_): +def adapter(data, headers, table_format=None, **_): """Wrap terminaltables inside a standard function for OutputFormatter.""" table_format_handler = { diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index dcc26e62c..a0e0f198d 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -13,11 +13,11 @@ to_string) from mycli.output_formatter.output_formatter import OutputFormatter from mycli.output_formatter.delimited_output_adapter import ( - delimiter_adapter as csv_wrapper) + adapter as csv_wrapper) from mycli.output_formatter.tabulate_adapter import ( - tabulate_adapter as tabulate_wrapper) + adapter as tabulate_wrapper) from mycli.output_formatter.terminaltables_adapter import ( - terminaltables_adapter as terminal_tables_wrapper) + adapter as terminal_tables_wrapper) def test_to_string(): From 8fa96a2f6f49f190b58e1c0f68d5eba788cc14f7 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 21:21:53 +0200 Subject: [PATCH 192/824] don't import as (also for tests/test_output_formatter.py) --- tests/test_output_formatter.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index a0e0f198d..9844c1919 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -12,12 +12,9 @@ override_missing_value, to_string) from mycli.output_formatter.output_formatter import OutputFormatter -from mycli.output_formatter.delimited_output_adapter import ( - adapter as csv_wrapper) -from mycli.output_formatter.tabulate_adapter import ( - adapter as tabulate_wrapper) -from mycli.output_formatter.terminaltables_adapter import ( - adapter as terminal_tables_wrapper) +from mycli.output_formatter import delimited_output_adapter +from mycli.output_formatter import tabulate_adapter +from mycli.output_formatter import terminaltables_adapter def test_to_string(): @@ -98,7 +95,7 @@ def test_tabulate_wrapper(): """Test the *output_formatter.tabulate_wrapper()* function.""" data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = tabulate_wrapper(data, headers, table_format='psql') + output = tabulate_adapter.adapter(data, headers, table_format='psql') assert output == dedent('''\ +-----------+----------+ | letters | number | @@ -113,7 +110,7 @@ def test_csv_wrapper(): # Test comma-delimited output. data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = csv_wrapper(data, headers) + output = delimited_output_adapter.adapter(data, headers) assert output == dedent('''\ letters,number\r\n\ abc,1\r\n\ @@ -122,7 +119,8 @@ def test_csv_wrapper(): # Test tab-delimited output. data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = csv_wrapper(data, headers, table_format='tsv') + output = delimited_output_adapter.adapter( + data, headers, table_format='tsv') assert output == dedent('''\ letters\tnumber\r\n\ abc\t1\r\n\ @@ -133,7 +131,8 @@ def test_terminal_tables_wrapper(): """Test the *output_formatter.terminal_tables_wrapper()* function.""" data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = terminal_tables_wrapper(data, headers, table_format='ascii') + output = terminaltables_adapter.adapter( + data, headers, table_format='ascii') assert output == dedent('''\ +---------+--------+ | letters | number | From 6e8e4b85996bfcbd6fc71a461212e98aa5e20115 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:05:03 -0500 Subject: [PATCH 193/824] Switch from pycryptodome to cryptography. --- mycli/config.py | 50 ++++++++++++++++++++++++++++++------------------- setup.py | 2 +- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 7f5e0cb25..88bbc48ec 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -6,12 +6,15 @@ from os.path import exists import struct import sys + from configobj import ConfigObj, ConfigObjError +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend + try: basestring except NameError: basestring = str -from Crypto.Cipher import AES logger = logging.getLogger(__name__) @@ -144,7 +147,8 @@ def read_and_decrypt_mylogin_cnf(f): rkey = struct.pack('16B', *rkey) # Create a cipher object using the key. - aes_cipher = AES.new(rkey, AES.MODE_ECB) + aes_cipher = _get_aes_cipher(rkey) + decryptor = aes_cipher.decryptor() # Create a bytes buffer to hold the plaintext. plaintext = BytesIO() @@ -158,24 +162,9 @@ def read_and_decrypt_mylogin_cnf(f): # Read cipher_len bytes from the file and decrypt. cipher = f.read(cipher_len) - pplain = aes_cipher.decrypt(cipher) - - try: - # Determine pad length. - pad_len = ord(pplain[-1:]) - except TypeError: - # ord() was unable to get the value of the byte. - logger.warning('Unable to remove pad.') - continue - - if pad_len > len(pplain) or len(set(pplain[-pad_len:])) != 1: - # Pad length should be less than or equal to the length of the - # plaintext. The pad should have a single unqiue byte. - logger.warning('Invalid pad found in login path file.') + plain = _remove_pad(decryptor.update(cipher)) + if plain is False: continue - - # Get rid of pad. - plain = pplain[:-pad_len] plaintext.write(plain) if plaintext.tell() == 0: @@ -201,3 +190,26 @@ def str_to_bool(s): return False else: raise ValueError('not a recognized boolean value: %s'.format(s)) + +def _get_aes_cipher(key): + """Get the AES cipher object.""" + return Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) + +def _remove_pad(line): + """Remove the pad from the *line*.""" + pad_length = ord(line[-1:]) + try: + # Determine pad length. + pad_length = ord(line[-1:]) + except TypeError: + # ord() was unable to get the value of the byte. + logger.warning('Unable to remove pad.') + return False + + if pad_length > len(line) or len(set(line[-pad_length:])) != 1: + # Pad length should be less than or equal to the length of the + # plaintext. The pad should have a single unqiue byte. + logger.warning('Invalid pad found in login path file.') + return False + + return line[:-pad_length] diff --git a/setup.py b/setup.py index 82cdef6e1..fb05509ef 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ 'PyMySQL >= 0.6.7', 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', - 'pycryptodome >= 3', + 'cryptography >= 1.0.0', ] setup( From 2b049ae41f87ce94218d15f07c2bea97dd3a3335 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:06:20 -0500 Subject: [PATCH 194/824] Pep8radius fixes. --- mycli/config.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mycli/config.py b/mycli/config.py index 88bbc48ec..85c29cd7f 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) + def log(logger, level, message): """Logs message to stderr if logging isn't initialized.""" @@ -27,6 +28,7 @@ def log(logger, level, message): else: print(message, file=sys.stderr) + def read_config_file(f): """Read a config file.""" @@ -47,6 +49,7 @@ def read_config_file(f): return config + def read_config_files(files): """Read and merge a list of config files.""" @@ -60,6 +63,7 @@ def read_config_files(files): return config + def write_default_config(source, destination, overwrite=False): destination = os.path.expanduser(destination) if not overwrite and exists(destination): @@ -67,6 +71,7 @@ def write_default_config(source, destination, overwrite=False): shutil.copyfile(source, destination) + def get_mylogin_cnf_path(): """Return the path to the login path file or None if it doesn't exist.""" mylogin_cnf_path = os.getenv('MYSQL_TEST_LOGIN_FILE') @@ -83,6 +88,7 @@ def get_mylogin_cnf_path(): return mylogin_cnf_path return None + def open_mylogin_cnf(name): """Open a readable version of .mylogin.cnf. @@ -105,6 +111,7 @@ def open_mylogin_cnf(name): return TextIOWrapper(plaintext) + def read_and_decrypt_mylogin_cnf(f): """Read and decrypt the contents of .mylogin.cnf. @@ -174,6 +181,7 @@ def read_and_decrypt_mylogin_cnf(f): plaintext.seek(0) return plaintext + def str_to_bool(s): """Convert a string value to its corresponding boolean value.""" if isinstance(s, bool): @@ -191,10 +199,12 @@ def str_to_bool(s): else: raise ValueError('not a recognized boolean value: %s'.format(s)) + def _get_aes_cipher(key): """Get the AES cipher object.""" return Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) + def _remove_pad(line): """Remove the pad from the *line*.""" pad_length = ord(line[-1:]) From 7d91c9280fc33bb45d410131f8b55d8bb62b0c93 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:09:18 -0500 Subject: [PATCH 195/824] Add cryptography package change to the changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 1ae07cfd4..43c023f78 100644 --- a/changelog.md +++ b/changelog.md @@ -25,6 +25,7 @@ Internal Changes: * Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). * Run pep8 checks in travis (Thanks: [Irina Truong]). * Remove temporary hack for sqlparse (Thanks: [Dick Marinus]). +* Switch from pycryptodome to cryptography (Thanks: [Thomas Roten]). 1.9.0: ====== From 6d2d1364d0b4f89d4b8ffcb54878dca97ac7836a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:16:38 -0500 Subject: [PATCH 196/824] Simplify decryptor/cipher call. --- mycli/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 85c29cd7f..4f17d9f62 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -153,9 +153,8 @@ def read_and_decrypt_mylogin_cnf(f): return None rkey = struct.pack('16B', *rkey) - # Create a cipher object using the key. - aes_cipher = _get_aes_cipher(rkey) - decryptor = aes_cipher.decryptor() + # Create a decryptor object using the key. + decryptor = _get_decryptor(rkey) # Create a bytes buffer to hold the plaintext. plaintext = BytesIO() @@ -200,9 +199,10 @@ def str_to_bool(s): raise ValueError('not a recognized boolean value: %s'.format(s)) -def _get_aes_cipher(key): +def _get_decryptor(key): """Get the AES cipher object.""" - return Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) + c = Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) + return c.decryptor() def _remove_pad(line): From 6927717f9c10d87165a25c8c6dc71dcb7c50e609 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:17:30 -0500 Subject: [PATCH 197/824] Fix docstring. --- mycli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/config.py b/mycli/config.py index 4f17d9f62..daf5695b3 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -200,7 +200,7 @@ def str_to_bool(s): def _get_decryptor(key): - """Get the AES cipher object.""" + """Get the AES decryptor.""" c = Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) return c.decryptor() From 17fa97653f74bc6214ece3ba213d31b188c6c7d0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 21:52:19 -0500 Subject: [PATCH 198/824] Releasing version 1.10.0 --- changelog.md | 4 ++-- mycli/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 1ae07cfd4..4ba32f259 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,5 @@ -TBD -=== +1.10.0: +======= Features: --------- diff --git a/mycli/__init__.py b/mycli/__init__.py index e5102d301..52af183e5 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.9.0' +__version__ = '1.10.0' From 1296aff5d8d19e6ab1f29dde94a29e93e07cddec Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 18 Apr 2017 21:10:03 -0700 Subject: [PATCH 199/824] Update authors and email. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 82cdef6e1..d7e256980 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,8 @@ setup( name='mycli', - author='Amjith Ramanujam', - author_email='amjith[dot]r[at]gmail.com', + author='Mycli Core Team', + author_email='thomas@roten.us', version=version, url='http://mycli.net', packages=find_packages(), From 1087f20c9b5508685a2ee3388d338906e3b80919 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 19 Apr 2017 07:26:26 -0500 Subject: [PATCH 200/824] Use mailing list email address for package metadata. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d7e256980..c13d281be 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name='mycli', author='Mycli Core Team', - author_email='thomas@roten.us', + author_email='mycli-users@googlegroups.com', version=version, url='http://mycli.net', packages=find_packages(), From a1da54a24a105d37ccf6446f9e47f550f578f299 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 19 Apr 2017 07:30:07 -0500 Subject: [PATCH 201/824] Keep pep8radius happy :) --- setup.py | 64 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/setup.py b/setup.py index c13d281be..d1b2df27b 100644 --- a/setup.py +++ b/setup.py @@ -22,35 +22,35 @@ ] setup( - name='mycli', - author='Mycli Core Team', - author_email='mycli-users@googlegroups.com', - version=version, - url='http://mycli.net', - packages=find_packages(), - package_data={'mycli': ['myclirc', '../AUTHORS', '../SPONSORS']}, - description=description, - long_description=description, - install_requires=install_requirements, - entry_points=''' - [console_scripts] - mycli=mycli.main:cli - ''', - classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Operating System :: Unix', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: SQL', - 'Topic :: Database', - 'Topic :: Database :: Front-Ends', - 'Topic :: Software Development', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - ) + name='mycli', + author='Mycli Core Team', + author_email='mycli-users@googlegroups.com', + version=version, + url='http://mycli.net', + packages=find_packages(), + package_data={'mycli': ['myclirc', '../AUTHORS', '../SPONSORS']}, + description=description, + long_description=description, + install_requires=install_requirements, + entry_points=''' + [console_scripts] + mycli=mycli.main:cli + ''', + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: Unix', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: SQL', + 'Topic :: Database', + 'Topic :: Database :: Front-Ends', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], +) From 081f0d6ca719006bea79a6f5a62ea2d322fad300 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 19 Apr 2017 18:59:19 -0500 Subject: [PATCH 202/824] Switch author_email to dev mailing list. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d1b2df27b..fd777e89c 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name='mycli', author='Mycli Core Team', - author_email='mycli-users@googlegroups.com', + author_email='mycli-dev@googlegroups.com', version=version, url='http://mycli.net', packages=find_packages(), From 35abc84cbb0b59c8da64a468d8a5774da7689fed Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Fri, 21 Apr 2017 15:07:50 +0200 Subject: [PATCH 203/824] mv authors --- changelog.md | 7 +++++++ AUTHORS => mycli/AUTHORS | 1 + SPONSORS => mycli/SPONSORS | 0 mycli/main.py | 2 +- setup.py | 2 +- tests/test_main.py | 2 +- 6 files changed, 11 insertions(+), 3 deletions(-) rename AUTHORS => mycli/AUTHORS (97%) rename SPONSORS => mycli/SPONSORS (100%) diff --git a/changelog.md b/changelog.md index af5c81fa1..701d9dbce 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,10 @@ +TBD +=== + +Internal Changes: +----------------- +* Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). + 1.10.0: ======= diff --git a/AUTHORS b/mycli/AUTHORS similarity index 97% rename from AUTHORS rename to mycli/AUTHORS index 2eb4af10b..a7bb77650 100644 --- a/AUTHORS +++ b/mycli/AUTHORS @@ -49,6 +49,7 @@ Contributors: * cxbig * chainkite * Michał Górny + * Terje Røsten Creator: -------- diff --git a/SPONSORS b/mycli/SPONSORS similarity index 100% rename from SPONSORS rename to mycli/SPONSORS diff --git a/mycli/main.py b/mycli/main.py index ff3e8d9cb..b24adeadf 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -439,7 +439,7 @@ def run_cli(self): if self.smart_completion: self.refresh_completions() - project_root = os.path.dirname(PACKAGE_ROOT) + project_root = os.path.join(os.path.dirname(PACKAGE_ROOT), 'mycli') author_file = os.path.join(project_root, 'AUTHORS') sponsor_file = os.path.join(project_root, 'SPONSORS') diff --git a/setup.py b/setup.py index 33ab724f4..6e982d9db 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ version=version, url='http://mycli.net', packages=find_packages(), - package_data={'mycli': ['myclirc', '../AUTHORS', '../SPONSORS']}, + package_data={'mycli': ['myclirc', 'AUTHORS', 'SPONSORS']}, description=description, long_description=description, install_requires=install_requirements, diff --git a/tests/test_main.py b/tests/test_main.py index 1271f18da..56c0cb267 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -164,7 +164,7 @@ def test_confirm_destructive_query_notty(executor): assert confirm_destructive_query(sql) is None def test_thanks_picker_utf8(): - project_root = os.path.dirname(PACKAGE_ROOT) + project_root = os.path.join(os.path.dirname(PACKAGE_ROOT), 'mycli') author_file = os.path.join(project_root, 'AUTHORS') sponsor_file = os.path.join(project_root, 'SPONSORS') From 4f77ec7a548ffd787b63c9916c6d955905e20a07 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 22 Apr 2017 20:18:21 +0200 Subject: [PATCH 204/824] rename tests/ to test/ rename tests/ to test so it will be included in sdist see: Anything that looks like a test script: test/test*.py (currently, the Distutils don't do anything with test scripts except include them in source distributions, but in the future there will be a standard for testing Python module distributions) https://docs.python.org/3/distutils/sourcedist.html --- .gitignore | 2 +- .travis.yml | 2 +- {tests => test}/conftest.py | 0 {tests => test}/features/basic_commands.feature | 0 {tests => test}/features/crud_database.feature | 0 {tests => test}/features/crud_table.feature | 0 {tests => test}/features/db_utils.py | 0 {tests => test}/features/environment.py | 0 {tests => test}/features/fixture_data/help.txt | 0 .../features/fixture_data/help_commands.txt | 0 {tests => test}/features/fixture_utils.py | 0 {tests => test}/features/iocommands.feature | 0 {tests => test}/features/named_queries.feature | 0 {tests => test}/features/specials.feature | 0 {tests => test}/features/steps/basic_commands.py | 0 {tests => test}/features/steps/crud_database.py | 0 {tests => test}/features/steps/crud_table.py | 0 {tests => test}/features/steps/iocommands.py | 0 {tests => test}/features/steps/named_queries.py | 0 {tests => test}/features/steps/specials.py | 0 {tests => test}/features/steps/wrappers.py | 0 {tests => test}/mylogin.cnf | Bin {tests => test}/test.txt | 0 {tests => test}/test_completion_engine.py | 0 {tests => test}/test_completion_refresher.py | 0 {tests => test}/test_config.py | 0 {tests => test}/test_dbspecial.py | 0 {tests => test}/test_expanded.py | 0 {tests => test}/test_main.py | 0 {tests => test}/test_naive_completion.py | 0 {tests => test}/test_output_formatter.py | 0 {tests => test}/test_parseutils.py | 0 {tests => test}/test_plan.wiki | 0 .../test_smart_completion_public_schema_only.py | 0 {tests => test}/test_special_iocommands.py | 0 {tests => test}/test_sqlexecute.py | 8 ++++---- {tests => test}/test_tabulate.py | 0 {tests => test}/utils.py | 0 38 files changed, 6 insertions(+), 6 deletions(-) rename {tests => test}/conftest.py (100%) rename {tests => test}/features/basic_commands.feature (100%) rename {tests => test}/features/crud_database.feature (100%) rename {tests => test}/features/crud_table.feature (100%) rename {tests => test}/features/db_utils.py (100%) rename {tests => test}/features/environment.py (100%) rename {tests => test}/features/fixture_data/help.txt (100%) rename {tests => test}/features/fixture_data/help_commands.txt (100%) rename {tests => test}/features/fixture_utils.py (100%) rename {tests => test}/features/iocommands.feature (100%) rename {tests => test}/features/named_queries.feature (100%) rename {tests => test}/features/specials.feature (100%) rename {tests => test}/features/steps/basic_commands.py (100%) rename {tests => test}/features/steps/crud_database.py (100%) rename {tests => test}/features/steps/crud_table.py (100%) rename {tests => test}/features/steps/iocommands.py (100%) rename {tests => test}/features/steps/named_queries.py (100%) rename {tests => test}/features/steps/specials.py (100%) rename {tests => test}/features/steps/wrappers.py (100%) rename {tests => test}/mylogin.cnf (100%) rename {tests => test}/test.txt (100%) rename {tests => test}/test_completion_engine.py (100%) rename {tests => test}/test_completion_refresher.py (100%) rename {tests => test}/test_config.py (100%) rename {tests => test}/test_dbspecial.py (100%) rename {tests => test}/test_expanded.py (100%) rename {tests => test}/test_main.py (100%) rename {tests => test}/test_naive_completion.py (100%) rename {tests => test}/test_output_formatter.py (100%) rename {tests => test}/test_parseutils.py (100%) rename {tests => test}/test_plan.wiki (100%) rename {tests => test}/test_smart_completion_public_schema_only.py (100%) rename {tests => test}/test_special_iocommands.py (100%) rename {tests => test}/test_sqlexecute.py (97%) rename {tests => test}/test_tabulate.py (100%) rename {tests => test}/utils.py (100%) diff --git a/.gitignore b/.gitignore index 59fa76be2..e907154fc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ /dist /mycli.egg-info /src -/tests/behave.ini +/test/behave.ini .vagrant *.pyc diff --git a/.travis.yml b/.travis.yml index f78d94b00..021be1c17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: script: - coverage run --source mycli -m py.test - - cd tests + - cd test - behave - cd .. # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. diff --git a/tests/conftest.py b/test/conftest.py similarity index 100% rename from tests/conftest.py rename to test/conftest.py diff --git a/tests/features/basic_commands.feature b/test/features/basic_commands.feature similarity index 100% rename from tests/features/basic_commands.feature rename to test/features/basic_commands.feature diff --git a/tests/features/crud_database.feature b/test/features/crud_database.feature similarity index 100% rename from tests/features/crud_database.feature rename to test/features/crud_database.feature diff --git a/tests/features/crud_table.feature b/test/features/crud_table.feature similarity index 100% rename from tests/features/crud_table.feature rename to test/features/crud_table.feature diff --git a/tests/features/db_utils.py b/test/features/db_utils.py similarity index 100% rename from tests/features/db_utils.py rename to test/features/db_utils.py diff --git a/tests/features/environment.py b/test/features/environment.py similarity index 100% rename from tests/features/environment.py rename to test/features/environment.py diff --git a/tests/features/fixture_data/help.txt b/test/features/fixture_data/help.txt similarity index 100% rename from tests/features/fixture_data/help.txt rename to test/features/fixture_data/help.txt diff --git a/tests/features/fixture_data/help_commands.txt b/test/features/fixture_data/help_commands.txt similarity index 100% rename from tests/features/fixture_data/help_commands.txt rename to test/features/fixture_data/help_commands.txt diff --git a/tests/features/fixture_utils.py b/test/features/fixture_utils.py similarity index 100% rename from tests/features/fixture_utils.py rename to test/features/fixture_utils.py diff --git a/tests/features/iocommands.feature b/test/features/iocommands.feature similarity index 100% rename from tests/features/iocommands.feature rename to test/features/iocommands.feature diff --git a/tests/features/named_queries.feature b/test/features/named_queries.feature similarity index 100% rename from tests/features/named_queries.feature rename to test/features/named_queries.feature diff --git a/tests/features/specials.feature b/test/features/specials.feature similarity index 100% rename from tests/features/specials.feature rename to test/features/specials.feature diff --git a/tests/features/steps/basic_commands.py b/test/features/steps/basic_commands.py similarity index 100% rename from tests/features/steps/basic_commands.py rename to test/features/steps/basic_commands.py diff --git a/tests/features/steps/crud_database.py b/test/features/steps/crud_database.py similarity index 100% rename from tests/features/steps/crud_database.py rename to test/features/steps/crud_database.py diff --git a/tests/features/steps/crud_table.py b/test/features/steps/crud_table.py similarity index 100% rename from tests/features/steps/crud_table.py rename to test/features/steps/crud_table.py diff --git a/tests/features/steps/iocommands.py b/test/features/steps/iocommands.py similarity index 100% rename from tests/features/steps/iocommands.py rename to test/features/steps/iocommands.py diff --git a/tests/features/steps/named_queries.py b/test/features/steps/named_queries.py similarity index 100% rename from tests/features/steps/named_queries.py rename to test/features/steps/named_queries.py diff --git a/tests/features/steps/specials.py b/test/features/steps/specials.py similarity index 100% rename from tests/features/steps/specials.py rename to test/features/steps/specials.py diff --git a/tests/features/steps/wrappers.py b/test/features/steps/wrappers.py similarity index 100% rename from tests/features/steps/wrappers.py rename to test/features/steps/wrappers.py diff --git a/tests/mylogin.cnf b/test/mylogin.cnf similarity index 100% rename from tests/mylogin.cnf rename to test/mylogin.cnf diff --git a/tests/test.txt b/test/test.txt similarity index 100% rename from tests/test.txt rename to test/test.txt diff --git a/tests/test_completion_engine.py b/test/test_completion_engine.py similarity index 100% rename from tests/test_completion_engine.py rename to test/test_completion_engine.py diff --git a/tests/test_completion_refresher.py b/test/test_completion_refresher.py similarity index 100% rename from tests/test_completion_refresher.py rename to test/test_completion_refresher.py diff --git a/tests/test_config.py b/test/test_config.py similarity index 100% rename from tests/test_config.py rename to test/test_config.py diff --git a/tests/test_dbspecial.py b/test/test_dbspecial.py similarity index 100% rename from tests/test_dbspecial.py rename to test/test_dbspecial.py diff --git a/tests/test_expanded.py b/test/test_expanded.py similarity index 100% rename from tests/test_expanded.py rename to test/test_expanded.py diff --git a/tests/test_main.py b/test/test_main.py similarity index 100% rename from tests/test_main.py rename to test/test_main.py diff --git a/tests/test_naive_completion.py b/test/test_naive_completion.py similarity index 100% rename from tests/test_naive_completion.py rename to test/test_naive_completion.py diff --git a/tests/test_output_formatter.py b/test/test_output_formatter.py similarity index 100% rename from tests/test_output_formatter.py rename to test/test_output_formatter.py diff --git a/tests/test_parseutils.py b/test/test_parseutils.py similarity index 100% rename from tests/test_parseutils.py rename to test/test_parseutils.py diff --git a/tests/test_plan.wiki b/test/test_plan.wiki similarity index 100% rename from tests/test_plan.wiki rename to test/test_plan.wiki diff --git a/tests/test_smart_completion_public_schema_only.py b/test/test_smart_completion_public_schema_only.py similarity index 100% rename from tests/test_smart_completion_public_schema_only.py rename to test/test_smart_completion_public_schema_only.py diff --git a/tests/test_special_iocommands.py b/test/test_special_iocommands.py similarity index 100% rename from tests/test_special_iocommands.py rename to test/test_special_iocommands.py diff --git a/tests/test_sqlexecute.py b/test/test_sqlexecute.py similarity index 97% rename from tests/test_sqlexecute.py rename to test/test_sqlexecute.py index a9b5fbec2..157b62341 100644 --- a/tests/test_sqlexecute.py +++ b/test/test_sqlexecute.py @@ -229,7 +229,7 @@ def test_system_command_not_found(executor): @dbtest def test_system_command_output(executor): - test_file_path = os.path.join(os.path.abspath('.'), 'tests/test.txt') + test_file_path = os.path.join(os.path.abspath('.'), 'test', 'test.txt') results = run(executor, 'system cat {0}'.format(test_file_path)) assert len(results) == 1 expected_line = u'mycli rocks!\n' @@ -237,9 +237,9 @@ def test_system_command_output(executor): @dbtest def test_cd_command_current_dir(executor): - tests_path = os.path.join(os.path.abspath('.'), 'tests') - results = run(executor, 'system cd {0}'.format(tests_path)) - assert os.getcwd() == tests_path + test_path = os.path.join(os.path.abspath('.'), 'test') + results = run(executor, 'system cd {0}'.format(test_path)) + assert os.getcwd() == test_path @dbtest def test_unicode_support(executor): diff --git a/tests/test_tabulate.py b/test/test_tabulate.py similarity index 100% rename from tests/test_tabulate.py rename to test/test_tabulate.py diff --git a/tests/utils.py b/test/utils.py similarity index 100% rename from tests/utils.py rename to test/utils.py From 9e20cc6711c89696843363e8c090abf2be235425 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 22 Apr 2017 12:48:37 +0200 Subject: [PATCH 205/824] add more files to MANIFEST.in --- MANIFEST.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 1d3bbc866..a50b73686 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ -include LICENSE.txt *.md +include LICENSE.txt *.md *.rst TODO requirements-dev.txt screenshots/* +include conftest.py .coveragerc pytest.ini test tox.ini From 26782c77520d9d0dc08a4820abf5450b90ca0846 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 22 Apr 2017 13:08:25 +0200 Subject: [PATCH 206/824] add to changelog.md --- changelog.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/changelog.md b/changelog.md index af5c81fa1..f00fc4359 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,12 @@ +TBD +=== + +Internal Changes: +----------------- + +* Rename tests/ to test/ . (Thanks: [Dick Marinus]). + + 1.10.0: ======= From a9af5d41a4b0e079776a5baf7c214196dbfc57a3 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 22 Apr 2017 20:08:31 +0200 Subject: [PATCH 207/824] fix pep8 code style --- test/conftest.py | 3 +- test/features/db_utils.py | 27 +-- test/features/environment.py | 17 +- test/features/fixture_utils.py | 9 +- test/features/steps/basic_commands.py | 33 ++-- test/features/steps/crud_database.py | 60 +++---- test/features/steps/crud_table.py | 63 +++---- test/features/steps/iocommands.py | 7 +- test/features/steps/named_queries.py | 33 ++-- test/features/steps/specials.py | 20 +-- test/features/steps/wrappers.py | 3 +- test/test_completion_engine.py | 170 +++++++++++------- test/test_completion_refresher.py | 15 +- test/test_config.py | 2 +- test/test_dbspecial.py | 3 +- test/test_main.py | 9 + test/test_naive_completion.py | 6 + test/test_parseutils.py | 21 ++- ...est_smart_completion_public_schema_only.py | 79 +++++--- test/test_special_iocommands.py | 15 +- test/test_sqlexecute.py | 30 +++- test/utils.py | 13 +- 22 files changed, 369 insertions(+), 269 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index d24d26bc4..d7d133400 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,5 +1,6 @@ import pytest -from utils import (HOST, USER, PASSWORD, PORT, CHARSET, create_db, db_connection) +from utils import (HOST, USER, PASSWORD, PORT, + CHARSET, create_db, db_connection) import mycli.sqlexecute diff --git a/test/features/db_utils.py b/test/features/db_utils.py index f604c608c..ef0b42ffc 100644 --- a/test/features/db_utils.py +++ b/test/features/db_utils.py @@ -4,15 +4,17 @@ import pymysql + def create_db(hostname='localhost', username=None, password=None, dbname=None): - """ - Create test database. + """Create test database. + :param hostname: string :param username: string :param password: string :param dbname: string :return: + """ cn = pymysql.connect( host=hostname, @@ -23,8 +25,8 @@ def create_db(hostname='localhost', username=None, password=None, ) with cn.cursor() as cr: - cr.execute('drop database if exists '+dbname) - cr.execute('create database '+dbname) + cr.execute('drop database if exists ' + dbname) + cr.execute('create database ' + dbname) cn.close() @@ -33,13 +35,14 @@ def create_db(hostname='localhost', username=None, password=None, def create_cn(hostname, password, username, dbname): - """ - Open connection to database. + """Open connection to database. + :param hostname: :param password: :param username: :param dbname: string :return: psycopg2.connection + """ cn = pymysql.connect( host=hostname, @@ -55,12 +58,13 @@ def create_cn(hostname, password, username, dbname): def drop_db(hostname='localhost', username=None, password=None, dbname=None): - """ - Drop database. + """Drop database. + :param hostname: string :param username: string :param password: string :param dbname: string + """ cn = pymysql.connect( host=hostname, @@ -72,15 +76,16 @@ def drop_db(hostname='localhost', username=None, password=None, ) with cn.cursor() as cr: - cr.execute('drop database if exists '+dbname) + cr.execute('drop database if exists ' + dbname) close_cn(cn) def close_cn(cn=None): - """ - Close connection. + """Close connection. + :param connection: pymysql.connection + """ if cn: cn.close() diff --git a/test/features/environment.py b/test/features/environment.py index 3f55757b8..e79e5740c 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -9,9 +9,7 @@ def before_all(context): - """ - Set env parameters. - """ + """Set env parameters.""" os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" os.environ['PAGER'] = 'cat' @@ -21,7 +19,8 @@ def before_all(context): context.exit_sent = False vi = '_'.join([str(x) for x in sys.version_info[:3]]) - db_name = context.config.userdata.get('my_test_db', None) or "mycli_behave_tests" + db_name = context.config.userdata.get( + 'my_test_db', None) or "mycli_behave_tests" db_name_full = '{0}_{1}'.format(db_name, vi) # Store get params from config/environment variables @@ -40,7 +39,7 @@ def before_all(context): ), 'cli_command': context.config.userdata.get( 'my_cli_command', None) or - sys.executable+' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"', + sys.executable + ' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"', 'dbname': db_name, 'dbname_tmp': db_name_full + '_tmp', 'vi': vi, @@ -54,9 +53,7 @@ def before_all(context): def after_all(context): - """ - Unset env parameters. - """ + """Unset env parameters.""" dbutils.close_cn(context.cn) dbutils.drop_db(context.conf['host'], context.conf['user'], context.conf['pass'], context.conf['dbname']) @@ -70,9 +67,7 @@ def after_all(context): def after_scenario(context, _): - """ - Cleans up after each test complete. - """ + """Cleans up after each test complete.""" if hasattr(context, 'cli') and not context.exit_sent: # Terminate nicely. diff --git a/test/features/fixture_utils.py b/test/features/fixture_utils.py index f3b490c40..a171e34ce 100644 --- a/test/features/fixture_utils.py +++ b/test/features/fixture_utils.py @@ -6,10 +6,11 @@ def read_fixture_lines(filename): - """ - Read lines of text from file. + """Read lines of text from file. + :param filename: string name :return: list of strings + """ lines = [] for line in io.open(filename, 'r', encoding='utf8'): @@ -18,9 +19,7 @@ def read_fixture_lines(filename): def read_fixture_files(): - """ - Read all files inside fixture_data directory. - """ + """Read all files inside fixture_data directory.""" fixture_dict = {} current_dir = os.path.dirname(__file__) diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index 7aa476640..109472b8a 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -14,9 +15,7 @@ @when('we run dbcli') def step_run_cli(context): - """ - Run the process using pexpect. - """ + """Run the process using pexpect.""" run_args = [] if context.conf.get('host', None): run_args.extend(('-h', context.conf['host'])) @@ -26,7 +25,8 @@ def step_run_cli(context): run_args.extend(('-p', context.conf['pass'])) if context.conf.get('dbname', None): run_args.extend(('-D', context.conf['dbname'])) - cli_cmd = context.conf.get('cli_command', None) or sys.executable+' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' + cli_cmd = context.conf.get('cli_command', None) or sys.executable + \ + ' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' cmd_parts = [cli_cmd] + run_args cmd = ' '.join(cmd_parts) @@ -36,27 +36,26 @@ def step_run_cli(context): @when('we wait for prompt') def step_wait_prompt(context): - """ - Make sure prompt is displayed. - """ + """Make sure prompt is displayed.""" user = context.conf['user'] host = context.conf['host'] dbname = context.conf['dbname'] - wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format(user, host, dbname), timeout=5) + wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( + user, host, dbname), timeout=5) @when('we send "ctrl + d"') def step_ctrl_d(context): - """ - Send Ctrl + D to hopefully exit. - """ + """Send Ctrl + D to hopefully exit.""" context.cli.sendcontrol('d') context.exit_sent = True @when('we send "\?" command') def step_send_help(context): - """ - Send \? to see help. + """Send \? + + to see help. + """ context.cli.sendline('\\?') diff --git a/test/features/steps/crud_database.py b/test/features/steps/crud_database.py index 3eab34d9d..d7b8eeb28 100644 --- a/test/features/steps/crud_database.py +++ b/test/features/steps/crud_database.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -14,9 +15,7 @@ @when('we create database') def step_db_create(context): - """ - Send create database. - """ + """Send create database.""" context.cli.sendline('create database {0};'.format( context.conf['dbname_tmp'])) @@ -27,78 +26,67 @@ def step_db_create(context): @when('we drop database') def step_db_drop(context): - """ - Send drop database. - """ + """Send drop database.""" context.cli.sendline('drop database {0};'.format( context.conf['dbname_tmp'])) - wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + wrappers.expect_exact( + context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) context.cli.sendline('y') + @when('we connect to test database') def step_db_connect_test(context): - """ - Send connect to database. - """ + """Send connect to database.""" db_name = context.conf['dbname'] context.cli.sendline('use {0}'.format(db_name)) @when('we connect to dbserver') def step_db_connect_dbserver(context): - """ - Send connect to database. - """ + """Send connect to database.""" context.cli.sendline('use mysql') @then('dbcli exits') def step_wait_exit(context): - """ - Make sure the cli exits. - """ + """Make sure the cli exits.""" wrappers.expect_exact(context, pexpect.EOF, timeout=5) @then('we see dbcli prompt') def step_see_prompt(context): - """ - Wait to see the prompt. - """ + """Wait to see the prompt.""" user = context.conf['user'] host = context.conf['host'] dbname = context.conf['dbname'] - wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format(user, host, dbname), timeout=5) + wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( + user, host, dbname), timeout=5) @then('we see help output') def step_see_help(context): for expected_line in context.fixture_data['help_commands.txt']: - wrappers.expect_exact(context, expected_line+'\r\n', timeout=1) + wrappers.expect_exact(context, expected_line + '\r\n', timeout=1) @then('we see database created') def step_see_db_created(context): - """ - Wait to see create database output. - """ + """Wait to see create database output.""" wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see database dropped') def step_see_db_dropped(context): - """ - Wait to see drop database output. - """ + """Wait to see drop database output.""" wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see database connected') def step_see_db_connected(context): - """ - Wait to see drop database output. - """ - wrappers.expect_exact(context, 'You are now connected to database "', timeout=2) + """Wait to see drop database output.""" + wrappers.expect_exact( + context, 'You are now connected to database "', timeout=2) wrappers.expect_exact(context, '"', timeout=2) - wrappers.expect_exact(context, ' as user "{0}"\r\n'.format(context.conf['user']), timeout=2) + wrappers.expect_exact(context, ' as user "{0}"\r\n'.format( + context.conf['user']), timeout=2) diff --git a/test/features/steps/crud_table.py b/test/features/steps/crud_table.py index b73ff0d6f..34301c890 100644 --- a/test/features/steps/crud_table.py +++ b/test/features/steps/crud_table.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -12,100 +13,78 @@ @when('we create table') def step_create_table(context): - """ - Send create table. - """ + """Send create table.""" context.cli.sendline('create table a(x text);') @when('we insert into table') def step_insert_into_table(context): - """ - Send insert into table. - """ + """Send insert into table.""" context.cli.sendline('''insert into a(x) values('xxx');''') @when('we update table') def step_update_table(context): - """ - Send insert into table. - """ + """Send insert into table.""" context.cli.sendline('''update a set x = 'yyy' where x = 'xxx';''') @when('we select from table') def step_select_from_table(context): - """ - Send select from table. - """ + """Send select from table.""" context.cli.sendline('select * from a;') @when('we delete from table') def step_delete_from_table(context): - """ - Send deete from table. - """ + """Send deete from table.""" context.cli.sendline('''delete from a where x = 'yyy';''') - wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + wrappers.expect_exact( + context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) context.cli.sendline('y') @when('we drop table') def step_drop_table(context): - """ - Send drop table. - """ + """Send drop table.""" context.cli.sendline('drop table a;') - wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + wrappers.expect_exact( + context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) context.cli.sendline('y') @then('we see table created') def step_see_table_created(context): - """ - Wait to see create table output. - """ + """Wait to see create table output.""" wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see record inserted') def step_see_record_inserted(context): - """ - Wait to see insert output. - """ + """Wait to see insert output.""" wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see record updated') def step_see_record_updated(context): - """ - Wait to see update output. - """ + """Wait to see update output.""" wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see data selected') def step_see_data_selected(context): - """ - Wait to see select output. - """ + """Wait to see select output.""" wrappers.expect_exact( context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') def step_see_data_deleted(context): - """ - Wait to see delete output. - """ + """Wait to see delete output.""" wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see table dropped') def step_see_table_dropped(context): - """ - Wait to see drop output. - """ + """Wait to see drop output.""" wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 885200469..73068fac2 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -8,14 +8,13 @@ @when('we start external editor providing a file name') def step_edit_file(context): - """ - Edit file with external editor. - """ + """Edit file with external editor.""" context.editor_file_name = 'test_file_{0}.sql'.format(context.conf['vi']) if os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) context.cli.sendline('\e {0}'.format(context.editor_file_name)) - wrappers.expect_exact(context, 'Entering Ex mode. Type "visual" to go to Normal mode.', timeout=2) + wrappers.expect_exact( + context, 'Entering Ex mode. Type "visual" to go to Normal mode.', timeout=2) wrappers.expect_exact(context, '\r\n:', timeout=2) diff --git a/test/features/steps/named_queries.py b/test/features/steps/named_queries.py index b53ad47db..60115c5cd 100644 --- a/test/features/steps/named_queries.py +++ b/test/features/steps/named_queries.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -12,48 +13,36 @@ @when('we save a named query') def step_save_named_query(context): - """ - Send \ns command - """ + """Send \ns command.""" context.cli.sendline('\\fs foo SELECT 12345') @when('we use a named query') def step_use_named_query(context): - """ - Send \n command - """ + """Send \n command.""" context.cli.sendline('\\f foo') @when('we delete a named query') def step_delete_named_query(context): - """ - Send \nd command - """ + """Send \nd command.""" context.cli.sendline('\\fd foo') @then('we see the named query saved') def step_see_named_query_saved(context): - """ - Wait to see query saved. - """ + """Wait to see query saved.""" wrappers.expect_exact(context, 'Saved.', timeout=1) @then('we see the named query executed') def step_see_named_query_executed(context): - """ - Wait to see select output. - """ + """Wait to see select output.""" wrappers.expect_exact(context, '12345', timeout=1) wrappers.expect_exact(context, 'SELECT 1', timeout=1) @then('we see the named query deleted') def step_see_named_query_deleted(context): - """ - Wait to see query deleted. - """ + """Wait to see query deleted.""" wrappers.expect_exact(context, 'foo: Deleted', timeout=1) diff --git a/test/features/steps/specials.py b/test/features/steps/specials.py index 790b2476f..c0a3c0feb 100644 --- a/test/features/steps/specials.py +++ b/test/features/steps/specials.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -12,15 +13,12 @@ @when('we refresh completions') def step_refresh_completions(context): - """ - Send refresh command. - """ + """Send refresh command.""" context.cli.sendline('rehash') @then('we see completions refresh started') def step_see_refresh_started(context): - """ - Wait to see refresh output. - """ - wrappers.expect_exact(context, 'Auto-completion refresh started in the background', timeout=2) + """Wait to see refresh output.""" + wrappers.expect_exact( + context, 'Auto-completion refresh started in the background', timeout=2) diff --git a/test/features/steps/wrappers.py b/test/features/steps/wrappers.py index eac7c8304..aea742033 100644 --- a/test/features/steps/wrappers.py +++ b/test/features/steps/wrappers.py @@ -9,7 +9,8 @@ def expect_exact(context, expected, timeout): context.cli.expect_exact(expected, timeout=timeout) except: # Strip color codes out of the output. - actual = re.sub(r'\x1b\[([0-9A-Za-z;?])+[m|K]?', '', context.cli.before) + actual = re.sub(r'\x1b\[([0-9A-Za-z;?])+[m|K]?', + '', context.cli.before) raise Exception('Expected:\n---\n{0!r}\n---\n\nActual:\n---\n{1!r}\n---'.format( expected, actual)) diff --git a/test/test_completion_engine.py b/test/test_completion_engine.py index 9b8771083..4f0406b08 100644 --- a/test/test_completion_engine.py +++ b/test/test_completion_engine.py @@ -1,25 +1,28 @@ from mycli.packages.completion_engine import suggest_type import pytest + def sorted_dicts(dicts): - """input is a list of dicts""" + """input is a list of dicts.""" return sorted(tuple(x.items()) for x in dicts) + def test_select_suggests_cols_with_visible_table_scope(): suggestions = suggest_type('SELECT FROM tabl', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + def test_select_suggests_cols_with_qualified_table_scope(): suggestions = suggest_type('SELECT FROM sch.tabl', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [('sch', 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [('sch', 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) @pytest.mark.parametrize('expression', [ @@ -37,10 +40,11 @@ def test_select_suggests_cols_with_qualified_table_scope(): def test_where_suggests_columns_functions(expression): suggestions = suggest_type(expression, expression) assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + @pytest.mark.parametrize('expression', [ 'SELECT * FROM tabl WHERE foo IN (', @@ -49,41 +53,49 @@ def test_where_suggests_columns_functions(expression): def test_where_in_suggests_columns(expression): suggestions = suggest_type(expression, expression) assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + def test_where_equals_any_suggests_columns_or_keywords(): text = 'SELECT * FROM tabl WHERE foo = ANY(' suggestions = suggest_type(text, text) assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}]) + {'type': 'column', 'tables': [(None, 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}]) + def test_lparen_suggests_cols(): suggestion = suggest_type('SELECT MAX( FROM tbl', 'SELECT MAX(') assert suggestion == [ {'type': 'column', 'tables': [(None, 'tbl', None)]}] + def test_operand_inside_function_suggests_cols1(): - suggestion = suggest_type('SELECT MAX(col1 + FROM tbl', 'SELECT MAX(col1 + ') + suggestion = suggest_type( + 'SELECT MAX(col1 + FROM tbl', 'SELECT MAX(col1 + ') assert suggestion == [ {'type': 'column', 'tables': [(None, 'tbl', None)]}] + def test_operand_inside_function_suggests_cols2(): - suggestion = suggest_type('SELECT MAX(col1 + col2 + FROM tbl', 'SELECT MAX(col1 + col2 + ') + suggestion = suggest_type( + 'SELECT MAX(col1 + col2 + FROM tbl', 'SELECT MAX(col1 + col2 + ') assert suggestion == [ {'type': 'column', 'tables': [(None, 'tbl', None)]}] + def test_select_suggests_cols_and_funcs(): suggestions = suggest_type('SELECT ', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': []}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': []}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + @pytest.mark.parametrize('expression', [ 'SELECT * FROM ', @@ -102,6 +114,7 @@ def test_expression_suggests_tables_views_and_schemas(expression): {'type': 'view', 'schema': []}, {'type': 'schema'}]) + @pytest.mark.parametrize('expression', [ 'SELECT * FROM sch.', 'INSERT INTO sch.', @@ -118,37 +131,43 @@ def test_expression_suggests_qualified_tables_views_and_schemas(expression): {'type': 'table', 'schema': 'sch'}, {'type': 'view', 'schema': 'sch'}]) + def test_truncate_suggests_tables_and_schemas(): suggestions = suggest_type('TRUNCATE ', 'TRUNCATE ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'table', 'schema': []}, {'type': 'schema'}]) + def test_truncate_suggests_qualified_tables(): suggestions = suggest_type('TRUNCATE sch.', 'TRUNCATE sch.') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'table', 'schema': 'sch'}]) + def test_distinct_suggests_cols(): suggestions = suggest_type('SELECT DISTINCT ', 'SELECT DISTINCT ') assert suggestions == [{'type': 'column', 'tables': []}] + def test_col_comma_suggests_cols(): suggestions = suggest_type('SELECT a, b, FROM tbl', 'SELECT a, b,') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tbl', None)]}, {'type': 'function', 'schema': []}, {'type': 'keyword'}, - ]) + ]) + def test_table_comma_suggests_tables_and_schemas(): suggestions = suggest_type('SELECT a, b FROM tbl1, ', - 'SELECT a, b FROM tbl1, ') + 'SELECT a, b FROM tbl1, ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'table', 'schema': []}, {'type': 'view', 'schema': []}, {'type': 'schema'}]) + def test_into_suggests_tables_and_schemas(): suggestion = suggest_type('INSERT INTO ', 'INSERT INTO ') assert sorted_dicts(suggestion) == sorted_dicts([ @@ -156,26 +175,31 @@ def test_into_suggests_tables_and_schemas(): {'type': 'view', 'schema': []}, {'type': 'schema'}]) + def test_insert_into_lparen_suggests_cols(): suggestions = suggest_type('INSERT INTO abc (', 'INSERT INTO abc (') assert suggestions == [{'type': 'column', 'tables': [(None, 'abc', None)]}] + def test_insert_into_lparen_partial_text_suggests_cols(): suggestions = suggest_type('INSERT INTO abc (i', 'INSERT INTO abc (i') assert suggestions == [{'type': 'column', 'tables': [(None, 'abc', None)]}] + def test_insert_into_lparen_comma_suggests_cols(): suggestions = suggest_type('INSERT INTO abc (id,', 'INSERT INTO abc (id,') assert suggestions == [{'type': 'column', 'tables': [(None, 'abc', None)]}] + def test_partially_typed_col_name_suggests_col_names(): suggestions = suggest_type('SELECT * FROM tabl WHERE col_n', - 'SELECT * FROM tabl WHERE col_n') + 'SELECT * FROM tabl WHERE col_n') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl', None)]}, {'type': 'function', 'schema': []}, {'type': 'keyword'}, - ]) + ]) + def test_dot_suggests_cols_of_a_table_or_schema_qualified_table(): suggestions = suggest_type('SELECT tabl. FROM tabl', 'SELECT tabl.') @@ -185,24 +209,27 @@ def test_dot_suggests_cols_of_a_table_or_schema_qualified_table(): {'type': 'view', 'schema': 'tabl'}, {'type': 'function', 'schema': 'tabl'}]) + def test_dot_suggests_cols_of_an_alias(): suggestions = suggest_type('SELECT t1. FROM tabl1 t1, tabl2 t2', - 'SELECT t1.') + 'SELECT t1.') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'table', 'schema': 't1'}, {'type': 'view', 'schema': 't1'}, {'type': 'column', 'tables': [(None, 'tabl1', 't1')]}, {'type': 'function', 'schema': 't1'}]) + def test_dot_col_comma_suggests_cols_or_schema_qualified_table(): suggestions = suggest_type('SELECT t1.a, t2. FROM tabl1 t1, tabl2 t2', - 'SELECT t1.a, t2.') + 'SELECT t1.a, t2.') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl2', 't2')]}, {'type': 'table', 'schema': 't2'}, {'type': 'view', 'schema': 't2'}, {'type': 'function', 'schema': 't2'}]) + @pytest.mark.parametrize('expression', [ 'SELECT * FROM (', 'SELECT * FROM foo WHERE EXISTS (', @@ -212,6 +239,7 @@ def test_sub_select_suggests_keyword(expression): suggestion = suggest_type(expression, expression) assert suggestion == [{'type': 'keyword'}] + @pytest.mark.parametrize('expression', [ 'SELECT * FROM (S', 'SELECT * FROM foo WHERE EXISTS (S', @@ -221,6 +249,7 @@ def test_sub_select_partial_text_suggests_keyword(expression): suggestion = suggest_type(expression, expression) assert suggestion == [{'type': 'keyword'}] + def test_outer_table_reference_in_exists_subquery_suggests_columns(): q = 'SELECT * FROM foo f WHERE EXISTS (SELECT 1 FROM bar WHERE f.' suggestions = suggest_type(q, q) @@ -230,6 +259,7 @@ def test_outer_table_reference_in_exists_subquery_suggests_columns(): {'type': 'view', 'schema': 'f'}, {'type': 'function', 'schema': 'f'}] + @pytest.mark.parametrize('expression', [ 'SELECT * FROM (SELECT * FROM ', 'SELECT * FROM foo WHERE EXISTS (SELECT * FROM ', @@ -242,32 +272,36 @@ def test_sub_select_table_name_completion(expression): {'type': 'view', 'schema': []}, {'type': 'schema'}]) + def test_sub_select_col_name_completion(): suggestions = suggest_type('SELECT * FROM (SELECT FROM abc', - 'SELECT * FROM (SELECT ') + 'SELECT * FROM (SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'abc', None)]}, {'type': 'function', 'schema': []}, {'type': 'keyword'}, - ]) + ]) + @pytest.mark.xfail def test_sub_select_multiple_col_name_completion(): suggestions = suggest_type('SELECT * FROM (SELECT a, FROM abc', - 'SELECT * FROM (SELECT a, ') + 'SELECT * FROM (SELECT a, ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'abc', None)]}, {'type': 'function', 'schema': []}]) + def test_sub_select_dot_col_name_completion(): suggestions = suggest_type('SELECT * FROM (SELECT t. FROM tabl t', - 'SELECT * FROM (SELECT t.') + 'SELECT * FROM (SELECT t.') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl', 't')]}, {'type': 'table', 'schema': 't'}, {'type': 'view', 'schema': 't'}, {'type': 'function', 'schema': 't'}]) + @pytest.mark.parametrize('join_type', ['', 'INNER', 'LEFT', 'RIGHT OUTER']) @pytest.mark.parametrize('tbl_alias', ['', 'foo']) def test_join_suggests_tables_and_schemas(tbl_alias, join_type): @@ -278,6 +312,7 @@ def test_join_suggests_tables_and_schemas(tbl_alias, join_type): {'type': 'view', 'schema': []}, {'type': 'schema'}]) + @pytest.mark.parametrize('sql', [ 'SELECT * FROM abc a JOIN def d ON a.', 'SELECT * FROM abc a JOIN def d ON a.id = d.id AND a.', @@ -290,6 +325,7 @@ def test_join_alias_dot_suggests_cols1(sql): {'type': 'view', 'schema': 'a'}, {'type': 'function', 'schema': 'a'}]) + @pytest.mark.parametrize('sql', [ 'SELECT * FROM abc a JOIN def d ON a.id = d.', 'SELECT * FROM abc a JOIN def d ON a.id = d.id AND a.id2 = d.', @@ -302,6 +338,7 @@ def test_join_alias_dot_suggests_cols2(sql): {'type': 'view', 'schema': 'd'}, {'type': 'function', 'schema': 'd'}]) + @pytest.mark.parametrize('sql', [ 'select a.x, b.y from abc a join bcd b on ', 'select a.x, b.y from abc a join bcd b on a.id = b.id OR ', @@ -310,6 +347,7 @@ def test_on_suggests_aliases(sql): suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['a', 'b']}] + @pytest.mark.parametrize('sql', [ 'select abc.x, bcd.y from abc join bcd on ', 'select abc.x, bcd.y from abc join bcd on abc.id = bcd.id AND ', @@ -318,6 +356,7 @@ def test_on_suggests_tables(sql): suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['abc', 'bcd']}] + @pytest.mark.parametrize('sql', [ 'select a.x, b.y from abc a join bcd b on a.id = ', 'select a.x, b.y from abc a join bcd b on a.id = b.id AND a.id2 = ', @@ -326,6 +365,7 @@ def test_on_suggests_aliases_right_side(sql): suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['a', 'b']}] + @pytest.mark.parametrize('sql', [ 'select abc.x, bcd.y from abc join bcd on ', 'select abc.x, bcd.y from abc join bcd on abc.id = bcd.id and ', @@ -348,57 +388,59 @@ def test_2_statements_2nd_current(): suggestions = suggest_type('select * from a; select * from ', 'select * from a; select * from ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) suggestions = suggest_type('select * from a; select from b', 'select * from a; select ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'b', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'b', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) # Should work even if first statement is invalid suggestions = suggest_type('select * from; select * from ', 'select * from; select * from ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) + def test_2_statements_1st_current(): suggestions = suggest_type('select * from ; select * from b', 'select * from ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) suggestions = suggest_type('select from a; select * from b', 'select ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'a', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'a', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + def test_3_statements_2nd_current(): suggestions = suggest_type('select * from a; select * from ; select * from c', 'select * from a; select * from ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) suggestions = suggest_type('select * from a; select from b; select * from c', 'select * from a; select ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'b', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'b', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) def test_create_db_with_template(): @@ -435,13 +477,15 @@ def test_handle_pre_completion_comma_gracefully(text): assert iter(suggestions) + def test_cross_join(): text = 'select * from v1 cross join v2 JOIN v1.id, ' suggestions = suggest_type(text, text) assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) + @pytest.mark.parametrize('expression', [ 'SELECT 1 AS ', diff --git a/test/test_completion_refresher.py b/test/test_completion_refresher.py index 8851eae65..1ed63774a 100644 --- a/test/test_completion_refresher.py +++ b/test/test_completion_refresher.py @@ -10,10 +10,11 @@ def refresher(): def test_ctor(refresher): - """ - Refresher object should contain a few handlers + """Refresher object should contain a few handlers. + :param refresher: :return: + """ assert len(refresher.refreshers) > 0 actual_handlers = list(refresher.refreshers.keys()) @@ -41,10 +42,11 @@ def test_refresh_called_once(refresher): def test_refresh_called_twice(refresher): - """ - If refresh is called a second time, it should be restarted + """If refresh is called a second time, it should be restarted. + :param refresher: :return: + """ callbacks = Mock() @@ -69,9 +71,10 @@ def dummy_bg_refresh(*args): def test_refresh_with_callbacks(refresher): - """ - Callbacks must be called + """Callbacks must be called. + :param refresher: + """ callbacks = [Mock()] sqlexecute_class = Mock() diff --git a/test/test_config.py b/test/test_config.py index 2a0d26c18..8ef8b7819 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -11,7 +11,7 @@ read_and_decrypt_mylogin_cnf, str_to_bool) with_pycryptodome = ['pycryptodome' in set([package.project_name for package in - pip.get_installed_distributions()])] + pip.get_installed_distributions()])] LOGIN_PATH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'mylogin.cnf')) diff --git a/test/test_dbspecial.py b/test/test_dbspecial.py index 17309b29c..3733c813f 100644 --- a/test/test_dbspecial.py +++ b/test/test_dbspecial.py @@ -2,10 +2,11 @@ from test_completion_engine import sorted_dicts from mycli.packages.special.utils import format_uptime + def test_u_suggests_databases(): suggestions = suggest_type('\\u ', '\\u ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'database'}]) + {'type': 'database'}]) def test_describe_table(): diff --git a/test/test_main.py b/test/test_main.py index 1271f18da..eb9c8623c 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -18,6 +18,7 @@ CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, '--password', PASSWORD, '_test_db'] + @dbtest def test_execute_arg(executor): run(executor, 'create table test (a text)') @@ -84,6 +85,7 @@ def test_batch_mode(executor): assert result.exit_code == 0 assert 'count(*)\n3\n\na\nabc\n' in result.output + @dbtest def test_batch_mode_table(executor): run(executor, '''create table test(a text)''') @@ -112,6 +114,7 @@ def test_batch_mode_table(executor): assert result.exit_code == 0 assert expected in result.output + @dbtest def test_batch_mode_csv(executor): run(executor, '''create table test(a text, b text)''') @@ -127,6 +130,7 @@ def test_batch_mode_csv(executor): assert result.exit_code == 0 assert expected in result.output + def test_query_starts_with(executor): query = 'USE test;' assert query_starts_with(query, ('use', )) is True @@ -134,10 +138,12 @@ def test_query_starts_with(executor): query = 'DROP DATABASE test;' assert query_starts_with(query, ('use', )) is False + def test_query_starts_with_comment(executor): query = '# comment\nUSE test;' assert query_starts_with(query, ('use', )) is True + def test_queries_start_with(executor): sql = ( '# comment\n' @@ -148,6 +154,7 @@ def test_queries_start_with(executor): assert queries_start_with(sql, ('use', 'drop')) is True assert queries_start_with(sql, ('delete', 'update')) is False + def test_is_destructive(executor): sql = ( 'use test;\n' @@ -156,6 +163,7 @@ def test_is_destructive(executor): ) assert is_destructive(sql) is True + def test_confirm_destructive_query_notty(executor): stdin = click.get_text_stream('stdin') assert stdin.isatty() is False @@ -163,6 +171,7 @@ def test_confirm_destructive_query_notty(executor): sql = 'drop database foo;' assert confirm_destructive_query(sql) is None + def test_thanks_picker_utf8(): project_root = os.path.dirname(PACKAGE_ROOT) author_file = os.path.join(project_root, 'AUTHORS') diff --git a/test/test_naive_completion.py b/test/test_naive_completion.py index 57d738bca..3282c7ed1 100644 --- a/test/test_naive_completion.py +++ b/test/test_naive_completion.py @@ -3,16 +3,19 @@ from prompt_toolkit.completion import Completion from prompt_toolkit.document import Document + @pytest.fixture def completer(): import mycli.sqlcompleter as sqlcompleter return sqlcompleter.SQLCompleter(smart_completion=False) + @pytest.fixture def complete_event(): from mock import Mock return Mock() + def test_empty_string_completion(completer, complete_event): text = '' position = 0 @@ -21,6 +24,7 @@ def test_empty_string_completion(completer, complete_event): complete_event)) assert result == set(map(Completion, completer.all_completions)) + def test_select_keyword_completion(completer, complete_event): text = 'SEL' position = len('SEL') @@ -29,6 +33,7 @@ def test_select_keyword_completion(completer, complete_event): complete_event)) assert result == set([Completion(text='SELECT', start_position=-3)]) + def test_function_name_completion(completer, complete_event): text = 'SELECT MA' position = len('SELECT MA') @@ -39,6 +44,7 @@ def test_function_name_completion(completer, complete_event): Completion(text='MAX', start_position=-2), Completion(text='MASTER', start_position=-2)]) + def test_column_name_completion(completer, complete_event): text = 'SELECT FROM users' position = len('SELECT ') diff --git a/test/test_parseutils.py b/test/test_parseutils.py index e512632c1..e45cab78b 100644 --- a/test/test_parseutils.py +++ b/test/test_parseutils.py @@ -6,50 +6,62 @@ def test_empty_string(): tables = extract_tables('') assert tables == [] + def test_simple_select_single_table(): tables = extract_tables('select * from abc') assert tables == [(None, 'abc', None)] + def test_simple_select_single_table_schema_qualified(): tables = extract_tables('select * from abc.def') assert tables == [('abc', 'def', None)] + def test_simple_select_multiple_tables(): tables = extract_tables('select * from abc, def') assert sorted(tables) == [(None, 'abc', None), (None, 'def', None)] + def test_simple_select_multiple_tables_schema_qualified(): tables = extract_tables('select * from abc.def, ghi.jkl') assert sorted(tables) == [('abc', 'def', None), ('ghi', 'jkl', None)] + def test_simple_select_with_cols_single_table(): tables = extract_tables('select a,b from abc') assert tables == [(None, 'abc', None)] + def test_simple_select_with_cols_single_table_schema_qualified(): tables = extract_tables('select a,b from abc.def') assert tables == [('abc', 'def', None)] + def test_simple_select_with_cols_multiple_tables(): tables = extract_tables('select a,b from abc, def') assert sorted(tables) == [(None, 'abc', None), (None, 'def', None)] + def test_simple_select_with_cols_multiple_tables_with_schema(): tables = extract_tables('select a,b from abc.def, def.ghi') assert sorted(tables) == [('abc', 'def', None), ('def', 'ghi', None)] + def test_select_with_hanging_comma_single_table(): tables = extract_tables('select a, from abc') assert tables == [(None, 'abc', None)] + def test_select_with_hanging_comma_multiple_tables(): tables = extract_tables('select a, from abc, def') assert sorted(tables) == [(None, 'abc', None), (None, 'def', None)] + def test_select_with_hanging_period_multiple_tables(): tables = extract_tables('SELECT t1. FROM tabl1 t1, tabl2 t2') assert sorted(tables) == [(None, 'tabl1', 't1'), (None, 'tabl2', 't2')] + def test_simple_insert_single_table(): tables = extract_tables('insert into abc (id, name) values (1, "def")') @@ -57,27 +69,34 @@ def test_simple_insert_single_table(): # assert tables == [(None, 'abc', None)] assert tables == [(None, 'abc', 'abc')] + @pytest.mark.xfail def test_simple_insert_single_table_schema_qualified(): tables = extract_tables('insert into abc.def (id, name) values (1, "def")') assert tables == [('abc', 'def', None)] + def test_simple_update_table(): tables = extract_tables('update abc set id = 1') assert tables == [(None, 'abc', None)] + def test_simple_update_table_with_schema(): tables = extract_tables('update abc.def set id = 1') assert tables == [('abc', 'def', None)] + def test_join_table(): tables = extract_tables('SELECT * FROM abc a JOIN def d ON a.id = d.num') assert sorted(tables) == [(None, 'abc', 'a'), (None, 'def', 'd')] + def test_join_table_schema_qualified(): - tables = extract_tables('SELECT * FROM abc.def x JOIN ghi.jkl y ON x.id = y.num') + tables = extract_tables( + 'SELECT * FROM abc.def x JOIN ghi.jkl y ON x.id = y.num') assert tables == [('abc', 'def', 'x'), ('ghi', 'jkl', 'y')] + def test_join_as_table(): tables = extract_tables('SELECT * FROM my_table AS m WHERE m.a > 5') assert tables == [(None, 'my_table', 'm')] diff --git a/test/test_smart_completion_public_schema_only.py b/test/test_smart_completion_public_schema_only.py index e99567a07..6cdc9c307 100644 --- a/test/test_smart_completion_public_schema_only.py +++ b/test/test_smart_completion_public_schema_only.py @@ -5,11 +5,12 @@ from prompt_toolkit.document import Document metadata = { - 'users': ['id', 'email', 'first_name', 'last_name'], - 'orders': ['id', 'ordered_date', 'status'], - 'select': ['id', 'insert', 'ABC'], - 'réveillé': ['id', 'insert', 'ABC'] - } + 'users': ['id', 'email', 'first_name', 'last_name'], + 'orders': ['id', 'ordered_date', 'status'], + 'select': ['id', 'insert', 'ABC'], + 'réveillé': ['id', 'insert', 'ABC'] +} + @pytest.fixture def completer(): @@ -30,11 +31,13 @@ def completer(): return comp + @pytest.fixture def complete_event(): from mock import Mock return Mock() + def test_empty_string_completion(completer, complete_event): text = '' position = 0 @@ -44,6 +47,7 @@ def test_empty_string_completion(completer, complete_event): complete_event)) assert set(map(Completion, completer.keywords)) == result + def test_select_keyword_completion(completer, complete_event): text = 'SEL' position = len('SEL') @@ -73,12 +77,14 @@ def test_function_name_completion(completer, complete_event): Completion(text='MASTER', start_position=-2), ]) + def test_suggested_column_names(completer, complete_event): - """ - Suggest column and function names when selecting from table + """Suggest column and function names when selecting from table. + :param completer: :param complete_event: :return: + """ text = 'SELECT from users' position = len('SELECT ') @@ -94,13 +100,15 @@ def test_suggested_column_names(completer, complete_event): list(map(Completion, completer.functions)) + list(map(Completion, completer.keywords))) + def test_suggested_column_names_in_function(completer, complete_event): - """ - Suggest column and function names when selecting multiple - columns from table + """Suggest column and function names when selecting multiple columns from + table. + :param completer: :param complete_event: :return: + """ text = 'SELECT MAX( from users' position = len('SELECT MAX(') @@ -114,12 +122,14 @@ def test_suggested_column_names_in_function(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_column_names_with_table_dot(completer, complete_event): - """ - Suggest column names on table name and dot + """Suggest column names on table name and dot. + :param completer: :param complete_event: :return: + """ text = 'SELECT users. from users' position = len('SELECT users.') @@ -133,12 +143,14 @@ def test_suggested_column_names_with_table_dot(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_column_names_with_alias(completer, complete_event): - """ - Suggest column names on table alias and dot + """Suggest column names on table alias and dot. + :param completer: :param complete_event: :return: + """ text = 'SELECT u. from users u' position = len('SELECT u.') @@ -152,13 +164,15 @@ def test_suggested_column_names_with_alias(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_multiple_column_names(completer, complete_event): - """ - Suggest column and function names when selecting multiple - columns from table + """Suggest column and function names when selecting multiple columns from + table. + :param completer: :param complete_event: :return: + """ text = 'SELECT id, from users u' position = len('SELECT id, ') @@ -174,13 +188,15 @@ def test_suggested_multiple_column_names(completer, complete_event): list(map(Completion, completer.functions)) + list(map(Completion, completer.keywords))) + def test_suggested_multiple_column_names_with_alias(completer, complete_event): - """ - Suggest column names on table alias and dot - when selecting multiple columns from table + """Suggest column names on table alias and dot when selecting multiple + columns from table. + :param completer: :param complete_event: :return: + """ text = 'SELECT u.id, u. from users u' position = len('SELECT u.id, u.') @@ -194,13 +210,15 @@ def test_suggested_multiple_column_names_with_alias(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_multiple_column_names_with_dot(completer, complete_event): - """ - Suggest column names on table names and dot - when selecting multiple columns from table + """Suggest column names on table names and dot when selecting multiple + columns from table. + :param completer: :param complete_event: :return: + """ text = 'SELECT users.id, users. from users u' position = len('SELECT users.id, users.') @@ -214,6 +232,7 @@ def test_suggested_multiple_column_names_with_dot(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_aliases_after_on(completer, complete_event): text = 'SELECT u.name, o.id FROM users u JOIN orders o ON ' position = len('SELECT u.name, o.id FROM users u JOIN orders o ON ') @@ -224,9 +243,11 @@ def test_suggested_aliases_after_on(completer, complete_event): Completion(text='u', start_position=0), Completion(text='o', start_position=0)]) + def test_suggested_aliases_after_on_right_side(completer, complete_event): text = 'SELECT u.name, o.id FROM users u JOIN orders o ON o.user_id = ' - position = len('SELECT u.name, o.id FROM users u JOIN orders o ON o.user_id = ') + position = len( + 'SELECT u.name, o.id FROM users u JOIN orders o ON o.user_id = ') result = set(completer.get_completions( Document(text=text, cursor_position=position), complete_event)) @@ -234,6 +255,7 @@ def test_suggested_aliases_after_on_right_side(completer, complete_event): Completion(text='u', start_position=0), Completion(text='o', start_position=0)]) + def test_suggested_tables_after_on(completer, complete_event): text = 'SELECT users.name, orders.id FROM users JOIN orders ON ' position = len('SELECT users.name, orders.id FROM users JOIN orders ON ') @@ -244,9 +266,11 @@ def test_suggested_tables_after_on(completer, complete_event): Completion(text='users', start_position=0), Completion(text='orders', start_position=0)]) + def test_suggested_tables_after_on_right_side(completer, complete_event): text = 'SELECT users.name, orders.id FROM users JOIN orders ON orders.user_id = ' - position = len('SELECT users.name, orders.id FROM users JOIN orders ON orders.user_id = ') + position = len( + 'SELECT users.name, orders.id FROM users JOIN orders ON orders.user_id = ') result = set(completer.get_completions( Document(text=text, cursor_position=position), complete_event)) @@ -254,6 +278,7 @@ def test_suggested_tables_after_on_right_side(completer, complete_event): Completion(text='users', start_position=0), Completion(text='orders', start_position=0)]) + def test_table_names_after_from(completer, complete_event): text = 'SELECT * FROM ' position = len('SELECT * FROM ') @@ -265,7 +290,8 @@ def test_table_names_after_from(completer, complete_event): Completion(text='orders', start_position=0), Completion(text='`réveillé`', start_position=0), Completion(text='`select`', start_position=0), - ]) + ]) + def test_auto_escaped_col_names(completer, complete_event): text = 'SELECT from `select`' @@ -281,6 +307,7 @@ def test_auto_escaped_col_names(completer, complete_event): list(map(Completion, completer.functions)) + list(map(Completion, completer.keywords))) + def test_un_escaped_table_names(completer, complete_event): text = 'SELECT from réveillé' position = len('SELECT ') diff --git a/test/test_special_iocommands.py b/test/test_special_iocommands.py index 2ed2dbad8..708c7fd77 100644 --- a/test/test_special_iocommands.py +++ b/test/test_special_iocommands.py @@ -23,18 +23,21 @@ def test_set_get_pager(): mycli.packages.special.disable_pager() assert not mycli.packages.special.is_pager_enabled() + def test_set_get_timing(): mycli.packages.special.set_timing_enabled(True) assert mycli.packages.special.is_timing_enabled() mycli.packages.special.set_timing_enabled(False) assert not mycli.packages.special.is_timing_enabled() + def test_set_get_expanded_output(): mycli.packages.special.set_expanded_output(True) assert mycli.packages.special.is_expanded_output() mycli.packages.special.set_expanded_output(False) assert not mycli.packages.special.is_expanded_output() + def test_editor_command(): assert mycli.packages.special.editor_command(r'hello\e') assert mycli.packages.special.editor_command(r'\ehello') @@ -45,14 +48,15 @@ def test_editor_command(): os.environ['EDITOR'] = 'true' mycli.packages.special.open_external_editor(r'select 1') == "select 1" + def test_tee_command(): - mycli.packages.special.write_tee(u"hello world") # write without file set + mycli.packages.special.write_tee(u"hello world") # write without file set with tempfile.NamedTemporaryFile() as f: - mycli.packages.special.execute(None, u"tee "+f.name) + mycli.packages.special.execute(None, u"tee " + f.name) mycli.packages.special.write_tee(u"hello world") assert f.read() == b"hello world\n" - mycli.packages.special.execute(None, u"tee -o "+f.name) + mycli.packages.special.execute(None, u"tee -o " + f.name) mycli.packages.special.write_tee(u"hello world") f.seek(0) assert f.read() == b"hello world\n" @@ -62,6 +66,7 @@ def test_tee_command(): f.seek(0) assert f.read() == b"hello world\n" + def test_tee_command_error(): with pytest.raises(TypeError): mycli.packages.special.execute(None, 'tee') @@ -71,8 +76,10 @@ def test_tee_command_error(): os.chmod(f.name, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) mycli.packages.special.execute(None, 'tee {}'.format(f.name)) + def test_favorite_query(): with utils.db_connection().cursor() as cur: query = u'select "✔"' mycli.packages.special.execute(cur, u'\\fs check {0}'.format(query)) - assert next(mycli.packages.special.execute(cur, u'\\f check'))[0] == "> " + query + assert next(mycli.packages.special.execute( + cur, u'\\f check'))[0] == "> " + query diff --git a/test/test_sqlexecute.py b/test/test_sqlexecute.py index 157b62341..aebfa1a66 100644 --- a/test/test_sqlexecute.py +++ b/test/test_sqlexecute.py @@ -20,6 +20,7 @@ def test_conn(executor): +-----+ 1 row in set""") + @dbtest def test_bools(executor): run(executor, '''create table test(a boolean)''') @@ -33,6 +34,7 @@ def test_bools(executor): +---+ 1 row in set""") + @dbtest def test_binary(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') @@ -46,6 +48,7 @@ def test_binary(executor): +----------------------------------------------------------------------------------------------+ 1 row in set""") + @dbtest def test_binary_expanded(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') @@ -57,6 +60,7 @@ def test_binary_expanded(executor): 1 row in set""") + @dbtest def test_table_and_columns_query(executor): run(executor, "create table a(x text, y text)") @@ -64,25 +68,29 @@ def test_table_and_columns_query(executor): assert set(executor.tables()) == set([('a',), ('b',)]) assert set(executor.table_columns()) == set( - [('a', 'x'), ('a', 'y'), ('b', 'z')]) + [('a', 'x'), ('a', 'y'), ('b', 'z')]) + @dbtest def test_database_list(executor): databases = executor.databases() assert '_test_db' in databases + @dbtest def test_invalid_syntax(executor): with pytest.raises(pymysql.ProgrammingError) as excinfo: run(executor, 'invalid syntax!') assert 'You have an error in your SQL syntax;' in str(excinfo.value) + @dbtest def test_invalid_column_name(executor): with pytest.raises(pymysql.InternalError) as excinfo: run(executor, 'select invalid command') assert "Unknown column 'invalid' in 'field list'" in str(excinfo.value) + @dbtest def test_unicode_support_in_output(executor): run(executor, "create table unicodechars(t text)") @@ -91,6 +99,7 @@ def test_unicode_support_in_output(executor): # See issue #24, this raises an exception without proper handling assert u'é' in run(executor, u"select * from unicodechars", join=True) + @dbtest def test_expanded_output(executor): run(executor, '''create table test(a text)''') @@ -112,19 +121,23 @@ def test_expanded_output(executor): assert results in expected_results + @dbtest def test_multiple_queries_same_line(executor): result = run(executor, "select 'foo'; select 'bar'") - assert len(result) == 4 # 2 for the results and 2 more for status messages. + # 2 for the results and 2 more for status messages. + assert len(result) == 4 assert "foo" in result[0] assert "bar" in result[2] + @dbtest def test_multiple_queries_same_line_syntaxerror(executor): with pytest.raises(pymysql.ProgrammingError) as excinfo: run(executor, "select 'foo'; invalid syntax") assert 'You have an error in your SQL syntax;' in str(excinfo.value) + @dbtest def test_favorite_query(executor): set_expanded_output(False) @@ -147,6 +160,7 @@ def test_favorite_query(executor): results = run(executor, "\\fd test-a") assert results == ['test-a: Deleted'] + @dbtest def test_favorite_query_multiple_statement(executor): set_expanded_output(False) @@ -176,6 +190,7 @@ def test_favorite_query_multiple_statement(executor): results = run(executor, "\\fd test-ad") assert results == ['test-ad: Deleted'] + @dbtest def test_favorite_query_expanded_output(executor): set_expanded_output(False) @@ -206,6 +221,7 @@ def test_favorite_query_expanded_output(executor): results = run(executor, "\\fd test-ae") assert results == ['test-ae: Deleted'] + @dbtest def test_special_command(executor): results = run(executor, '\\?') @@ -213,6 +229,7 @@ def test_special_command(executor): assert len(results) == 1 assert expected_line in results[0] + @dbtest def test_cd_command_without_a_folder_name(executor): results = run(executor, 'system cd') @@ -220,6 +237,7 @@ def test_cd_command_without_a_folder_name(executor): assert len(results) == 1 assert expected_line in results[0] + @dbtest def test_system_command_not_found(executor): results = run(executor, 'system xyz') @@ -227,6 +245,7 @@ def test_system_command_not_found(executor): expected_line = 'OSError:' assert expected_line in results[0] + @dbtest def test_system_command_output(executor): test_file_path = os.path.join(os.path.abspath('.'), 'test', 'test.txt') @@ -235,16 +254,19 @@ def test_system_command_output(executor): expected_line = u'mycli rocks!\n' assert expected_line == results[0] + @dbtest def test_cd_command_current_dir(executor): test_path = os.path.join(os.path.abspath('.'), 'test') results = run(executor, 'system cd {0}'.format(test_path)) assert os.getcwd() == test_path + @dbtest def test_unicode_support(executor): assert u'日本語' in run(executor, u"SELECT '日本語' AS japanese;", join=True) + @dbtest def test_favorite_query_multiline_statement(executor): set_expanded_output(False) @@ -274,6 +296,7 @@ def test_favorite_query_multiline_statement(executor): results = run(executor, "\\fd test-ad") assert results == ['test-ad: Deleted'] + @dbtest def test_timestamp_null(executor): run(executor, '''create table ts_null(a timestamp)''') @@ -287,6 +310,7 @@ def test_timestamp_null(executor): +---------------------+ 1 row in set""") + @dbtest def test_datetime_null(executor): run(executor, '''create table dt_null(a datetime)''') @@ -300,6 +324,7 @@ def test_datetime_null(executor): +---------------------+ 1 row in set""") + @dbtest def test_date_null(executor): run(executor, '''create table date_null(a date)''') @@ -313,6 +338,7 @@ def test_date_null(executor): +------------+ 1 row in set""") + @dbtest def test_time_null(executor): run(executor, '''create table time_null(a time)''') diff --git a/test/utils.py b/test/utils.py index b29e5e072..0d6b6a991 100644 --- a/test/utils.py +++ b/test/utils.py @@ -11,13 +11,15 @@ PORT = getenv('PYTEST_PORT', 3306) CHARSET = getenv('PYTEST_CHARSET', 'utf8') + def db_connection(dbname=None): conn = pymysql.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, - charset=CHARSET, - local_infile=False) + charset=CHARSET, + local_infile=False) conn.autocommit = True return conn + try: db_connection() CAN_CONNECT_TO_DB = True @@ -28,6 +30,7 @@ def db_connection(dbname=None): not CAN_CONNECT_TO_DB, reason="Need a mysql instance at localhost accessible by user 'root'") + def create_db(dbname): with db_connection().cursor() as cur: try: @@ -36,8 +39,9 @@ def create_db(dbname): except: pass + def run(executor, sql, join=False): - " Return string output for the sql to be run " + """Return string output for the sql to be run.""" result = [] # TODO: this needs to go away. `run()` should not test formatted output. @@ -51,6 +55,7 @@ def run(executor, sql, join=False): result = '\n'.join(result) return result + def set_expanded_output(is_expanded): - """ Pass-through for the tests """ + """Pass-through for the tests.""" return special.set_expanded_output(is_expanded) From 072f343989c33352536ee52293e58ad0cf0da60c Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Fri, 21 Apr 2017 20:48:46 +0200 Subject: [PATCH 208/824] Add AUTHORS.rst and SPONSORS.rst with links to the text files --- AUTHORS.rst | 3 +++ SPONSORS.rst | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 AUTHORS.rst create mode 100644 SPONSORS.rst diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 000000000..995327f4b --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,3 @@ +Check out our `AUTHORS`_. + +.. _AUTHORS: mycli/AUTHORS diff --git a/SPONSORS.rst b/SPONSORS.rst new file mode 100644 index 000000000..173555c30 --- /dev/null +++ b/SPONSORS.rst @@ -0,0 +1,3 @@ +Check out our `SPONSORS`_. + +.. _SPONSORS: mycli/SPONSORS From 393ee3185423a3c2b8c21acb6965c1b758629cc3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 22 Apr 2017 12:16:20 -0500 Subject: [PATCH 209/824] Simplify author/sponsor file location code. --- mycli/main.py | 5 ++--- tests/test_main.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index b24adeadf..df3f789c3 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -439,9 +439,8 @@ def run_cli(self): if self.smart_completion: self.refresh_completions() - project_root = os.path.join(os.path.dirname(PACKAGE_ROOT), 'mycli') - author_file = os.path.join(project_root, 'AUTHORS') - sponsor_file = os.path.join(project_root, 'SPONSORS') + author_file = os.path.join(PACKAGE_ROOT, 'AUTHORS') + sponsor_file = os.path.join(PACKAGE_ROOT, 'SPONSORS') key_binding_manager = mycli_bindings() diff --git a/tests/test_main.py b/tests/test_main.py index 56c0cb267..791fd5dae 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -164,9 +164,8 @@ def test_confirm_destructive_query_notty(executor): assert confirm_destructive_query(sql) is None def test_thanks_picker_utf8(): - project_root = os.path.join(os.path.dirname(PACKAGE_ROOT), 'mycli') - author_file = os.path.join(project_root, 'AUTHORS') - sponsor_file = os.path.join(project_root, 'SPONSORS') + author_file = os.path.join(PACKAGE_ROOT, 'AUTHORS') + sponsor_file = os.path.join(PACKAGE_ROOT, 'SPONSORS') name = thanks_picker((author_file, sponsor_file)) assert isinstance(name, text_type) From d9490a7c7f080039eff0f8f53a0a59027846495d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 22 Apr 2017 14:03:25 -0500 Subject: [PATCH 210/824] Move cryptography changelog item to appropriate section. --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index dfc4c7270..c1643e0d3 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,7 @@ Internal Changes: * Rename tests/ to test/. (Thanks: [Dick Marinus]). * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). +* Switch from pycryptodome to cryptography (Thanks: [Thomas Roten]). 1.10.0: ======= @@ -37,7 +38,6 @@ Internal Changes: * Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). * Run pep8 checks in travis (Thanks: [Irina Truong]). * Remove temporary hack for sqlparse (Thanks: [Dick Marinus]). -* Switch from pycryptodome to cryptography (Thanks: [Thomas Roten]). 1.9.0: ====== From 13df01d379a4469291d57155e2d8a714b2af9ad7 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 25 Apr 2017 07:38:23 +0200 Subject: [PATCH 211/824] behave pager wrapper --- changelog.md | 1 + test/features/environment.py | 4 +++- test/features/steps/basic_commands.py | 2 ++ test/features/steps/crud_database.py | 8 ++++++-- test/features/steps/crud_table.py | 22 +++++++++++++++------- test/features/steps/iocommands.py | 2 +- test/features/steps/named_queries.py | 4 ++-- test/features/steps/specials.py | 6 +++++- test/features/steps/wrappers.py | 5 +++++ test/features/wrappager.py | 16 ++++++++++++++++ 10 files changed, 56 insertions(+), 14 deletions(-) create mode 100755 test/features/wrappager.py diff --git a/changelog.md b/changelog.md index b7cc8c3a3..961eb54c6 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,7 @@ Internal Changes: * Rename tests/ to test/. (Thanks: [Dick Marinus]). * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). +* Add pager wrapper for behave tests (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/features/environment.py b/test/features/environment.py index e79e5740c..376fce723 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -12,7 +12,6 @@ def before_all(context): """Set env parameters.""" os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" - os.environ['PAGER'] = 'cat' os.environ['EDITOR'] = 'ex' os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../.coveragerc" @@ -43,7 +42,10 @@ def before_all(context): 'dbname': db_name, 'dbname_tmp': db_name_full + '_tmp', 'vi': vi, + 'pager_boundary': '---boundary---', } + os.environ['PAGER'] = "{0} {1} {2}".format( + sys.executable, "test/features/wrappager.py", context.conf['pager_boundary']) context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], context.conf['pass'], diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index 109472b8a..845fea17e 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -59,3 +59,5 @@ def step_send_help(context): """ context.cli.sendline('\\?') + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) diff --git a/test/features/steps/crud_database.py b/test/features/steps/crud_database.py index d7b8eeb28..a7616dad3 100644 --- a/test/features/steps/crud_database.py +++ b/test/features/steps/crud_database.py @@ -73,20 +73,24 @@ def step_see_help(context): @then('we see database created') def step_see_db_created(context): """Wait to see create database output.""" - wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see database dropped') def step_see_db_dropped(context): """Wait to see drop database output.""" - wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see database connected') def step_see_db_connected(context): """Wait to see drop database output.""" + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) wrappers.expect_exact( context, 'You are now connected to database "', timeout=2) wrappers.expect_exact(context, '"', timeout=2) wrappers.expect_exact(context, ' as user "{0}"\r\n'.format( context.conf['user']), timeout=2) + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) diff --git a/test/features/steps/crud_table.py b/test/features/steps/crud_table.py index 34301c890..f6e9f044e 100644 --- a/test/features/steps/crud_table.py +++ b/test/features/steps/crud_table.py @@ -9,6 +9,7 @@ import wrappers from behave import when, then +from textwrap import dedent @when('we create table') @@ -56,35 +57,42 @@ def step_drop_table(context): @then('we see table created') def step_see_table_created(context): """Wait to see create table output.""" - wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see record inserted') def step_see_record_inserted(context): """Wait to see insert output.""" - wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see record updated') def step_see_record_updated(context): """Wait to see update output.""" - wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see data selected') def step_see_data_selected(context): """Wait to see select output.""" - wrappers.expect_exact( - context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) + wrappers.expect_pager( + context, dedent("""\ + +-----+\r + | x |\r + +-----+\r + | yyy |\r + +-----+\r + 1 row in set\r + """), timeout=1) @then('we see record deleted') def step_see_data_deleted(context): """Wait to see delete output.""" - wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see table dropped') def step_see_table_dropped(context): """Wait to see drop output.""" - wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 0 rows affected\r\n', timeout=2) diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 73068fac2..83cda6bc7 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -23,7 +23,7 @@ def step_edit_type_sql(context): context.cli.sendline('i') context.cli.sendline('select * from abc') context.cli.sendline('.') - wrappers.expect_exact(context, ':', timeout=2) + wrappers.expect_exact(context, '\r\n:', timeout=2) @when('we exit the editor') diff --git a/test/features/steps/named_queries.py b/test/features/steps/named_queries.py index 60115c5cd..40bf5bc72 100644 --- a/test/features/steps/named_queries.py +++ b/test/features/steps/named_queries.py @@ -32,7 +32,7 @@ def step_delete_named_query(context): @then('we see the named query saved') def step_see_named_query_saved(context): """Wait to see query saved.""" - wrappers.expect_exact(context, 'Saved.', timeout=1) + wrappers.expect_pager(context, 'Saved.\r\n', timeout=1) @then('we see the named query executed') @@ -45,4 +45,4 @@ def step_see_named_query_executed(context): @then('we see the named query deleted') def step_see_named_query_deleted(context): """Wait to see query deleted.""" - wrappers.expect_exact(context, 'foo: Deleted', timeout=1) + wrappers.expect_pager(context, 'foo: Deleted\r\n', timeout=1) diff --git a/test/features/steps/specials.py b/test/features/steps/specials.py index c0a3c0feb..f7715b9ed 100644 --- a/test/features/steps/specials.py +++ b/test/features/steps/specials.py @@ -21,4 +21,8 @@ def step_refresh_completions(context): def step_see_refresh_started(context): """Wait to see refresh output.""" wrappers.expect_exact( - context, 'Auto-completion refresh started in the background', timeout=2) + context, context.conf['pager_boundary'] + '\r\n', timeout=5) + wrappers.expect_exact( + context, 'Auto-completion refresh started in the background.\r\n', timeout=2) + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) diff --git a/test/features/steps/wrappers.py b/test/features/steps/wrappers.py index aea742033..e8d9204ab 100644 --- a/test/features/steps/wrappers.py +++ b/test/features/steps/wrappers.py @@ -14,3 +14,8 @@ def expect_exact(context, expected, timeout): raise Exception('Expected:\n---\n{0!r}\n---\n\nActual:\n---\n{1!r}\n---'.format( expected, actual)) + + +def expect_pager(context, expected, timeout): + expect_exact(context, "{0}\r\n{1}{0}\r\n".format( + context.conf['pager_boundary'], expected), timeout=timeout) diff --git a/test/features/wrappager.py b/test/features/wrappager.py new file mode 100755 index 000000000..51d490956 --- /dev/null +++ b/test/features/wrappager.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +import sys + + +def wrappager(boundary): + print(boundary) + while 1: + buf = sys.stdin.read(2048) + if not buf: + break + sys.stdout.write(buf) + print(boundary) + + +if __name__ == "__main__": + wrappager(sys.argv[1]) From 5074cccdbbb492d5416f6e7b540aae86f70332da Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Wed, 26 Apr 2017 17:21:16 -0700 Subject: [PATCH 212/824] Fail on first error in travis script. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 021be1c17..de590fc05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,12 +11,14 @@ install: - pip install git+https://github.com/hayd/pep8radius.git script: + - set -e - coverage run --source mycli -m py.test - cd test - behave - cd .. # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) + - set +e after_success: - coverage combine From 724883308f60eb5ddba31d901bc9b437967a1f6e Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Fri, 28 Apr 2017 21:30:31 +0200 Subject: [PATCH 213/824] test using behave the source command --- changelog.md | 1 + test/features/basic_commands.feature | 6 ++++++ test/features/steps/basic_commands.py | 11 +++++++++++ 3 files changed, 18 insertions(+) diff --git a/changelog.md b/changelog.md index 961eb54c6..1df05d693 100644 --- a/changelog.md +++ b/changelog.md @@ -7,6 +7,7 @@ Internal Changes: * Rename tests/ to test/. (Thanks: [Dick Marinus]). * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). * Add pager wrapper for behave tests (Thanks: [Dick Marinus]). +* Behave test source command (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/features/basic_commands.feature b/test/features/basic_commands.feature index 227fe769b..025b58502 100644 --- a/test/features/basic_commands.feature +++ b/test/features/basic_commands.feature @@ -12,6 +12,12 @@ Feature: run the cli, and we send "\?" command then we see help output + Scenario: run source command + When we run dbcli + and we wait for prompt + and we send source command + then we see help output + Scenario: run the cli and exit When we run dbcli and we wait for prompt diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index 845fea17e..37f1e88a5 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -8,6 +8,7 @@ from __future__ import unicode_literals import pexpect +import tempfile from behave import when import wrappers @@ -61,3 +62,13 @@ def step_send_help(context): context.cli.sendline('\\?') wrappers.expect_exact( context, context.conf['pager_boundary'] + '\r\n', timeout=5) + + +@when(u'we send source command') +def step_send_source_command(context): + with tempfile.NamedTemporaryFile() as f: + f.write(b'\?') + f.flush() + context.cli.sendline('\. {0}'.format(f.name)) + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) From b2b84bc8af9dd17baff9125ca67bddf6a50bd189 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 07:51:33 +0200 Subject: [PATCH 214/824] behave fix clean up In an earlier commit I've changed the current working directory and the removal of a temporary file didn't take that into account. --- changelog.md | 1 + test/features/steps/iocommands.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 1df05d693..8db8c1a16 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,7 @@ Internal Changes: * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). * Add pager wrapper for behave tests (Thanks: [Dick Marinus]). * Behave test source command (Thanks: [Dick Marinus]). +* Behave fix clean up. (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 83cda6bc7..9a713a961 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -9,10 +9,12 @@ @when('we start external editor providing a file name') def step_edit_file(context): """Edit file with external editor.""" - context.editor_file_name = 'test_file_{0}.sql'.format(context.conf['vi']) + context.editor_file_name = '../test_file_{0}.sql'.format( + context.conf['vi']) if os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) - context.cli.sendline('\e {0}'.format(context.editor_file_name)) + context.cli.sendline('\e {0}'.format( + os.path.basename(context.editor_file_name))) wrappers.expect_exact( context, 'Entering Ex mode. Type "visual" to go to Normal mode.', timeout=2) wrappers.expect_exact(context, '\r\n:', timeout=2) From e5b889ba39675de10da9e094ede4a552744c0c9d Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 07:56:04 +0200 Subject: [PATCH 215/824] test using behave the tee command --- changelog.md | 1 + test/features/iocommands.feature | 11 +++++++++ test/features/steps/iocommands.py | 39 +++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/changelog.md b/changelog.md index 1df05d693..d031714f6 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,7 @@ Internal Changes: * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). * Add pager wrapper for behave tests (Thanks: [Dick Marinus]). * Behave test source command (Thanks: [Dick Marinus]). +* Test using behave the tee command (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/features/iocommands.feature b/test/features/iocommands.feature index d043dc2ea..4bcdf6e65 100644 --- a/test/features/iocommands.feature +++ b/test/features/iocommands.feature @@ -8,3 +8,14 @@ Feature: I/O commands and we exit the editor then we see dbcli prompt and we see the sql in prompt + + Scenario: tee output from query + When we run dbcli + and we wait for prompt + and we tee output + and we wait for prompt + and we query "select 123456" + and we wait for prompt + and we notee output + and we wait for prompt + then we see 123456 in tee output diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 83cda6bc7..c8293a10c 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -4,6 +4,7 @@ import wrappers from behave import when, then +from textwrap import dedent @when('we start external editor providing a file name') @@ -41,3 +42,41 @@ def step_edit_done_sql(context): # Cleanup the edited file. if context.editor_file_name and os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) + + +@when(u'we tee output') +def step_tee_ouptut(context): + context.tee_file_name = '../tee_file_{0}.sql'.format(context.conf['vi']) + if os.path.exists(context.tee_file_name): + os.remove(context.tee_file_name) + context.cli.sendline('tee {0}'.format( + os.path.basename(context.tee_file_name))) + wrappers.expect_pager(context, "\r\n", timeout=5) + + +@when(u'we query "select 123456"') +def step_query_select_123456(context): + context.cli.sendline('select 123456') + wrappers.expect_pager(context, dedent("""\ + +--------+\r + | 123456 |\r + +--------+\r + | 123456 |\r + +--------+\r + 1 row in set\r + """), timeout=5) + + +@when(u'we notee output') +def step_notee_output(context): + context.cli.sendline('notee') + wrappers.expect_pager(context, "\r\n", timeout=5) + + +@then(u'we see 123456 in tee output') +def step_see_123456_in_ouput(context): + with open(context.tee_file_name) as f: + assert '123456' in f.read() + if os.path.exists(context.tee_file_name): + os.remove(context.tee_file_name) + context.atprompt = True From 8c6e73842131658fd11ab9907cd59a31c7c8384d Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Thu, 27 Apr 2017 20:39:10 +0200 Subject: [PATCH 216/824] behave quit mycli nicely Before this patch mycli is killed by expect and it the coverage data cannot be written. --- changelog.md | 1 + test/features/environment.py | 20 ++++++++++++++++++-- test/features/steps/basic_commands.py | 4 +++- test/features/steps/crud_database.py | 5 ++++- test/features/steps/iocommands.py | 2 +- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 18c332713..c47dd255b 100644 --- a/changelog.md +++ b/changelog.md @@ -11,6 +11,7 @@ Internal Changes: * Behave test source command (Thanks: [Dick Marinus]). * Test using behave the tee command (Thanks: [Dick Marinus]). * Behave fix clean up. (Thanks: [Dick Marinus]). +* Behave quit mycli nicely (Thanks: [Dick Marinus]) 1.10.0: ======= diff --git a/test/features/environment.py b/test/features/environment.py index 376fce723..a0456b99f 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -6,6 +6,7 @@ import sys import db_utils as dbutils import fixture_utils as fixutils +import pexpect def before_all(context): @@ -68,12 +69,27 @@ def after_all(context): # os.environ[k] = v +def before_step(context, _): + context.atprompt = False + + def after_scenario(context, _): """Cleans up after each test complete.""" if hasattr(context, 'cli') and not context.exit_sent: - # Terminate nicely. - context.cli.terminate() + # Quit nicely. + if not context.atprompt: + user = context.conf['user'] + host = context.conf['host'] + dbname = context.currentdb + context.cli.expect_exact( + 'mysql {0}@{1}:{2}> '.format( + user, host, dbname + ), + timeout=5 + ) + context.cli.sendcontrol('d') + context.cli.expect_exact(pexpect.EOF, timeout=5) # TODO: uncomment to debug a failure # def after_step(context, step): diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index 37f1e88a5..df97ee0e7 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -33,6 +33,7 @@ def step_run_cli(context): cmd = ' '.join(cmd_parts) context.cli = pexpect.spawnu(cmd, cwd='..') context.exit_sent = False + context.currentdb = context.conf['dbname'] @when('we wait for prompt') @@ -40,9 +41,10 @@ def step_wait_prompt(context): """Make sure prompt is displayed.""" user = context.conf['user'] host = context.conf['host'] - dbname = context.conf['dbname'] + dbname = context.currentdb wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( user, host, dbname), timeout=5) + context.atprompt = True @when('we send "ctrl + d"') diff --git a/test/features/steps/crud_database.py b/test/features/steps/crud_database.py index a7616dad3..af8580e16 100644 --- a/test/features/steps/crud_database.py +++ b/test/features/steps/crud_database.py @@ -39,12 +39,14 @@ def step_db_drop(context): def step_db_connect_test(context): """Send connect to database.""" db_name = context.conf['dbname'] + context.currentdb = db_name context.cli.sendline('use {0}'.format(db_name)) @when('we connect to dbserver') def step_db_connect_dbserver(context): """Send connect to database.""" + context.currentdb = 'mysql' context.cli.sendline('use mysql') @@ -59,9 +61,10 @@ def step_see_prompt(context): """Wait to see the prompt.""" user = context.conf['user'] host = context.conf['host'] - dbname = context.conf['dbname'] + dbname = context.currentdb wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( user, host, dbname), timeout=5) + context.atprompt = True @then('we see help output') diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 6641ce952..5de640ffb 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -40,7 +40,7 @@ def step_edit_done_sql(context): for match in 'select * from abc'.split(' '): wrappers.expect_exact(context, match, timeout=1) # Cleanup the command line. - context.cli.sendcontrol('u') + context.cli.sendcontrol('c') # Cleanup the edited file. if context.editor_file_name and os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) From 50bf54e04b00909f790457dd791d1c2b91a7d219 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 16:43:59 -0500 Subject: [PATCH 217/824] Do not add time from multiple queries together. --- mycli/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index df3f789c3..788e123eb 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -535,8 +535,7 @@ def one_iteration(document=None): max_width) output.extend(formatted) - end = time() - total += end - start + total = time() - start mutating = mutating or is_mutating(status) except KeyboardInterrupt: # get last connection id From 82b65bbc08918037d4d4286eb3fc23927b55266b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 16:45:28 -0500 Subject: [PATCH 218/824] Add timing bugfix to changelog. --- changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.md b/changelog.md index 18c332713..11465d0cd 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Bug Fixes: +---------- + +* Fixed incorrect timekeeping when running queries from a file. (Thanks: [Thomas Roten]). + Internal Changes: ----------------- From 88d470d2fda97dfae835ba7ac0ba99d997c3e907 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 20:41:52 -0500 Subject: [PATCH 219/824] Add configurable reserved lines for completion menu. --- changelog.md | 5 +++++ mycli/main.py | 25 ++++++++++++++----------- mycli/myclirc | 3 +++ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/changelog.md b/changelog.md index 18c332713..8422296d1 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Features: +--------- + +* Add option to control how much space is reserved for the completion menu. (Thanks: [Thomas Roten]). + Internal Changes: ----------------- diff --git a/mycli/main.py b/mycli/main.py index df3f789c3..043db60bf 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -113,6 +113,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') + self.min_num_menu_lines = c['main'].as_int('min_num_menu_lines') c_dest_warning = c['main'].as_bool('destructive_warning') self.destructive_warning = c_dest_warning if warn is None else warn self.login_path_as_host = c['main'].as_bool('login_path_as_host') @@ -602,17 +603,19 @@ def one_iteration(document=None): get_toolbar_tokens = create_toolbar_tokens_func(self.completion_refresher.is_refreshing) - layout = create_prompt_layout(lexer=MyCliLexer, - multiline=True, - get_prompt_tokens=prompt_tokens, - get_continuation_tokens=get_continuation_tokens, - get_bottom_toolbar_tokens=get_toolbar_tokens, - display_completions_in_columns=self.wider_completion_menu, - extra_input_processors=[ - ConditionalProcessor( - processor=HighlightMatchingBracketProcessor(chars='[](){}'), - filter=HasFocus(DEFAULT_BUFFER) & ~IsDone()), - ]) + layout = create_prompt_layout( + lexer=MyCliLexer, + multiline=True, + get_prompt_tokens=prompt_tokens, + get_continuation_tokens=get_continuation_tokens, + get_bottom_toolbar_tokens=get_toolbar_tokens, + display_completions_in_columns=self.wider_completion_menu, + extra_input_processors=[ConditionalProcessor( + processor=HighlightMatchingBracketProcessor(chars='[](){}'), + filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() + )], + reserve_space_for_menu=self.min_num_menu_lines + ) with self._completer_lock: buf = CLIBuffer(always_multiline=self.multi_line, completer=self.completer, history=FileHistory(os.path.expanduser(os.environ.get('MYCLI_HISTFILE', '~/.mycli-history'))), diff --git a/mycli/myclirc b/mycli/myclirc index 01a114265..2f45778b5 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -51,6 +51,9 @@ key_bindings = emacs # Enabling this option will show the suggestions in a wider menu. Thus more items are suggested. wider_completion_menu = False +# Number of lines to reserve for the completion menu. +min_num_menu_lines = 8 + # MySQL prompt # \t - Product type (Percona, MySQL, Mariadb) # \u - Username From b3a6839c69e2a37094ef894002e038ef0afc3df5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 21:09:15 -0500 Subject: [PATCH 220/824] Add current vi mode to toolbar. --- changelog.md | 5 +++++ mycli/clitoolbar.py | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 11465d0cd..dab38caab 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Features: +--------- + +* Display current vi mode in toolbar. (Thanks: [Thomas Roten]). + Bug Fixes: ---------- diff --git a/mycli/clitoolbar.py b/mycli/clitoolbar.py index b62d8edbe..79f2a8a75 100644 --- a/mycli/clitoolbar.py +++ b/mycli/clitoolbar.py @@ -1,5 +1,6 @@ from pygments.token import Token from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode +from prompt_toolkit.key_binding.vi_state import InputMode def create_toolbar_tokens_func(get_is_refreshing): """ @@ -26,7 +27,10 @@ def get_toolbar_tokens(cli): ' (Semi-colon [;] will end the line)')) if cli.editing_mode == EditingMode.VI: - result.append((token.On, '[F4] Vi-mode')) + result.append(( + token.On, + '[F4] Vi-mode ({})'.format(_get_vi_mode(cli)) + )) else: result.append((token.On, '[F4] Emacs-mode')) @@ -35,3 +39,13 @@ def get_toolbar_tokens(cli): return result return get_toolbar_tokens + + +def _get_vi_mode(cli): + """Get the current vi mode for display.""" + return { + InputMode.INSERT: 'I', + InputMode.NAVIGATION: 'N', + InputMode.REPLACE: 'R', + InputMode.INSERT_MULTIPLE: 'M' + }[cli.vi_state.input_mode] From f75ea9bec59121346c20b298828071413dd5e54c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 22:22:17 -0500 Subject: [PATCH 221/824] Make reserved space automatically calculate. --- changelog.md | 2 +- mycli/main.py | 10 ++++++++-- mycli/myclirc | 3 --- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index de0b12159..3819ce1a6 100644 --- a/changelog.md +++ b/changelog.md @@ -4,7 +4,7 @@ TBD Features: --------- -* Add option to control how much space is reserved for the completion menu. (Thanks: [Thomas Roten]). +* Handle reserved space for completion menu better in small windows. (Thanks: [Thomas Roten]). Bug Fixes: ---------- diff --git a/mycli/main.py b/mycli/main.py index 5f36cc193..eff09340a 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -113,7 +113,6 @@ def __init__(self, sqlexecute=None, prompt=None, self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') - self.min_num_menu_lines = c['main'].as_int('min_num_menu_lines') c_dest_warning = c['main'].as_bool('destructive_warning') self.destructive_warning = c_dest_warning if warn is None else warn self.login_path_as_host = c['main'].as_bool('login_path_as_host') @@ -613,7 +612,7 @@ def one_iteration(document=None): processor=HighlightMatchingBracketProcessor(chars='[](){}'), filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() )], - reserve_space_for_menu=self.min_num_menu_lines + reserve_space_for_menu=self.get_reserved_space() ) with self._completer_lock: buf = CLIBuffer(always_multiline=self.multi_line, completer=self.completer, @@ -750,6 +749,13 @@ def format_output(self, title, cur, headers, status, expanded=False, return output + def get_reserved_space(self): + """Get the number of lines to reserve for the completion menu.""" + reserved_space_ratio = .2 + max_reserved_space = 8 + _, height = click.get_terminal_size() + return min(int(height * reserved_space_ratio), max_reserved_space) + @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') diff --git a/mycli/myclirc b/mycli/myclirc index 2f45778b5..01a114265 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -51,9 +51,6 @@ key_bindings = emacs # Enabling this option will show the suggestions in a wider menu. Thus more items are suggested. wider_completion_menu = False -# Number of lines to reserve for the completion menu. -min_num_menu_lines = 8 - # MySQL prompt # \t - Product type (Percona, MySQL, Mariadb) # \u - Username From 535bac41984342e39723ff32419aa3a0efa5375d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:03:34 -0500 Subject: [PATCH 222/824] Increase reserved space ratio. --- mycli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index eff09340a..5df93408a 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -751,10 +751,10 @@ def format_output(self, title, cur, headers, status, expanded=False, def get_reserved_space(self): """Get the number of lines to reserve for the completion menu.""" - reserved_space_ratio = .2 + reserved_space_ratio = .45 max_reserved_space = 8 _, height = click.get_terminal_size() - return min(int(height * reserved_space_ratio), max_reserved_space) + return min(round(height * reserved_space_ratio), max_reserved_space) @click.command() From 7ec1c242d5b6a34c8c5e9a88a046b035de43cde2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:32:41 -0500 Subject: [PATCH 223/824] Add CLI Helpers dependency. --- mycli/main.py | 26 +++++++++++++------------- mycli/myclirc | 2 +- setup.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 5df93408a..c3a4aef0d 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -13,6 +13,7 @@ from random import choice from io import open +from cli_helpers.tabular_output import TabularOutputFormatter import click import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction @@ -37,7 +38,6 @@ from .config import (write_default_config, get_mylogin_cnf_path, open_mylogin_cnf, read_config_files, str_to_bool) from .key_bindings import mycli_bindings -from .output_formatter import output_formatter from .encodingutils import utf8tounicode from .lexer import MyCliLexer from .__init__ import __version__ @@ -107,7 +107,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.formatter = output_formatter.OutputFormatter( + self.formatter = TabularOutputFormatter( format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') @@ -149,7 +149,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.smart_completion = c['main'].as_bool('smart_completion') self.completer = SQLCompleter( self.smart_completion, - supported_formats=self.formatter.supported_formats()) + supported_formats=self.formatter.supported_formats) self._completer_lock = threading.Lock() # Register custom special commands. @@ -185,13 +185,13 @@ def register_special_commands(self): def change_table_format(self, arg, **_): try: - self.formatter.set_format_name(arg) + self.formatter.format_name = arg yield (None, None, None, 'Changed table type to {}'.format(arg)) except ValueError: msg = 'Table type {} not yet implemented. Allowed types:'.format( arg) - for table_type in self.formatter.supported_formats(): + for table_type in self.formatter.supported_formats: msg += "\n\t{}".format(table_type) yield (None, None, None, msg) @@ -673,7 +673,7 @@ def refresh_completions(self, reset=False): self.completion_refresher.refresh( self.sqlexecute, self._on_completions_refreshed, {'smart_completion': self.smart_completion, - 'supported_formats': self.formatter.supported_formats()}) + 'supported_formats': self.formatter.supported_formats}) return [(None, None, None, 'Auto-completion refresh started in the background.')] @@ -726,7 +726,7 @@ def run_query(self, query, new_line=True): def format_output(self, title, cur, headers, status, expanded=False, max_width=None): - expanded = expanded or self.formatter.get_format_name() == 'expanded' + expanded = expanded or self.formatter.format_name == 'vertical' output = [] if title: # Only print the title if it's not None. @@ -735,12 +735,12 @@ def format_output(self, title, cur, headers, status, expanded=False, if cur: rows = list(cur) formatted = self.formatter.format_output( - rows, headers, format_name='expanded' if expanded else None) + rows, headers, format_name='vertical' if expanded else None) if (not expanded and max_width and rows and content_exceeds_width(rows[0], max_width) and headers): formatted = self.formatter.format_output( - rows, headers, format_name='expanded') + rows, headers, format_name='vertical') output.append(formatted) @@ -855,9 +855,9 @@ def cli(database, user, host, port, socket, password, dbname, if execute: try: if csv: - mycli.formatter.set_format_name('csv') + mycli.formatter.format_name = 'csv' elif not table: - mycli.formatter.set_format_name('tsv') + mycli.formatter.format_name = 'tsv' mycli.run_query(execute) exit(0) @@ -883,10 +883,10 @@ def cli(database, user, host, port, socket, password, dbname, new_line = True if csv: - mycli.formatter.set_format_name('csv') + mycli.formatter.format_name = 'csv' new_line = False elif not table: - mycli.formatter.set_format_name('tsv') + mycli.formatter.format_name = 'tsv' mycli.run_query(stdin_text, new_line=new_line) exit(0) diff --git a/mycli/myclirc b/mycli/myclirc index 01a114265..ab57a2ed9 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -32,7 +32,7 @@ timing = True # Table format. Possible values: ascii, double, github, # psql, plain, simple, grid, fancy_grid, pipe, orgtbl, rst, mediawiki, html, -# latex, latex_booktabs, textile, moinmoin, jira, expanded, tsv, csv. +# latex, latex_booktabs, textile, moinmoin, jira, vertical, tsv, csv. # Recommended: ascii table_format = ascii diff --git a/setup.py b/setup.py index 27b761d90..5140b7e9b 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', 'cryptography >= 1.0.0', - 'terminaltables >= 3.0.0', + 'cli_helpers >= 0.1.0', ] setup( From 2baed47e9397600c413af235163f3ec9318eadee Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:33:02 -0500 Subject: [PATCH 224/824] Remove output formatter code and tests. --- mycli/output_formatter/__init__.py | 0 .../delimited_output_adapter.py | 28 - mycli/output_formatter/expanded.py | 34 - mycli/output_formatter/output_formatter.py | 95 -- mycli/output_formatter/preprocessors.py | 87 - mycli/output_formatter/tabulate_adapter.py | 22 - .../terminaltables_adapter.py | 25 - mycli/packages/tabulate.py | 1432 ----------------- test/test_expanded.py | 19 - test/test_output_formatter.py | 160 -- test/test_tabulate.py | 17 - 11 files changed, 1919 deletions(-) delete mode 100644 mycli/output_formatter/__init__.py delete mode 100644 mycli/output_formatter/delimited_output_adapter.py delete mode 100644 mycli/output_formatter/expanded.py delete mode 100644 mycli/output_formatter/output_formatter.py delete mode 100644 mycli/output_formatter/preprocessors.py delete mode 100644 mycli/output_formatter/tabulate_adapter.py delete mode 100644 mycli/output_formatter/terminaltables_adapter.py delete mode 100644 mycli/packages/tabulate.py delete mode 100644 test/test_expanded.py delete mode 100644 test/test_output_formatter.py delete mode 100644 test/test_tabulate.py diff --git a/mycli/output_formatter/__init__.py b/mycli/output_formatter/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py deleted file mode 100644 index a01a28433..000000000 --- a/mycli/output_formatter/delimited_output_adapter.py +++ /dev/null @@ -1,28 +0,0 @@ -import contextlib -import csv -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -from .preprocessors import override_missing_value, bytes_to_string - -supported_formats = ('csv', 'tsv') -preprocessors = (override_missing_value, bytes_to_string) - - -def adapter(data, headers, table_format='csv', **_): - """Wrap CSV formatting inside a standard function for OutputFormatter.""" - with contextlib.closing(StringIO()) as content: - if table_format == 'csv': - writer = csv.writer(content, delimiter=',') - elif table_format == 'tsv': - writer = csv.writer(content, delimiter='\t') - else: - raise ValueError('Invalid table_format specified.') - - writer.writerow(headers) - for row in data: - writer.writerow(row) - - return content.getvalue() diff --git a/mycli/output_formatter/expanded.py b/mycli/output_formatter/expanded.py deleted file mode 100644 index f77c1ee38..000000000 --- a/mycli/output_formatter/expanded.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Format data into a vertical, expanded table layout.""" - -from __future__ import unicode_literals - - -def get_separator(num): - """Get a row separator for row *num*.""" - return "{divider}[ {n}. row ]{divider}\n".format( - divider='*' * 27, n=num + 1) - - -def format_row(headers, row): - """Format a row.""" - formatted_row = [' | '.join(field) for field in zip(headers, row)] - return '\n'.join(formatted_row) - - -def expanded_table(rows, headers, **_): - """Format *rows* and *headers* as an expanded table. - - The values in *rows* and *headers* must be strings. - - """ - header_len = max([len(x) for x in headers]) - padded_headers = [x.ljust(header_len) for x in headers] - formatted_rows = [format_row(padded_headers, row) for row in rows] - - output = [] - for i, result in enumerate(formatted_rows): - output.append(get_separator(i)) - output.append(result) - output.append('\n') - - return ''.join(output) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py deleted file mode 100644 index 61e3c8d52..000000000 --- a/mycli/output_formatter/output_formatter.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- -"""A generic output formatter interface.""" - -from __future__ import unicode_literals -from collections import namedtuple - -from .expanded import expanded_table -from .preprocessors import (override_missing_value, convert_to_string) - -from . import delimited_output_adapter -from . import tabulate_adapter -from . import terminaltables_adapter - -MISSING_VALUE = '' - -OutputFormatHandler = namedtuple( - 'OutputFormatHandler', - 'format_name preprocessors formatter formatter_args') - - -class OutputFormatter(object): - """A class with a standard interface for various formatting libraries.""" - - _output_formats = {} - - def __init__(self, format_name=None): - """Set the default *format_name*.""" - self._format_name = format_name - - def set_format_name(self, format_name): - """Set the OutputFormatter's default format.""" - if format_name in self.supported_formats(): - self._format_name = format_name - else: - raise ValueError('unrecognized format_name: {}'.format( - format_name)) - - def get_format_name(self): - """Get the OutputFormatter's default format.""" - return self._format_name - - def supported_formats(self): - """Return the supported output format names.""" - return tuple(self._output_formats.keys()) - - @classmethod - def register_new_formatter(cls, format_name, handler, preprocessors=(), - kwargs={}): - """Register a new formatter to format the output.""" - cls._output_formats[format_name] = OutputFormatHandler( - format_name, preprocessors, handler, kwargs) - - def format_output(self, data, headers, format_name=None, **kwargs): - """Format the headers and data using a specific formatter. - - *format_name* must be a formatter available in `supported_formats()`. - - All keyword arguments are passed to the specified formatter. - - """ - format_name = format_name or self._format_name - if format_name not in self.supported_formats(): - raise ValueError('unrecognized format: {}'.format(format_name)) - - (_, preprocessors, formatter, - fkwargs) = self._output_formats[format_name] - fkwargs.update(kwargs) - if preprocessors: - for f in preprocessors: - data, headers = f(data, headers, **fkwargs) - return formatter(data, headers, **fkwargs) - - -OutputFormatter.register_new_formatter('expanded', expanded_table, - (override_missing_value, - convert_to_string), - {'missing_value': MISSING_VALUE}) - -for delimiter_format in delimited_output_adapter.supported_formats: - OutputFormatter.register_new_formatter( - delimiter_format, delimited_output_adapter.adapter, - delimited_output_adapter.preprocessors, - {'table_format': delimiter_format, 'missing_value': MISSING_VALUE}) - -for tabulate_format in tabulate_adapter.supported_formats: - OutputFormatter.register_new_formatter( - tabulate_format, tabulate_adapter.adapter, - tabulate_adapter.preprocessors, - {'table_format': tabulate_format, 'missing_value': MISSING_VALUE}) - -for terminaltables_format in terminaltables_adapter.supported_formats: - OutputFormatter.register_new_formatter( - terminaltables_format, terminaltables_adapter.adapter, - terminaltables_adapter.preprocessors, - {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py deleted file mode 100644 index 6f2e459c0..000000000 --- a/mycli/output_formatter/preprocessors.py +++ /dev/null @@ -1,87 +0,0 @@ -from decimal import Decimal - -from mycli import encodingutils - - -def to_string(value): - """Convert *value* to a string.""" - if isinstance(value, encodingutils.binary_type): - return encodingutils.bytes_to_string(value) - else: - return encodingutils.text_type(value) - - -def convert_to_string(data, headers, **_): - """Convert all *data* and *headers* to strings.""" - return ([[to_string(v) for v in row] for row in data], - [to_string(h) for h in headers]) - - -def override_missing_value(data, headers, missing_value='', **_): - """Override missing values in the data with *missing_value*.""" - return ([[missing_value if v is None else v for v in row] for row in data], - headers) - - -def bytes_to_string(data, headers, **_): - """Convert all *data* and *headers* bytes to strings.""" - return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], - [encodingutils.bytes_to_string(h) for h in headers]) - - -def intlen(value): - """Find (character) length. - - >>> intlen('11.1') - 2 - >>> intlen('11') - 2 - >>> intlen('1.1') - 1 - - """ - pos = value.find('.') - if pos < 0: - pos = len(value) - return pos - - -def align_decimals(data, headers, **_): - """Align decimals to decimal point.""" - pointpos = len(headers) * [0] - for row in data: - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - pointpos[i] = max(intlen(v), pointpos[i]) - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - result.append((pointpos[i] - intlen(v)) * " " + v) - else: - result.append(v) - results.append(result) - return results, headers - - -def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote leading/trailing whitespace.""" - quote = len(headers) * [False] - for row in data: - for i, v in enumerate(row): - v = encodingutils.text_type(v) - if v.startswith(' ') or v.endswith(' '): - quote[i] = True - - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - quotation = quotestyle if quote[i] else '' - result.append('{quotestyle}{value}{quotestyle}'.format( - quotestyle=quotation, value=v)) - results.append(result) - return results, headers diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py deleted file mode 100644 index b89dcc0bd..000000000 --- a/mycli/output_formatter/tabulate_adapter.py +++ /dev/null @@ -1,22 +0,0 @@ -from mycli.packages import tabulate -from .preprocessors import bytes_to_string, align_decimals - -tabulate.PRESERVE_WHITESPACE = True - -supported_markup_formats = ('mediawiki', 'html', 'latex', 'latex_booktabs', - 'textile', 'moinmoin', 'jira') -supported_table_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'psql', 'rst') -supported_formats = supported_markup_formats + supported_table_formats - -preprocessors = (bytes_to_string, align_decimals) - - -def adapter(data, headers, table_format=None, missing_value='', **_): - """Wrap tabulate inside a standard function for OutputFormatter.""" - kwargs = {'tablefmt': table_format, 'missingval': missing_value, - 'disable_numparse': True} - if table_format in supported_markup_formats: - kwargs.update(numalign=None, stralign=None) - - return tabulate.tabulate(data, headers, **kwargs) diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py deleted file mode 100644 index a8f50f985..000000000 --- a/mycli/output_formatter/terminaltables_adapter.py +++ /dev/null @@ -1,25 +0,0 @@ -import terminaltables - -from .preprocessors import (bytes_to_string, align_decimals, - override_missing_value) - -supported_formats = ('ascii', 'double', 'github') -preprocessors = (bytes_to_string, override_missing_value, align_decimals) - - -def adapter(data, headers, table_format=None, **_): - """Wrap terminaltables inside a standard function for OutputFormatter.""" - - table_format_handler = { - 'ascii': terminaltables.AsciiTable, - 'double': terminaltables.DoubleTable, - 'github': terminaltables.GithubFlavoredMarkdownTable, - } - - try: - table = table_format_handler[table_format] - except KeyError: - raise ValueError('unrecognized table format: {}'.format(table_format)) - - t = table([headers] + data) - return t.table diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py deleted file mode 100644 index 1e67cea7d..000000000 --- a/mycli/packages/tabulate.py +++ /dev/null @@ -1,1432 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Pretty-print tabular data.""" - -from __future__ import print_function -from __future__ import unicode_literals -from collections import namedtuple, Iterable -from platform import python_version_tuple -import re - - -if python_version_tuple()[0] < "3": - from itertools import izip_longest - from functools import partial - _none_type = type(None) - _bool_type = bool - _int_type = int - _long_type = long - _float_type = float - _text_type = unicode - _binary_type = str - - def _is_file(f): - return isinstance(f, file) - -else: - from itertools import zip_longest as izip_longest - from functools import reduce, partial - _none_type = type(None) - _bool_type = bool - _int_type = int - _long_type = int - _float_type = float - _text_type = str - _binary_type = bytes - basestring = str - - import io - - def _is_file(f): - return isinstance(f, io.IOBase) - -try: - import wcwidth # optional wide-character (CJK) support -except ImportError: - wcwidth = None - - -__all__ = ["tabulate", "tabulate_formats"] -__version__ = "0.8.0" - - -# minimum extra space in headers -MIN_PADDING = 2 - -PRESERVE_WHITESPACE = False - -_DEFAULT_FLOATFMT = "g" -_DEFAULT_MISSINGVAL = "" - - -# if True, enable wide-character (CJK) support -WIDE_CHARS_MODE = wcwidth is not None - - -Line = namedtuple("Line", ["begin", "hline", "sep", "end"]) - - -DataRow = namedtuple("DataRow", ["begin", "sep", "end"]) - - -# A table structure is suppposed to be: -# -# --- lineabove --------- -# headerrow -# --- linebelowheader --- -# datarow -# --- linebewteenrows --- -# ... (more datarows) ... -# --- linebewteenrows --- -# last datarow -# --- linebelow --------- -# -# TableFormat's line* elements can be -# -# - either None, if the element is not used, -# - or a Line tuple, -# - or a function: [col_widths], [col_alignments] -> string. -# -# TableFormat's *row elements can be -# -# - either None, if the element is not used, -# - or a DataRow tuple, -# - or a function: [cell_values], [col_widths], [col_alignments] -> string. -# -# padding (an integer) is the amount of white space around data values. -# -# with_header_hide: -# -# - either None, to display all table elements unconditionally, -# - or a list of elements not to be displayed if the table has column -# headers. -# -TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", - "linebetweenrows", "linebelow", - "headerrow", "datarow", - "padding", "with_header_hide"]) - - -def _pipe_segment_with_colons(align, colwidth): - """Return a segment of a horizontal line with optional colons which - indicate column's alignment (as in `pipe` output format).""" - w = colwidth - if align in ["right", "decimal"]: - return ('-' * (w - 1)) + ":" - elif align == "center": - return ":" + ('-' * (w - 2)) + ":" - elif align == "left": - return ":" + ('-' * (w - 1)) - else: - return '-' * w - - -def _pipe_line_with_colons(colwidths, colaligns): - """Return a horizontal line with optional colons to indicate column's - alignment (as in `pipe` output format).""" - segments = [_pipe_segment_with_colons(a, w) for a, w in - zip(colaligns, colwidths)] - return "|" + "|".join(segments) + "|" - - -def _mediawiki_row_with_attrs(separator, cell_values, colwidths, colaligns): - alignment = {"left": '', - "right": 'align="right"| ', - "center": 'align="center"| ', - "decimal": 'align="right"| '} - # hard-coded padding _around_ align attribute and value together - # rather than padding parameter which affects only the value - values_with_attrs = [' ' + alignment.get(a, '') + c + ' ' - for c, a in zip(cell_values, colaligns)] - colsep = separator*2 - return (separator + colsep.join(values_with_attrs)).rstrip() - - -def _textile_row_with_attrs(cell_values, colwidths, colaligns): - cell_values[0] += ' ' - alignment = {"left": "<.", "right": ">.", "center": "=.", "decimal": ">."} - values = (alignment.get(a, '') + v for a, v in zip(colaligns, cell_values)) - return '|' + '|'.join(values) + '|' - - -def _html_begin_table_without_header(colwidths_ignore, colaligns_ignore): - # this table header will be suppressed if there is a header row - return "\n".join(["", ""]) - - -def _html_row_with_attrs(celltag, cell_values, colwidths, colaligns): - alignment = {"left": '', - "right": ' style="text-align: right;"', - "center": ' style="text-align: center;"', - "decimal": ' style="text-align: right;"'} - values_with_attrs = ["<{0}{1}>{2}".format( - celltag, alignment.get(a, ''), c) for c, a in - zip(cell_values, colaligns)] - rowhtml = "" + "".join(values_with_attrs).rstrip() + "" - if celltag == "th": # it's a header row, create a new table header - rowhtml = "\n".join(["
", - "", - rowhtml, - "", - ""]) - return rowhtml - - -def _moin_row_with_attrs(celltag, cell_values, colwidths, colaligns, - header=''): - alignment = {"left": '', - "right": '', - "center": '', - "decimal": ''} - values_with_attrs = ["{0}{1} {2} ".format(celltag, - alignment.get(a, ''), - header + c + header) - for c, a in zip(cell_values, colaligns)] - return "".join(values_with_attrs) + "||" - - -def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): - alignment = {"left": "l", "right": "r", "center": "c", "decimal": "r"} - tabular_columns_fmt = "".join([alignment.get(a, "l") for a in colaligns]) - return "\n".join(["\\begin{tabular}{" + tabular_columns_fmt + "}", - "\\toprule" if booktabs else "\hline"]) - - -LATEX_ESCAPE_RULES = {r"&": r"\&", r"%": r"\%", r"$": r"\$", r"#": r"\#", - r"_": r"\_", r"^": r"\^{}", r"{": r"\{", r"}": r"\}", - r"~": r"\textasciitilde{}", "\\": r"\textbackslash{}", - r"<": r"\ensuremath{<}", r">": r"\ensuremath{>}"} - - -def _latex_row(cell_values, colwidths, colaligns, escrules=LATEX_ESCAPE_RULES): - def escape_char(c): - return escrules.get(c, c) - escaped_values = ["".join(map(escape_char, cell)) for cell in cell_values] - rowfmt = DataRow("", "&", "\\\\") - return _build_simple_row(escaped_values, rowfmt) - - -def _rst_escape_first_column(rows, headers): - def escape_empty(val): - if isinstance(val, (_text_type, _binary_type)) and val.strip() is "": - return ".." - else: - return val - new_headers = list(headers) - new_rows = [] - if headers: - new_headers[0] = escape_empty(headers[0]) - for row in rows: - new_row = list(row) - if new_row: - new_row[0] = escape_empty(row[0]) - new_rows.append(new_row) - return new_rows, new_headers - - -_table_formats = {"simple": - TableFormat( - lineabove=Line("", "-", " ", ""), - linebelowheader=Line("", "-", " ", ""), - linebetweenrows=None, - linebelow=Line("", "-", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, - with_header_hide=["lineabove", "linebelow"]), - "plain": - TableFormat( - lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None), - "grid": - TableFormat( - lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("+", "=", "+", "+"), - linebetweenrows=Line("+", "-", "+", "+"), - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), - "fancy_grid": - TableFormat( - lineabove=Line("╒", "═", "╤", "╕"), - linebelowheader=Line("╞", "═", "╪", "╡"), - linebetweenrows=Line("├", "─", "┼", "┤"), - linebelow=Line("╘", "═", "╧", "╛"), - headerrow=DataRow("│", "│", "│"), - datarow=DataRow("│", "│", "│"), - padding=1, with_header_hide=None), - "pipe": - TableFormat( - lineabove=_pipe_line_with_colons, - linebelowheader=_pipe_line_with_colons, - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, - with_header_hide=["lineabove"]), - "orgtbl": - TableFormat( - lineabove=None, - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), - "jira": - TableFormat( - lineabove=None, - linebelowheader=None, - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("||", "||", "||"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), - "psql": - TableFormat( - lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), - "rst": - TableFormat( - lineabove=Line("", "=", " ", ""), - linebelowheader=Line("", "=", " ", ""), - linebetweenrows=None, - linebelow=Line("", "=", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None), - "mediawiki": - TableFormat(lineabove=Line( - "{| class=\"wikitable\" style=\"text-align: left;\"", - "", "", "\n|+ \n|-"), - linebelowheader=Line("|-", "", "", ""), - linebetweenrows=Line("|-", "", "", ""), - linebelow=Line("|}", "", "", ""), - headerrow=partial(_mediawiki_row_with_attrs, "!"), - datarow=partial(_mediawiki_row_with_attrs, "|"), - padding=0, with_header_hide=None), - "moinmoin": - TableFormat( - lineabove=None, - linebelowheader=None, - linebetweenrows=None, - linebelow=None, - headerrow=partial(_moin_row_with_attrs, "||", - header="'''"), - datarow=partial(_moin_row_with_attrs, "||"), - padding=1, with_header_hide=None), - "html": - TableFormat( - lineabove=_html_begin_table_without_header, - linebelowheader="", - linebetweenrows=None, - linebelow=Line("\n
", "", "", ""), - headerrow=partial(_html_row_with_attrs, "th"), - datarow=partial(_html_row_with_attrs, "td"), - padding=0, with_header_hide=["lineabove"]), - "latex": - TableFormat( - lineabove=_latex_line_begin_tabular, - linebelowheader=Line("\\hline", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None), - "latex_raw": - TableFormat( - lineabove=_latex_line_begin_tabular, - linebelowheader=Line("\\hline", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), - headerrow=partial(_latex_row, escrules={}), - datarow=partial(_latex_row, escrules={}), - padding=1, with_header_hide=None), - "latex_booktabs": - TableFormat( - lineabove=partial(_latex_line_begin_tabular, - booktabs=True), - linebelowheader=Line("\\midrule", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", - ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None), - "textile": - TableFormat( - lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("|_. ", "|_.", "|"), - datarow=_textile_row_with_attrs, - padding=1, with_header_hide=None)} - - -tabulate_formats = list(sorted(_table_formats.keys())) - - -# ANSI color codes -_invisible_codes = re.compile(r"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") -_invisible_codes_bytes = re.compile(b"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") - - -def _isconvertible(conv, string): - try: - n = conv(string) - return True - except (ValueError, TypeError): - return False - - -def _isnumber(string): - """ - >>> _isnumber("123.45") - True - >>> _isnumber("123") - True - >>> _isnumber("spam") - False - """ - return _isconvertible(float, string) - - -def _isint(string, inttype=int): - """ - >>> _isint("123") - True - >>> _isint("123.45") - False - """ - return type(string) is inttype or\ - (isinstance(string, _binary_type) or isinstance(string, _text_type))\ - and\ - _isconvertible(inttype, string) - - -def _isbool(string): - """ - >>> _isbool(True) - True - >>> _isbool("False") - True - >>> _isbool(1) - False - """ - return type(string) is _bool_type or\ - (isinstance(string, (_binary_type, _text_type)) and - string in ("True", "False")) - - -def _type(string, has_invisible=True, numparse=True): - """The least generic type (type(None), int, float, str, unicode). - - >>> _type(None) is type(None) - True - >>> _type("foo") is type("") - True - >>> _type("1") is type(1) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - - """ - - if has_invisible and \ - (isinstance(string, _text_type) or isinstance(string, _binary_type)): - string = _strip_invisible(string) - - if string is None: - return _none_type - elif hasattr(string, "isoformat"): # datetime.datetime, date, and time - return _text_type - elif _isbool(string): - return _bool_type - elif _isint(string) and numparse: - return int - elif _isint(string, _long_type) and numparse: - return int - elif _isnumber(string) and numparse: - return float - elif isinstance(string, _binary_type): - return _binary_type - else: - return _text_type - - -def _afterpoint(string): - """Symbols after a decimal point, -1 if the string lacks the decimal point. - - >>> _afterpoint("123.45") - 2 - >>> _afterpoint("1001") - -1 - >>> _afterpoint("eggs") - -1 - >>> _afterpoint("123e45") - 2 - - """ - if _isnumber(string): - if _isint(string): - return -1 - else: - pos = string.rfind(".") - pos = string.lower().rfind("e") if pos < 0 else pos - if pos >= 0: - return len(string) - pos - 1 - else: - return -1 # no point - else: - return -1 # not a number - - -def _padleft(width, s): - """Flush right. - - >>> _padleft(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430' - True - - """ - fmt = "{0:>%ds}" % width - return fmt.format(s) - - -def _padright(width, s): - """Flush left. - - >>> _padright(6, '\u044f\u0439\u0446\u0430') == '\u044f\u0439\u0446\u0430 ' - True - - """ - fmt = "{0:<%ds}" % width - return fmt.format(s) - - -def _padboth(width, s): - """Center string. - - >>> _padboth(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430 ' - True - - """ - fmt = "{0:^%ds}" % width - return fmt.format(s) - - -def _strip_invisible(s): - "Remove invisible ANSI color codes." - if isinstance(s, _text_type): - return re.sub(_invisible_codes, "", s) - else: # a bytestring - return re.sub(_invisible_codes_bytes, "", s) - - -def _visible_width(s): - """Visible width of a printed string. ANSI color codes are removed. - - >>> _visible_width('\x1b[31mhello\x1b[0m'), _visible_width("world") - (5, 5) - - """ - # optional wide-character support - if wcwidth is not None and WIDE_CHARS_MODE: - len_fn = wcwidth.wcswidth - else: - len_fn = len - if isinstance(s, _text_type) or isinstance(s, _binary_type): - return len_fn(_strip_invisible(s)) - else: - return len_fn(_text_type(s)) - - -def _align_column(strings, alignment, minwidth=0, has_invisible=True): - """[string] -> [padded_string] - - >>> list(map(str,_align_column( - ... ["12.345", "-1234.5", "1.23", "1234.5", "1e+234", "1.0e234"], - ... "decimal"))) - [' 12.345 ', '-1234.5 ', ' 1.23 ', ' 1234.5 ', ' 1e+234 ', ' 1.0e234'] - - >>> list(map(str,_align_column(['123.4', '56.7890'], None))) - ['123.4', '56.7890'] - - """ - if alignment == "right": - if not PRESERVE_WHITESPACE: - strings = [s.strip() for s in strings] - padfn = _padleft - elif alignment == "center": - if not PRESERVE_WHITESPACE: - strings = [s.strip() for s in strings] - padfn = _padboth - elif alignment == "decimal": - if has_invisible: - decimals = [_afterpoint(_strip_invisible(s)) for s in strings] - else: - decimals = [_afterpoint(s) for s in strings] - maxdecimals = max(decimals) - strings = [s + (maxdecimals - decs) * " " - for s, decs in zip(strings, decimals)] - padfn = _padleft - elif not alignment: - return strings - else: - if not PRESERVE_WHITESPACE: - strings = [s.strip() for s in strings] - padfn = _padright - - enable_widechars = wcwidth is not None and WIDE_CHARS_MODE - if has_invisible: - width_fn = _visible_width - elif enable_widechars: # optional wide-character support if available - width_fn = wcwidth.wcswidth - else: - width_fn = len - - s_lens = list(map(len, strings)) - s_widths = list(map(width_fn, strings)) - maxwidth = max(max(s_widths), minwidth) - if not enable_widechars and not has_invisible: - padded_strings = [padfn(maxwidth, s) for s in strings] - else: - # enable wide-character width corrections - visible_widths = [maxwidth - (w - l) for w, l in zip(s_widths, s_lens)] - # wcswidth and _visible_width don't count invisible characters; - # padfn doesn't need to apply another correction - padded_strings = [padfn(w, s) for s, w in zip(strings, visible_widths)] - return padded_strings - - -def _more_generic(type1, type2): - types = {_none_type: 0, _bool_type: 1, int: 2, float: 3, _binary_type: 4, - _text_type: 5} - invtypes = {5: _text_type, 4: _binary_type, 3: float, 2: int, - 1: _bool_type, 0: _none_type} - moregeneric = max(types.get(type1, 5), types.get(type2, 5)) - return invtypes[moregeneric] - - -def _column_type(strings, has_invisible=True, numparse=True): - """The least generic type all column values are convertible to. - - >>> _column_type([True, False]) is _bool_type - True - >>> _column_type(["1", "2"]) is _int_type - True - >>> _column_type(["1", "2.3"]) is _float_type - True - >>> _column_type(["1", "2.3", "four"]) is _text_type - True - >>> _column_type(["four", '\u043f\u044f\u0442\u044c']) is _text_type - True - >>> _column_type([None, "brux"]) is _text_type - True - >>> _column_type([1, 2, None]) is _int_type - True - >>> import datetime as dt - >>> _column_type([dt.datetime(1991,2,19), dt.time(17,35)]) is _text_type - True - - """ - types = [_type(s, has_invisible, numparse) for s in strings] - return reduce(_more_generic, types, _bool_type) - - -def _format(val, valtype, floatfmt, missingval="", has_invisible=True): - """Format a value accoding to its type. - - Unicode is supported: - - >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', - ... '\u0446\u0438\u0444\u0440\u0430'] - >>> tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] - >>> good_result = ('\\u0431\\u0443\\u043a\\u0432\\u0430 ' - ... '\\u0446\\u0438\\u0444\\u0440\\u0430\\n------- ' - ... '-------\\n\\u0430\\u0437 ' - ... '2\\n\\u0431\\u0443\\u043a\\u0438 4') - >>> tabulate(tbl, headers=hrow) == good_result - True - - """ - if val is None: - return missingval - - if valtype in [int, _text_type]: - return "{0}".format(val) - elif valtype is _binary_type: - try: - return _text_type(val, "ascii") - except TypeError: - return _text_type(val) - elif valtype is float: - is_a_colored_number = (has_invisible and - isinstance(val, (_text_type, _binary_type))) - if is_a_colored_number: - raw_val = _strip_invisible(val) - formatted_val = format(float(raw_val), floatfmt) - return val.replace(raw_val, formatted_val) - else: - return format(float(val), floatfmt) - else: - return "{0}".format(val) - - -def _align_header(header, alignment, width, visible_width): - """Pad string header to width chars given known visible_width of the - header.""" - width += len(header) - visible_width - if alignment == "left": - return _padright(width, header) - elif alignment == "center": - return _padboth(width, header) - elif not alignment: - return "{0}".format(header) - else: - return _padleft(width, header) - - -def _prepend_row_index(rows, index): - """Add a left-most index column.""" - if index is None or index is False: - return rows - if len(index) != len(rows): - print('index=', index) - print('rows=', rows) - raise ValueError('index must be as long as the number of data rows') - rows = [[v] + list(row) for v, row in zip(index, rows)] - return rows - - -def _bool(val): - """A wrapper around standard bool() which doesn't throw on NumPy - arrays.""" - try: - return bool(val) - except ValueError: # val is likely to be a numpy array with many elements - return False - - -def _normalize_tabular_data(tabular_data, headers, showindex="default"): - """Transform a supported data type to a list of lists, and a list of - headers. - - Supported tabular data types: - - * list-of-lists or another iterable of iterables - - * list of named tuples (usually used with headers="keys") - - * list of dicts (usually used with headers="keys") - - * list of OrderedDicts (usually used with headers="keys") - - * 2D NumPy arrays - - * NumPy record arrays (usually used with headers="keys") - - * dict of iterables (usually used with headers="keys") - - * pandas.DataFrame (usually used with headers="keys") - - The first row can be used as headers if headers="firstrow", - column indices can be used as headers if headers="keys". - - If showindex="default", show row indices of the pandas.DataFrame. - If showindex="always", show row indices for all types of data. - If showindex="never", don't show row indices for all types of data. - If showindex is an iterable, show its values as row indices. - - """ - - try: - bool(headers) - is_headers2bool_broken = False - except ValueError: # numpy.ndarray, pandas.core.index.Index, ... - is_headers2bool_broken = True - headers = list(headers) - - index = None - if hasattr(tabular_data, "keys") and hasattr(tabular_data, "values"): - # dict-like and pandas.DataFrame? - if hasattr(tabular_data.values, "__call__"): - # likely a conventional dict - keys = tabular_data.keys() - # columns have to be transposed - rows = list(izip_longest(*tabular_data.values())) - elif hasattr(tabular_data, "index"): - # values is a property, has .index => it's likely a - # pandas.DataFrame (pandas 0.11.0) - keys = list(tabular_data) - if tabular_data.index.name is not None: - if isinstance(tabular_data.index.name, list): - keys[:0] = tabular_data.index.name - else: - keys[:0] = [tabular_data.index.name] - # values matrix doesn't need to be transposed - vals = tabular_data.values - # for DataFrames add an index per default - index = list(tabular_data.index) - rows = [list(row) for row in vals] - else: - raise ValueError( - "tabular data doesn't appear to be a dict or a DataFrame") - - if headers == "keys": - headers = list(map(_text_type, keys)) # headers should be strings - - else: # it's a usual an iterable of iterables, or a NumPy array - rows = list(tabular_data) - - if (headers == "keys" and not rows): - # an empty table (issue #81) - headers = [] - elif (headers == "keys" and - hasattr(tabular_data, "dtype") and - getattr(tabular_data.dtype, "names")): - # numpy record array - headers = tabular_data.dtype.names - elif (headers == "keys" - and len(rows) > 0 - and isinstance(rows[0], tuple) - and hasattr(rows[0], "_fields")): - # namedtuple - headers = list(map(_text_type, rows[0]._fields)) - elif (len(rows) > 0 - and isinstance(rows[0], dict)): - # dict or OrderedDict - uniq_keys = set() # implements hashed lookup - keys = [] # storage for set - if headers == "firstrow": - firstdict = rows[0] if len(rows) > 0 else {} - keys.extend(firstdict.keys()) - uniq_keys.update(keys) - rows = rows[1:] - for row in rows: - for k in row.keys(): - # Save unique items in input order - if k not in uniq_keys: - keys.append(k) - uniq_keys.add(k) - if headers == 'keys': - headers = keys - elif isinstance(headers, dict): - # a dict of headers for a list of dicts - headers = [headers.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - elif headers == "firstrow": - if len(rows) > 0: - headers = [firstdict.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - else: - headers = [] - elif headers: - raise ValueError( - 'headers for a list of dicts is not a dict or a keyword') - rows = [[row.get(k) for k in keys] for row in rows] - - elif (headers == "keys" - and hasattr(tabular_data, "description") - and hasattr(tabular_data, "fetchone") - and hasattr(tabular_data, "rowcount")): - # Python Database API cursor object (PEP 0249) - # print tabulate(cursor, headers='keys') - headers = [column[0] for column in tabular_data.description] - - elif headers == "keys" and len(rows) > 0: - # keys are column indices - headers = list(map(_text_type, range(len(rows[0])))) - - # take headers from the first row if necessary - if headers == "firstrow" and len(rows) > 0: - if index is not None: - headers = [index[0]] + list(rows[0]) - index = index[1:] - else: - headers = rows[0] - headers = list(map(_text_type, headers)) # headers should be strings - rows = rows[1:] - - headers = list(map(_text_type, headers)) - rows = list(map(list, rows)) - - # add or remove an index column - showindex_is_a_str = type(showindex) in [_text_type, _binary_type] - if showindex == "default" and index is not None: - rows = _prepend_row_index(rows, index) - elif isinstance(showindex, Iterable) and not showindex_is_a_str: - rows = _prepend_row_index(rows, list(showindex)) - elif (showindex == "always" or - (_bool(showindex) and not showindex_is_a_str)): - if index is None: - index = list(range(len(rows))) - rows = _prepend_row_index(rows, index) - elif (showindex == "never" or - (not _bool(showindex) and not showindex_is_a_str)): - pass - - # pad with empty headers for initial columns if necessary - if headers and len(rows) > 0: - nhs = len(headers) - ncols = len(rows[0]) - if nhs < ncols: - headers = [""] * (ncols - nhs) + headers - - return rows, headers - - -def tabulate(tabular_data, headers=(), tablefmt="simple", - floatfmt=_DEFAULT_FLOATFMT, numalign="decimal", stralign="left", - missingval=_DEFAULT_MISSINGVAL, showindex="default", - disable_numparse=False): - """Format a fixed width table for pretty printing. - - >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])) - --- --------- - 1 2.34 - -56 8.999 - 2 10001 - --- --------- - - The first required argument (`tabular_data`) can be a - list-of-lists (or another iterable of iterables), a list of named - tuples, a dictionary of iterables, an iterable of dictionaries, - a two-dimensional NumPy array, NumPy record array, or a Pandas' - dataframe. - - - Table headers - ------------- - - To print nice column headers, supply the second argument (`headers`): - - - `headers` can be an explicit list of column headers - - if `headers="firstrow"`, then the first row of data is used - - if `headers="keys"`, then dictionary keys or column indices are used - - Otherwise a headerless table is produced. - - If the number of headers is less than the number of columns, they - are supposed to be names of the last columns. This is consistent - with the plain-text format of R and Pandas' dataframes. - - >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], - ... headers="firstrow")) - sex age - ----- ----- ----- - Alice F 24 - Bob M 19 - - By default, pandas.DataFrame data have an additional column called - row index. To add a similar column to all other types of data, - use `showindex="always"` or `showindex=True`. To suppress row indices - for all types of data, pass `showindex="never" or `showindex=False`. - To add a custom row index column, pass `showindex=some_iterable`. - - >>> print(tabulate([["F",24],["M",19]], showindex="always")) - - - -- - 0 F 24 - 1 M 19 - - - -- - - - Column alignment - ---------------- - - `tabulate` tries to detect column types automatically, and aligns - the values properly. By default it aligns decimal points of the - numbers (or flushes integer numbers to the right), and flushes - everything else to the left. Possible column alignments - (`numalign`, `stralign`) are: "right", "center", "left", "decimal" - (only for `numalign`), and None (to disable alignment). - - - Table formats - ------------- - - `floatfmt` is a format specification used for columns which - contain numeric data with a decimal point. This can also be - a list or tuple of format strings, one per column. - - `None` values are replaced with a `missingval` string (like - `floatfmt`, this can also be a list of values for different - columns): - - >>> print(tabulate([["spam", 1, None], - ... ["eggs", 42, 3.14], - ... ["other", None, 2.7]], missingval="?")) - ----- -- ---- - spam 1 ? - eggs 42 3.14 - other ? 2.7 - ----- -- ---- - - Various plain-text table formats (`tablefmt`) are supported: - 'plain', 'simple', 'grid', 'pipe', 'orgtbl', 'rst', 'mediawiki', - 'latex', 'latex_raw' and 'latex_booktabs'. Variable `tabulate_formats` - contains the list of currently supported formats. - - "plain" format doesn't use any pseudographics to draw tables, - it separates columns with a double space: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "plain")) - strings numbers - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="plain")) - spam 41.9999 - eggs 451 - - "simple" format is like Pandoc simple_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "simple")) - strings numbers - --------- --------- - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="simple")) - ---- -------- - spam 41.9999 - eggs 451 - ---- -------- - - "grid" is similar to tables produced by Emacs table.el package or - Pandoc grid_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "grid")) - +-----------+-----------+ - | strings | numbers | - +===========+===========+ - | spam | 41.9999 | - +-----------+-----------+ - | eggs | 451 | - +-----------+-----------+ - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="grid")) - +------+----------+ - | spam | 41.9999 | - +------+----------+ - | eggs | 451 | - +------+----------+ - - "fancy_grid" draws a grid using box-drawing characters: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "fancy_grid")) - ╒═══════════╤═══════════╕ - │ strings │ numbers │ - ╞═══════════╪═══════════╡ - │ spam │ 41.9999 │ - ├───────────┼───────────┤ - │ eggs │ 451 │ - ╘═══════════╧═══════════╛ - - "pipe" is like tables in PHP Markdown Extra extension or Pandoc - pipe_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "pipe")) - | strings | numbers | - |:----------|----------:| - | spam | 41.9999 | - | eggs | 451 | - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="pipe")) - |:-----|---------:| - | spam | 41.9999 | - | eggs | 451 | - - "orgtbl" is like tables in Emacs org-mode and orgtbl-mode. They - are slightly different from "pipe" format by not using colons to - define column alignment, and using a "+" sign to indicate line - intersections: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "orgtbl")) - | strings | numbers | - |-----------+-----------| - | spam | 41.9999 | - | eggs | 451 | - - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="orgtbl")) - | spam | 41.9999 | - | eggs | 451 | - - "rst" is like a simple table format from reStructuredText; please - note that reStructuredText accepts also "grid" tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "rst")) - ========= ========= - strings numbers - ========= ========= - spam 41.9999 - eggs 451 - ========= ========= - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")) - ==== ======== - spam 41.9999 - eggs 451 - ==== ======== - - "mediawiki" produces a table markup used in Wikipedia and on other - MediaWiki-based sites: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], - ... ["eggs", "451.0"]], headers="firstrow", - ... tablefmt="mediawiki")) - {| class="wikitable" style="text-align: left;" - |+ - |- - ! strings !! align="right"| numbers - |- - | spam || align="right"| 41.9999 - |- - | eggs || align="right"| 451 - |} - - "html" produces HTML markup: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], - ... ["eggs", "451.0"]], headers="firstrow", - ... tablefmt="html")) - - - - - - - - -
strings numbers
spam 41.9999
eggs 451
- - "latex" produces a tabular environment of LaTeX document markup: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="latex")) - \\begin{tabular}{lr} - \\hline - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\hline - \\end{tabular} - - "latex_raw" is similar to "latex", but doesn't escape special characters, - such as backslash and underscore, so LaTeX commands may embedded into - cells' values: - - >>> print(tabulate([["spam$_9$", 41.9999], ["\\\\emph{eggs}", "451.0"]], - ... tablefmt="latex_raw")) - \\begin{tabular}{lr} - \\hline - spam$_9$ & 41.9999 \\\\ - \\emph{eggs} & 451 \\\\ - \\hline - \\end{tabular} - - "latex_booktabs" produces a tabular environment of LaTeX document markup - using the booktabs.sty package: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="latex_booktabs")) - \\begin{tabular}{lr} - \\toprule - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\bottomrule - \end{tabular} - - Number parsing - -------------- - By default, anything which can be parsed as a number is a number. - This ensures numbers represented as strings are aligned properly. - This can lead to weird results for particular strings such as - specific git SHAs e.g. "42992e1" will be parsed into the number - 429920 and aligned as such. - - To completely disable number parsing (and alignment), use - `disable_numparse=True`. For more fine grained control, a list column - indices is used to disable number parsing only on those columns - e.g. `disable_numparse=[0, 2]` would disable number parsing only on the - first and third columns. - - """ - if tabular_data is None: - tabular_data = [] - list_of_lists, headers = _normalize_tabular_data( - tabular_data, headers, showindex=showindex) - - # empty values in the first column of RST tables should be escaped - # (issue #82). "" should be escaped as "\\ " or ".." - if tablefmt == 'rst': - list_of_lists, headers = _rst_escape_first_column(list_of_lists, - headers) - - # optimization: look for ANSI control codes once, - # enable smart width functions only if a control code is found - plain_text = '\n'.join(['\t'.join(map(_text_type, headers))] + - ['\t'.join(map(_text_type, row)) - for row in list_of_lists]) - - has_invisible = re.search(_invisible_codes, plain_text) - enable_widechars = wcwidth is not None and WIDE_CHARS_MODE - if has_invisible: - width_fn = _visible_width - elif enable_widechars: # optional wide-character support if available - width_fn = wcwidth.wcswidth - else: - width_fn = len - - # format rows and columns, convert numeric values to strings - cols = list(izip_longest(*list_of_lists)) - numparses = _expand_numparse(disable_numparse, len(cols)) - coltypes = [_column_type(col, numparse=np) for col, np in - zip(cols, numparses)] - if isinstance(floatfmt, basestring): # old version - # just duplicate the string to use in each column - float_formats = len(cols) * [floatfmt] - else: # if floatfmt is list, tuple etc we have one per column - float_formats = list(floatfmt) - if len(float_formats) < len(cols): - float_formats.extend((len(cols) - len(float_formats)) * - [_DEFAULT_FLOATFMT]) - if isinstance(missingval, basestring): - missing_vals = len(cols) * [missingval] - else: - missing_vals = list(missingval) - if len(missing_vals) < len(cols): - missing_vals.extend((len(cols) - len(missing_vals)) * - [_DEFAULT_MISSINGVAL]) - cols = [[_format(v, ct, fl_fmt, miss_v, has_invisible) for v in c] - for c, ct, fl_fmt, miss_v in zip(cols, coltypes, float_formats, - missing_vals)] - - # align columns - aligns = [numalign if ct in [int, float] else stralign for ct in coltypes] - minwidths = [width_fn(h) + MIN_PADDING - for h in headers] if headers else [0] * len(cols) - cols = [_align_column(c, a, minw, has_invisible) - for c, a, minw in zip(cols, aligns, minwidths)] - - if headers: - # align headers and add headers - t_cols = cols or [['']] * len(headers) - t_aligns = aligns or [stralign] * len(headers) - minwidths = [max(minw, width_fn(c[0])) - for minw, c in zip(minwidths, t_cols)] - headers = [_align_header(h, a, minw, width_fn(h)) - for h, a, minw in zip(headers, t_aligns, minwidths)] - rows = list(zip(*cols)) - else: - minwidths = [width_fn(c[0]) for c in cols] - rows = list(zip(*cols)) - - if not isinstance(tablefmt, TableFormat): - tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) - - return _format_table(tablefmt, headers, rows, minwidths, aligns) - - -def _expand_numparse(disable_numparse, column_count): - """Return a list of bools of length `column_count` which indicates whether - number parsing should be used on each column. - - If `disable_numparse` is a list of indices, each of those indices - are False, and everything else is True. If `disable_numparse` is a - bool, then the returned list is all the same. - - """ - if isinstance(disable_numparse, Iterable): - numparses = [True] * column_count - for index in disable_numparse: - numparses[index] = False - return numparses - else: - return [not disable_numparse] * column_count - - -def _build_simple_row(padded_cells, rowfmt): - "Format row according to DataRow format without padding." - begin, sep, end = rowfmt - return (begin + sep.join(padded_cells) + end).rstrip() - - -def _build_row(padded_cells, colwidths, colaligns, rowfmt): - "Return a string which represents a row of data cells." - if not rowfmt: - return None - if hasattr(rowfmt, "__call__"): - return rowfmt(padded_cells, colwidths, colaligns) - else: - return _build_simple_row(padded_cells, rowfmt) - - -def _build_line(colwidths, colaligns, linefmt): - "Return a string which represents a horizontal line." - if not linefmt: - return None - if hasattr(linefmt, "__call__"): - return linefmt(colwidths, colaligns) - else: - begin, fill, sep, end = linefmt - cells = [fill*w for w in colwidths] - return _build_simple_row(cells, (begin, sep, end)) - - -def _pad_row(cells, padding): - if cells: - pad = " "*padding - padded_cells = [pad + cell + pad for cell in cells] - return padded_cells - else: - return cells - - -def _format_table(fmt, headers, rows, colwidths, colaligns): - """Produce a plain-text representation of the table.""" - lines = [] - hidden = fmt.with_header_hide if (headers and fmt.with_header_hide) else [] - pad = fmt.padding - headerrow = fmt.headerrow - - padded_widths = [(w + 2*pad) for w in colwidths] - padded_headers = _pad_row(headers, pad) - padded_rows = [_pad_row(row, pad) for row in rows] - - if fmt.lineabove and "lineabove" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.lineabove)) - - if padded_headers: - lines.append(_build_row(padded_headers, padded_widths, colaligns, - headerrow)) - if fmt.linebelowheader and "linebelowheader" not in hidden: - lines.append(_build_line(padded_widths, colaligns, - fmt.linebelowheader)) - - if padded_rows and fmt.linebetweenrows and "linebetweenrows" not in hidden: - # initial rows with a line below - for row in padded_rows[:-1]: - lines.append(_build_row(row, padded_widths, colaligns, - fmt.datarow)) - lines.append(_build_line(padded_widths, colaligns, - fmt.linebetweenrows)) - # the last row without a line below - lines.append(_build_row(padded_rows[-1], padded_widths, colaligns, - fmt.datarow)) - else: - for row in padded_rows: - lines.append(_build_row(row, padded_widths, colaligns, - fmt.datarow)) - - if fmt.linebelow and "linebelow" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.linebelow)) - - if headers or rows: - return "\n".join(lines) - else: # a completely empty table - return "" - - -def _main(): - """\ Usage: tabulate [options] [FILE ...] - - Pretty-print tabular data. - See also https://bitbucket.org/astanin/python-tabulate - - FILE a filename of the file with tabular data; - if "-" or missing, read data from stdin. - - Options: - - -h, --help show this message - -1, --header use the first row of data as a table header - -o FILE, --output FILE print table to FILE (default: stdout) - -s REGEXP, --sep REGEXP use a custom column separator (default: whitespace) - -F FPFMT, --float FPFMT floating point number format (default: g) - -f FMT, --format FMT set output table format; supported formats: - plain, simple, grid, fancy_grid, pipe, orgtbl, - rst, mediawiki, html, latex, latex_raw, - latex_booktabs, tsv - (default: simple) - - """ - import getopt - import sys - import textwrap - usage = textwrap.dedent(_main.__doc__) - try: - opts, args = getopt.getopt( - sys.argv[1:], "h1o:s:F:f:", - ["help", "header", "output", "sep=", "float=", "format="]) - except getopt.GetoptError as e: - print(e) - print(usage) - sys.exit(2) - headers = [] - floatfmt = _DEFAULT_FLOATFMT - tablefmt = "simple" - sep = r"\s+" - outfile = "-" - for opt, value in opts: - if opt in ["-1", "--header"]: - headers = "firstrow" - elif opt in ["-o", "--output"]: - outfile = value - elif opt in ["-F", "--float"]: - floatfmt = value - elif opt in ["-f", "--format"]: - if value not in tabulate_formats: - print("%s is not a supported table format" % value) - print(usage) - sys.exit(3) - tablefmt = value - elif opt in ["-s", "--sep"]: - sep = value - elif opt in ["-h", "--help"]: - print(usage) - sys.exit(0) - files = [sys.stdin] if not args else args - with (sys.stdout if outfile == "-" else open(outfile, "w")) as out: - for f in files: - if f == "-": - f = sys.stdin - if _is_file(f): - _pprint_file(f, headers=headers, tablefmt=tablefmt, - sep=sep, floatfmt=floatfmt, file=out) - else: - with open(f) as fobj: - _pprint_file(fobj, headers=headers, tablefmt=tablefmt, - sep=sep, floatfmt=floatfmt, file=out) - - -def _pprint_file(fobject, headers, tablefmt, sep, floatfmt, file): - rows = fobject.readlines() - table = [re.split(sep, r.rstrip()) for r in rows if r.strip()] - print(tabulate(table, headers, tablefmt, floatfmt=floatfmt), file=file) - - -if __name__ == "__main__": - _main() diff --git a/test/test_expanded.py b/test/test_expanded.py deleted file mode 100644 index 7233e91ce..000000000 --- a/test/test_expanded.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Test the vertical, expanded table formatter.""" -from textwrap import dedent - -from mycli.output_formatter.expanded import expanded_table -from mycli.encodingutils import text_type - - -def test_expanded_table_renders(): - results = [('hello', text_type(123)), ('world', text_type(456))] - - expected = dedent("""\ - ***************************[ 1. row ]*************************** - name | hello - age | 123 - ***************************[ 2. row ]*************************** - name | world - age | 456 - """) - assert expected == expanded_table(results, ('name', 'age')) diff --git a/test/test_output_formatter.py b/test/test_output_formatter.py deleted file mode 100644 index 9844c1919..000000000 --- a/test/test_output_formatter.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -"""Test the generic output formatter interface.""" - -from __future__ import unicode_literals -from decimal import Decimal -from textwrap import dedent - -from mycli.output_formatter.preprocessors import (align_decimals, - bytes_to_string, - convert_to_string, - quote_whitespaces, - override_missing_value, - to_string) -from mycli.output_formatter.output_formatter import OutputFormatter -from mycli.output_formatter import delimited_output_adapter -from mycli.output_formatter import tabulate_adapter -from mycli.output_formatter import terminaltables_adapter - - -def test_to_string(): - """Test the *output_formatter.to_string()* function.""" - assert 'a' == to_string('a') - assert 'a' == to_string(b'a') - assert '1' == to_string(1) - assert '1.23' == to_string(1.23) - - -def test_convert_to_string(): - """Test the *output_formatter.convert_to_string()* function.""" - data = [[1, 'John'], [2, 'Jill']] - headers = [0, 'name'] - expected = ([['1', 'John'], ['2', 'Jill']], ['0', 'name']) - - assert expected == convert_to_string(data, headers) - - -def test_override_missing_values(): - """Test the *output_formatter.override_missing_values()* function.""" - data = [[1, None], [2, 'Jill']] - headers = [0, 'name'] - expected = ([[1, ''], [2, 'Jill']], [0, 'name']) - - assert expected == override_missing_value(data, headers, - missing_value='') - - -def test_bytes_to_string(): - """Test the *output_formatter.bytes_to_string()* function.""" - data = [[1, 'John'], [2, b'Jill']] - headers = [0, 'name'] - expected = ([[1, 'John'], [2, 'Jill']], [0, 'name']) - - assert expected == bytes_to_string(data, headers) - - -def test_align_decimals(): - """Test the *align_decimals()* function.""" - data = [[Decimal('200'), Decimal('1')], [ - Decimal('1.00002'), Decimal('1.0')]] - headers = ['num1', 'num2'] - expected = ([['200', '1'], [' 1.00002', '1.0']], ['num1', 'num2']) - - assert expected == align_decimals(data, headers) - - -def test_align_decimals_empty_result(): - """Test *align_decimals()* with no results.""" - data = [] - headers = ['num1', 'num2'] - expected = ([], ['num1', 'num2']) - - assert expected == align_decimals(data, headers) - - -def test_quote_whitespaces(): - """Test the *quote_whitespaces()* function.""" - data = [[" before", "after "], [" both ", "none"]] - headers = ['h1', 'h2'] - expected = ([["' before'", "'after '"], ["' both '", "'none'"]], - ['h1', 'h2']) - - assert expected == quote_whitespaces(data, headers) - - -def test_quote_whitespaces_empty_result(): - """Test the *quote_whitespaces()* function with no results.""" - data = [] - headers = ['h1', 'h2'] - expected = ([], ['h1', 'h2']) - - assert expected == quote_whitespaces(data, headers) - - -def test_tabulate_wrapper(): - """Test the *output_formatter.tabulate_wrapper()* function.""" - data = [['abc', 1], ['d', 456]] - headers = ['letters', 'number'] - output = tabulate_adapter.adapter(data, headers, table_format='psql') - assert output == dedent('''\ - +-----------+----------+ - | letters | number | - |-----------+----------| - | abc | 1 | - | d | 456 | - +-----------+----------+''') - - -def test_csv_wrapper(): - """Test the *output_formatter.csv_wrapper()* function.""" - # Test comma-delimited output. - data = [['abc', 1], ['d', 456]] - headers = ['letters', 'number'] - output = delimited_output_adapter.adapter(data, headers) - assert output == dedent('''\ - letters,number\r\n\ - abc,1\r\n\ - d,456\r\n''') - - # Test tab-delimited output. - data = [['abc', 1], ['d', 456]] - headers = ['letters', 'number'] - output = delimited_output_adapter.adapter( - data, headers, table_format='tsv') - assert output == dedent('''\ - letters\tnumber\r\n\ - abc\t1\r\n\ - d\t456\r\n''') - - -def test_terminal_tables_wrapper(): - """Test the *output_formatter.terminal_tables_wrapper()* function.""" - data = [['abc', 1], ['d', 456]] - headers = ['letters', 'number'] - output = terminaltables_adapter.adapter( - data, headers, table_format='ascii') - assert output == dedent('''\ - +---------+--------+ - | letters | number | - +---------+--------+ - | abc | 1 | - | d | 456 | - +---------+--------+''') - - -def test_output_formatter(): - """Test the *output_formatter.OutputFormatter* class.""" - data = [['abc', Decimal(1)], ['defg', Decimal('11.1')], - ['hi', Decimal('1.1')]] - headers = ['text', 'numeric'] - expected = dedent('''\ - +------+---------+ - | text | numeric | - +------+---------+ - | abc | 1 | - | defg | 11.1 | - | hi | 1.1 | - +------+---------+''') - - assert expected == OutputFormatter().format_output(data, headers, - format_name='ascii') diff --git a/test/test_tabulate.py b/test/test_tabulate.py deleted file mode 100644 index ae7c25ce1..000000000 --- a/test/test_tabulate.py +++ /dev/null @@ -1,17 +0,0 @@ -from textwrap import dedent - -from mycli.packages import tabulate - -tabulate.PRESERVE_WHITESPACE = True - - -def test_dont_strip_leading_whitespace(): - data = [[' abc']] - headers = ['xyz'] - tbl = tabulate.tabulate(data, headers, tablefmt='psql') - assert tbl == dedent(''' - +---------+ - | xyz | - |---------| - | abc | - +---------+ ''').strip() From 02ae6b9d8d7e9c18bc87e0087b3c01e753575c15 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:34:41 -0500 Subject: [PATCH 225/824] Add cli_helpers dependency to changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 5001405c9..1e00f27e9 100644 --- a/changelog.md +++ b/changelog.md @@ -22,6 +22,7 @@ Internal Changes: * Behave test source command (Thanks: [Dick Marinus]). * Test using behave the tee command (Thanks: [Dick Marinus]). * Behave fix clean up. (Thanks: [Dick Marinus]). +* Remove output formatter code in favor of CLI Helpers dependency (Thanks: [Thomas Roten]). 1.10.0: ======= From 317eae173e07af337a004a9ec4f4fc73a6e8539c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:44:16 -0500 Subject: [PATCH 226/824] Remove unused function. --- mycli/encodingutils.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index 1a8b5bbb1..078e67726 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import binascii import sys PY2 = sys.version_info[0] == 2 @@ -37,22 +36,3 @@ def utf8tounicode(arg): if PY2 and isinstance(arg, binary_type): return arg.decode('utf-8') return arg - - -def bytes_to_string(b): - """Convert bytes to a string. Hexlify bytes that can't be decoded. - - >>> print(bytes_to_string(b"\\xff")) - 0xff - >>> print(bytes_to_string('abc')) - abc - >>> print(bytes_to_string('✌')) - ✌ - - """ - if isinstance(b, binary_type): - try: - return b.decode('utf8') - except UnicodeDecodeError: - return '0x' + binascii.hexlify(b).decode('ascii') - return b From a6fb51809371ecb633591068fa5bd35efb9bfb71 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 21:33:36 +0200 Subject: [PATCH 227/824] Added a regression test to test_completion_engine.py for sqlparse >= 0.2.3 --- changelog.md | 1 + test/test_completion_engine.py | 1 + 2 files changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index 18c332713..8acc448b1 100644 --- a/changelog.md +++ b/changelog.md @@ -11,6 +11,7 @@ Internal Changes: * Behave test source command (Thanks: [Dick Marinus]). * Test using behave the tee command (Thanks: [Dick Marinus]). * Behave fix clean up. (Thanks: [Dick Marinus]). +* Added a regression test for sqlparse >= 0.2.3 (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/test_completion_engine.py b/test/test_completion_engine.py index 4f0406b08..42f4e2e02 100644 --- a/test/test_completion_engine.py +++ b/test/test_completion_engine.py @@ -234,6 +234,7 @@ def test_dot_col_comma_suggests_cols_or_schema_qualified_table(): 'SELECT * FROM (', 'SELECT * FROM foo WHERE EXISTS (', 'SELECT * FROM foo WHERE bar AND NOT EXISTS (', + 'SELECT 1 AS', ]) def test_sub_select_suggests_keyword(expression): suggestion = suggest_type(expression, expression) From 02629c910f6af7e7bc02660287ae8d831fe0d22c Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 21:30:51 +0200 Subject: [PATCH 228/824] Revert "remove temporary hack" This reverts commit 21c600256de4c01fb4c4aa97b84ebd28f3f81c9f. --- changelog.md | 1 + mycli/packages/completion_engine.py | 44 ++++++++++++++++------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/changelog.md b/changelog.md index 8acc448b1..feedcf4da 100644 --- a/changelog.md +++ b/changelog.md @@ -12,6 +12,7 @@ Internal Changes: * Test using behave the tee command (Thanks: [Dick Marinus]). * Behave fix clean up. (Thanks: [Dick Marinus]). * Added a regression test for sqlparse >= 0.2.3 (Thanks: [Dick Marinus]). +* Reverted removal of temporary hack for sqlparse (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index b97cadf71..efbd41f61 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -28,28 +28,32 @@ def suggest_type(full_text, text_before_cursor): identifier = None - # If we've partially typed a word then word_before_cursor won't be an empty - # string. In that case we want to remove the partially typed string before - # sending it to the sqlparser. Otherwise the last token will always be the - # partially typed string which renders the smart completion useless because - # it will always return the list of keywords as completion. - if word_before_cursor: - if word_before_cursor.endswith( - '(') or word_before_cursor.startswith('\\'): - parsed = sqlparse.parse(text_before_cursor) - else: - parsed = sqlparse.parse( - text_before_cursor[:-len(word_before_cursor)]) + # here should be removed once sqlparse has been fixed + try: + # If we've partially typed a word then word_before_cursor won't be an empty + # string. In that case we want to remove the partially typed string before + # sending it to the sqlparser. Otherwise the last token will always be the + # partially typed string which renders the smart completion useless because + # it will always return the list of keywords as completion. + if word_before_cursor: + if word_before_cursor.endswith( + '(') or word_before_cursor.startswith('\\'): + parsed = sqlparse.parse(text_before_cursor) + else: + parsed = sqlparse.parse( + text_before_cursor[:-len(word_before_cursor)]) - # word_before_cursor may include a schema qualification, like - # "schema_name.partial_name" or "schema_name.", so parse it - # separately - p = sqlparse.parse(word_before_cursor)[0] + # word_before_cursor may include a schema qualification, like + # "schema_name.partial_name" or "schema_name.", so parse it + # separately + p = sqlparse.parse(word_before_cursor)[0] - if p.tokens and isinstance(p.tokens[0], Identifier): - identifier = p.tokens[0] - else: - parsed = sqlparse.parse(text_before_cursor) + if p.tokens and isinstance(p.tokens[0], Identifier): + identifier = p.tokens[0] + else: + parsed = sqlparse.parse(text_before_cursor) + except (TypeError, AttributeError): + return [] if len(parsed) > 1: # Multiple statements being edited -- isolate the current one by From 21cd71d0f4cf4dfb21882aa8bfca349a40809779 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 21:32:30 +0200 Subject: [PATCH 229/824] make test/test_completion_engine.py happy --- mycli/packages/completion_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index efbd41f61..88fabfa13 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -53,7 +53,7 @@ def suggest_type(full_text, text_before_cursor): else: parsed = sqlparse.parse(text_before_cursor) except (TypeError, AttributeError): - return [] + return [{'type': 'keyword'}] if len(parsed) > 1: # Multiple statements being edited -- isolate the current one by From 4bd32370a44713dcd266f25974d0152f4eb0619e Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 2 May 2017 09:08:04 +0200 Subject: [PATCH 230/824] move boiler plate code to before_scenario --- conftest.py | 7 ++++++ test/features/__init__.py | 0 test/features/basic_commands.feature | 16 +++----------- test/features/crud_database.feature | 8 ++----- test/features/crud_table.feature | 4 +--- test/features/environment.py | 7 ++++++ test/features/iocommands.feature | 8 ++----- test/features/named_queries.feature | 4 +--- test/features/specials.feature | 4 +--- test/features/steps/__init__.py | 0 test/features/steps/basic_commands.py | 31 +++----------------------- test/features/steps/wrappers.py | 32 +++++++++++++++++++++++++++ 12 files changed, 59 insertions(+), 62 deletions(-) create mode 100644 test/features/__init__.py create mode 100644 test/features/steps/__init__.py diff --git a/conftest.py b/conftest.py index d2cd1336c..41e72adae 100644 --- a/conftest.py +++ b/conftest.py @@ -3,4 +3,11 @@ "setup.py", "mycli/magic.py", "mycli/packages/parseutils.py", + "test/features/environment.py", + "test/features/steps/basic_commands.py", + "test/features/steps/crud_database.py", + "test/features/steps/crud_table.py", + "test/features/steps/iocommands.py", + "test/features/steps/named_queries.py", + "test/features/steps/specials.py", ] diff --git a/test/features/__init__.py b/test/features/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/features/basic_commands.feature b/test/features/basic_commands.feature index 025b58502..249a68e6e 100644 --- a/test/features/basic_commands.feature +++ b/test/features/basic_commands.feature @@ -2,24 +2,14 @@ Feature: run the cli, call the help command, exit the cli - Scenario: run the cli - When we run dbcli - then we see dbcli prompt - Scenario: run "\?" command - When we run dbcli - and we wait for prompt - and we send "\?" command + When we send "\?" command then we see help output Scenario: run source command - When we run dbcli - and we wait for prompt - and we send source command + When we send source command then we see help output Scenario: run the cli and exit - When we run dbcli - and we wait for prompt - and we send "ctrl + d" + When we send "ctrl + d" then dbcli exits diff --git a/test/features/crud_database.feature b/test/features/crud_database.feature index c72468c30..32d1e38d4 100644 --- a/test/features/crud_database.feature +++ b/test/features/crud_database.feature @@ -2,9 +2,7 @@ Feature: manipulate databases: create, drop, connect, disconnect Scenario: create and drop temporary database - When we run dbcli - and we wait for prompt - and we create database + When we create database then we see database created when we drop database then we see database dropped @@ -12,9 +10,7 @@ Feature: manipulate databases: then we see database connected Scenario: connect and disconnect from test database - When we run dbcli - and we wait for prompt - and we connect to test database + When we connect to test database then we see database connected when we connect to dbserver then we see database connected diff --git a/test/features/crud_table.feature b/test/features/crud_table.feature index d2209fd0e..bcef9292e 100644 --- a/test/features/crud_table.feature +++ b/test/features/crud_table.feature @@ -2,9 +2,7 @@ Feature: manipulate tables: create, insert, update, select, delete from, drop Scenario: create, insert, select from, update, drop table - When we run dbcli - and we wait for prompt - and we connect to test database + When we connect to test database then we see database connected when we create table then we see table created diff --git a/test/features/environment.py b/test/features/environment.py index a0456b99f..138f88f15 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -8,6 +8,8 @@ import fixture_utils as fixutils import pexpect +from steps.wrappers import run_cli, wait_prompt + def before_all(context): """Set env parameters.""" @@ -73,6 +75,11 @@ def before_step(context, _): context.atprompt = False +def before_scenario(context, _): + run_cli(context) + wait_prompt(context) + + def after_scenario(context, _): """Cleans up after each test complete.""" diff --git a/test/features/iocommands.feature b/test/features/iocommands.feature index 4bcdf6e65..38efbbb0a 100644 --- a/test/features/iocommands.feature +++ b/test/features/iocommands.feature @@ -1,18 +1,14 @@ Feature: I/O commands Scenario: edit sql in file with external editor - When we run dbcli - and we wait for prompt - and we start external editor providing a file name + When we start external editor providing a file name and we type sql in the editor and we exit the editor then we see dbcli prompt and we see the sql in prompt Scenario: tee output from query - When we run dbcli - and we wait for prompt - and we tee output + When we tee output and we wait for prompt and we query "select 123456" and we wait for prompt diff --git a/test/features/named_queries.feature b/test/features/named_queries.feature index 79f31ac3a..74201b92a 100644 --- a/test/features/named_queries.feature +++ b/test/features/named_queries.feature @@ -2,9 +2,7 @@ Feature: named queries: save, use and delete named queries Scenario: save, use and delete named queries - When we run dbcli - and we wait for prompt - and we connect to test database + When we connect to test database then we see database connected when we save a named query then we see the named query saved diff --git a/test/features/specials.feature b/test/features/specials.feature index 9bacec45a..bb3675784 100644 --- a/test/features/specials.feature +++ b/test/features/specials.feature @@ -2,8 +2,6 @@ Feature: Special commands @wip Scenario: run refresh command - When we run dbcli - and we wait for prompt - and we refresh completions + When we refresh completions and we wait for prompt then we see completions refresh started diff --git a/test/features/steps/__init__.py b/test/features/steps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index df97ee0e7..299893def 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -7,44 +7,19 @@ """ from __future__ import unicode_literals -import pexpect -import tempfile - from behave import when +import tempfile import wrappers @when('we run dbcli') def step_run_cli(context): - """Run the process using pexpect.""" - run_args = [] - if context.conf.get('host', None): - run_args.extend(('-h', context.conf['host'])) - if context.conf.get('user', None): - run_args.extend(('-u', context.conf['user'])) - if context.conf.get('pass', None): - run_args.extend(('-p', context.conf['pass'])) - if context.conf.get('dbname', None): - run_args.extend(('-D', context.conf['dbname'])) - cli_cmd = context.conf.get('cli_command', None) or sys.executable + \ - ' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' - - cmd_parts = [cli_cmd] + run_args - cmd = ' '.join(cmd_parts) - context.cli = pexpect.spawnu(cmd, cwd='..') - context.exit_sent = False - context.currentdb = context.conf['dbname'] + wrappers.run_cli(context) @when('we wait for prompt') def step_wait_prompt(context): - """Make sure prompt is displayed.""" - user = context.conf['user'] - host = context.conf['host'] - dbname = context.currentdb - wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( - user, host, dbname), timeout=5) - context.atprompt = True + wrappers.wait_prompt(context) @when('we send "ctrl + d"') diff --git a/test/features/steps/wrappers.py b/test/features/steps/wrappers.py index e8d9204ab..3855ede07 100644 --- a/test/features/steps/wrappers.py +++ b/test/features/steps/wrappers.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import re +import pexpect def expect_exact(context, expected, timeout): @@ -19,3 +20,34 @@ def expect_exact(context, expected, timeout): def expect_pager(context, expected, timeout): expect_exact(context, "{0}\r\n{1}{0}\r\n".format( context.conf['pager_boundary'], expected), timeout=timeout) + + +def run_cli(context): + """Run the process using pexpect.""" + run_args = [] + if context.conf.get('host', None): + run_args.extend(('-h', context.conf['host'])) + if context.conf.get('user', None): + run_args.extend(('-u', context.conf['user'])) + if context.conf.get('pass', None): + run_args.extend(('-p', context.conf['pass'])) + if context.conf.get('dbname', None): + run_args.extend(('-D', context.conf['dbname'])) + cli_cmd = context.conf.get('cli_command', None) or sys.executable + \ + ' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' + + cmd_parts = [cli_cmd] + run_args + cmd = ' '.join(cmd_parts) + context.cli = pexpect.spawnu(cmd, cwd='..') + context.exit_sent = False + context.currentdb = context.conf['dbname'] + + +def wait_prompt(context): + """Make sure prompt is displayed.""" + user = context.conf['user'] + host = context.conf['host'] + dbname = context.currentdb + expect_exact(context, 'mysql {0}@{1}:{2}> '.format( + user, host, dbname), timeout=5) + context.atprompt = True From 1a24358e1c4cab3101f4353b4ca2792345e89036 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 2 May 2017 23:03:29 -0500 Subject: [PATCH 231/824] Edit last command in external editor. --- mycli/main.py | 9 +++++++-- mycli/packages/special/iocommands.py | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 5df93408a..7d81f5c56 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -419,8 +419,9 @@ def handle_editor_command(self, cli, document): saved_callables = cli.application.pre_run_callables while special.editor_command(document.text): filename = special.get_filename(document.text) - sql, message = special.open_external_editor(filename, - sql=document.text) + sql, message = special.open_external_editor( + filename, sql=document.text, + default_text=self.get_last_query() or '') if message: # Something went wrong. Raise an exception and bail. raise RuntimeError(message) @@ -756,6 +757,10 @@ def get_reserved_space(self): _, height = click.get_terminal_size() return min(round(height * reserved_space_ratio), max_reserved_space) + def get_last_query(self): + """Get the last query executed or None.""" + return self.query_history[-1][0] if self.query_history else None + @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 4d4fcc655..2cdc86ca6 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -102,7 +102,7 @@ def get_filename(sql): return filename.strip() or None @export -def open_external_editor(filename=None, sql=''): +def open_external_editor(filename=None, sql='', default_text=''): """ Open external editor, wait for the user to type in his query, return the query. @@ -118,6 +118,8 @@ def open_external_editor(filename=None, sql=''): while pattern.search(sql): sql = pattern.sub('', sql) + text = sql if sql else default_text + message = None filename = filename.strip().split(' ', 1)[0] if filename else None @@ -125,7 +127,7 @@ def open_external_editor(filename=None, sql=''): # Populate the editor buffer with the partial sql (if available) and a # placeholder comment. - query = click.edit(sql + '\n\n' + MARKER, filename=filename, + query = click.edit(text + '\n\n' + MARKER, filename=filename, extension='.sql') if filename: From 52881034f8f77a1ae7f33be840698c9dcbc59cef Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 2 May 2017 23:04:47 -0500 Subject: [PATCH 232/824] Simplify conditional logic. --- mycli/packages/special/iocommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 2cdc86ca6..5398d0c69 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -118,7 +118,7 @@ def open_external_editor(filename=None, sql='', default_text=''): while pattern.search(sql): sql = pattern.sub('', sql) - text = sql if sql else default_text + text = sql or default_text message = None filename = filename.strip().split(' ', 1)[0] if filename else None From eac5dbc70ada397dd1fbf8f66584725a5e811524 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 2 May 2017 23:10:50 -0500 Subject: [PATCH 233/824] Add external editor change to changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 5001405c9..51a8748c6 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,7 @@ Features: * Handle reserved space for completion menu better in small windows. (Thanks: [Thomas Roten]). * Display current vi mode in toolbar. (Thanks: [Thomas Roten]). +* Opening an external editor will edit the last-run query. (Thanks: [Thomas Roten]). Bug Fixes: ---------- From b4a8ea41a956cd7735de1e63b88d0758758b2fb1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:41:00 -0500 Subject: [PATCH 234/824] Simplify editor command logic. --- mycli/main.py | 7 ++++--- mycli/packages/special/iocommands.py | 21 ++++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 7d81f5c56..9676161d6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -419,9 +419,10 @@ def handle_editor_command(self, cli, document): saved_callables = cli.application.pre_run_callables while special.editor_command(document.text): filename = special.get_filename(document.text) - sql, message = special.open_external_editor( - filename, sql=document.text, - default_text=self.get_last_query() or '') + query = (special.get_editor_query(document.text) or + self.get_last_query() or '') + sql, message = special.open_external_editor(filename, + sql=query) if message: # Something went wrong. Raise an exception and bail. raise RuntimeError(message) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 5398d0c69..abcdfa3c6 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -102,13 +102,8 @@ def get_filename(sql): return filename.strip() or None @export -def open_external_editor(filename=None, sql='', default_text=''): - """ - Open external editor, wait for the user to type in his query, - return the query. - :return: list with one tuple, query as first element. - """ - +def get_editor_query(sql): + """Get the query part of an editor command.""" sql = sql.strip() # The reason we can't simply do .strip('\e') is that it strips characters, @@ -118,7 +113,15 @@ def open_external_editor(filename=None, sql='', default_text=''): while pattern.search(sql): sql = pattern.sub('', sql) - text = sql or default_text + return sql + +@export +def open_external_editor(filename=None, sql='', default_text=''): + """ + Open external editor, wait for the user to type in his query, + return the query. + :return: list with one tuple, query as first element. + """ message = None filename = filename.strip().split(' ', 1)[0] if filename else None @@ -127,7 +130,7 @@ def open_external_editor(filename=None, sql='', default_text=''): # Populate the editor buffer with the partial sql (if available) and a # placeholder comment. - query = click.edit(text + '\n\n' + MARKER, filename=filename, + query = click.edit(sql + '\n\n' + MARKER, filename=filename, extension='.sql') if filename: From 541929ff7ed2cb8c8c2903694f547c20bd2130dc Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:43:24 -0500 Subject: [PATCH 235/824] Remove unused argument. --- mycli/packages/special/iocommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index abcdfa3c6..8472792b5 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -116,7 +116,7 @@ def get_editor_query(sql): return sql @export -def open_external_editor(filename=None, sql='', default_text=''): +def open_external_editor(filename=None, sql=''): """ Open external editor, wait for the user to type in his query, return the query. From b5cd5f31f4cac078e2050cf8524b58393a57a1a4 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:44:19 -0500 Subject: [PATCH 236/824] Pep8 fix. --- mycli/packages/special/iocommands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 8472792b5..0e09a9ecc 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -117,9 +117,9 @@ def get_editor_query(sql): @export def open_external_editor(filename=None, sql=''): - """ - Open external editor, wait for the user to type in his query, - return the query. + """Open external editor, wait for the user to type in their query, return + the query. + :return: list with one tuple, query as first element. """ From 466d0f3ea01d7f83e9c5470e9b61221d4a2c7165 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:48:40 -0500 Subject: [PATCH 237/824] Simplify editing logic. --- mycli/main.py | 2 +- mycli/packages/special/iocommands.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 9676161d6..af535ca30 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -420,7 +420,7 @@ def handle_editor_command(self, cli, document): while special.editor_command(document.text): filename = special.get_filename(document.text) query = (special.get_editor_query(document.text) or - self.get_last_query() or '') + self.get_last_query()) sql, message = special.open_external_editor(filename, sql=query) if message: diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 0e09a9ecc..29a9f99e5 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -116,7 +116,7 @@ def get_editor_query(sql): return sql @export -def open_external_editor(filename=None, sql=''): +def open_external_editor(filename=None, sql=None): """Open external editor, wait for the user to type in their query, return the query. @@ -126,12 +126,13 @@ def open_external_editor(filename=None, sql=''): message = None filename = filename.strip().split(' ', 1)[0] if filename else None + sql = sql or '' MARKER = '# Type your query above this line.\n' # Populate the editor buffer with the partial sql (if available) and a # placeholder comment. - query = click.edit(sql + '\n\n' + MARKER, filename=filename, - extension='.sql') + query = click.edit("{sql}\n\n{marker}".format(sql=sql, marker=MARKER), + filename=filename, extension='.sql') if filename: try: From b68c3b2b21730cf69da24f1cd8ad9a8886b5ea56 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:49:36 -0500 Subject: [PATCH 238/824] Single quotes instead of double. --- mycli/packages/special/iocommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 29a9f99e5..87d682737 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -131,7 +131,7 @@ def open_external_editor(filename=None, sql=None): # Populate the editor buffer with the partial sql (if available) and a # placeholder comment. - query = click.edit("{sql}\n\n{marker}".format(sql=sql, marker=MARKER), + query = click.edit('{sql}\n\n{marker}'.format(sql=sql, marker=MARKER), filename=filename, extension='.sql') if filename: From f34c042075dd1df10c32a69d7e5460f0409e2e3a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:50:15 -0500 Subject: [PATCH 239/824] Extra newlines for PEP 8. --- mycli/packages/special/iocommands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 87d682737..19c2dc735 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -101,6 +101,7 @@ def get_filename(sql): command, _, filename = sql.partition(' ') return filename.strip() or None + @export def get_editor_query(sql): """Get the query part of an editor command.""" @@ -115,6 +116,7 @@ def get_editor_query(sql): return sql + @export def open_external_editor(filename=None, sql=None): """Open external editor, wait for the user to type in their query, return From 133e197eb1e3af2e339c07039e4839496fce1d8f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:56:14 -0500 Subject: [PATCH 240/824] Last pep8 fix :) --- mycli/main.py | 3 +-- mycli/packages/special/iocommands.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index af535ca30..2e2913d00 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -421,8 +421,7 @@ def handle_editor_command(self, cli, document): filename = special.get_filename(document.text) query = (special.get_editor_query(document.text) or self.get_last_query()) - sql, message = special.open_external_editor(filename, - sql=query) + sql, message = special.open_external_editor(filename, sql=query) if message: # Something went wrong. Raise an exception and bail. raise RuntimeError(message) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 19c2dc735..92773f8de 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -123,6 +123,7 @@ def open_external_editor(filename=None, sql=None): the query. :return: list with one tuple, query as first element. + """ message = None From e8bbe5eb30c683c99d359bfd4c7425c98a065b2a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 07:34:04 -0500 Subject: [PATCH 241/824] Add docstring and use io.open. --- release.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/release.py b/release.py index 817695838..f75277f03 100755 --- a/release.py +++ b/release.py @@ -1,7 +1,10 @@ #!/usr/bin/env python +"""A script to publish a release of mycli to PyPI.""" + from __future__ import print_function import re import ast +import io import subprocess import sys from optparse import OptionParser @@ -51,9 +54,8 @@ def run_step(*args): def version(version_file): _version_re = re.compile(r'__version__\s+=\s+(.*)') - with open(version_file, 'rb') as f: - ver = str(ast.literal_eval(_version_re.search( - f.read().decode('utf-8')).group(1))) + with io.open(version_file, encoding='utf-8') as f: + ver = str(ast.literal_eval(_version_re.search(f.read()).group(1))) return ver @@ -61,7 +63,8 @@ def version(version_file): def commit_for_release(version_file, ver): run_step('git', 'reset') run_step('git', 'add', version_file) - run_step('git', 'commit', '--message', 'Releasing version %s' % ver) + run_step('git', 'commit', '--message', + 'Releasing version {}'.format(ver)) def create_git_tag(tag_name): @@ -128,7 +131,7 @@ def checklist(questions): sys.exit(1) commit_for_release('mycli/__init__.py', ver) - create_git_tag('v%s' % ver) + create_git_tag('v{}'.format(ver)) register_with_pypi() create_distribution_files() push_to_github() From 3cd6492b6105a0babdf45ca0b58d09745fcf2e00 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 07:36:46 -0500 Subject: [PATCH 242/824] Don't evaluate Python code. --- release.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release.py b/release.py index f75277f03..02d0e1f5d 100755 --- a/release.py +++ b/release.py @@ -2,9 +2,8 @@ """A script to publish a release of mycli to PyPI.""" from __future__ import print_function -import re -import ast import io +import re import subprocess import sys from optparse import OptionParser @@ -52,10 +51,11 @@ def run_step(*args): def version(version_file): - _version_re = re.compile(r'__version__\s+=\s+(.*)') + _version_re = re.compile( + r'__version__\s+=\s+(?P[\'"])(?P.*)(?P=quote)') with io.open(version_file, encoding='utf-8') as f: - ver = str(ast.literal_eval(_version_re.search(f.read()).group(1))) + ver = _version_re.search(f.read()).group('version') return ver From f7194c6b19b57879ebee6abac6a7ab22b553680f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 09:29:02 -0500 Subject: [PATCH 243/824] Use click's confirm for release script. --- release.py | 17 +++++------------ requirements-dev.txt | 1 + 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/release.py b/release.py index 02d0e1f5d..499b9cb0b 100755 --- a/release.py +++ b/release.py @@ -3,15 +3,12 @@ from __future__ import print_function import io +from optparse import OptionParser import re import subprocess import sys -from optparse import OptionParser -try: - input = raw_input -except NameError: - pass +import click DEBUG = False CONFIRM_STEPS = False @@ -26,9 +23,7 @@ def skip_step(): global CONFIRM_STEPS if CONFIRM_STEPS: - choice = input("--- Confirm step? (y/N) [y] ") - if choice.lower() == 'n': - return True + return not click.confirm('--- Run this step?', default=True) return False @@ -93,8 +88,7 @@ def push_tags_to_github(): def checklist(questions): for question in questions: - choice = input(question + ' (y/N) [n] ') - if choice.lower() != 'y': + if not click.confirm('--- {}'.format(question), default=False): sys.exit(1) @@ -126,8 +120,7 @@ def checklist(questions): CONFIRM_STEPS = popts.confirm_steps DRY_RUN = popts.dry_run - choice = input('Are you sure? (y/N) [n] ') - if choice.lower() != 'y': + if not click.confirm('Are you sure?', default=False): sys.exit(1) commit_for_release('mycli/__init__.py', ver) diff --git a/requirements-dev.txt b/requirements-dev.txt index b7e6e2dca..cf552f484 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,4 @@ behave pexpect coverage==4.3.4 pep8radius +click==6.7 From 945744537348a185b9bfe6a7ad7ad0d2237e2153 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 09:30:19 -0500 Subject: [PATCH 244/824] Remove unneeded register step. --- release.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/release.py b/release.py index 499b9cb0b..18e1b8f27 100755 --- a/release.py +++ b/release.py @@ -66,10 +66,6 @@ def create_git_tag(tag_name): run_step('git', 'tag', tag_name) -def register_with_pypi(): - run_step('python', 'setup.py', 'register') - - def create_distribution_files(): run_step('python', 'setup.py', 'sdist', 'bdist_wheel') @@ -125,7 +121,6 @@ def checklist(questions): commit_for_release('mycli/__init__.py', ver) create_git_tag('v{}'.format(ver)) - register_with_pypi() create_distribution_files() push_to_github() push_tags_to_github() From 1f1c9369d1fb862d6120ae3540205ff78fb996bd Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 09:47:22 -0500 Subject: [PATCH 245/824] Pin the correct version of pep8radius for development. --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b7e6e2dca..19df802e9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,4 @@ twine==1.8.1 behave pexpect coverage==4.3.4 -pep8radius +git+https://github.com/hayd/pep8radius.git@c8aebd0e1d272160896124e104773b97a6249c3e#egg=pep8radius From c27a2939002aad4c65da75279b46935a10a33fc7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 10:18:31 -0500 Subject: [PATCH 246/824] Add Makefile. --- Makefile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ad6cfbbcc --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +.PHONY: clean lint lint-fix test test-all + +help: + @echo "clean - remove all build artifacts" + @echo "lint - check code changes against PEP 8" + @echo "lint-fix - automatically fix PEP 8 violations" + @echo "test - run tests quickly with the current Python" + @echo "test-all - run tests in all environments" + +clean: + rm -rf build dist egg *.egg-info + find . -name '*.py[co]' -exec rm -f {} + + +lint: + pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) + +lint-fix: + pep8radius master --docformatter --in-place + +test: + if pytest ; then cd test && behave ; fi + +test-all: + tox From a0c38677e3323773129466dd3499633d7cd219e1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 10:22:03 -0500 Subject: [PATCH 247/824] Add Makefile to manifest. --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index a50b73686..f787b9455 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include LICENSE.txt *.md *.rst TODO requirements-dev.txt screenshots/* +include LICENSE.txt *.md *.rst TODO requirements-dev.txt Makefile screenshots/* include conftest.py .coveragerc pytest.ini test tox.ini From 469e0adbb144419da9d89bfd2475b8874b1955e0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 10:22:43 -0500 Subject: [PATCH 248/824] Add Makefile to changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 1e9fd5fda..ad9edfb12 100644 --- a/changelog.md +++ b/changelog.md @@ -27,6 +27,7 @@ Internal Changes: * Better handle common before/after scenarios in behave. (Thanks: [Dick Marinus]) * Added a regression test for sqlparse >= 0.2.3 (Thanks: [Dick Marinus]). * Reverted removal of temporary hack for sqlparse (Thanks: [Dick Marinus]). +* Add Makefile to simplify development tasks (Thanks: [Thomas Roten]). 1.10.0: ======= From 5f75dc18942f62f72a7dbfe94270c43fa3a8f775 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 14:13:57 -0500 Subject: [PATCH 249/824] Streamline development guide using make commands. --- DEVELOP.rst | 99 ++++++++++++++++++++++++----------------------------- 1 file changed, 44 insertions(+), 55 deletions(-) diff --git a/DEVELOP.rst b/DEVELOP.rst index 0d53e6ad7..c4699f626 100644 --- a/DEVELOP.rst +++ b/DEVELOP.rst @@ -1,86 +1,75 @@ Development Guide ----------------- + This is a guide for developers who would like to contribute to this project. +If you're interested in contributing to mycli, thank you. We'd love your help! +You'll always get credit for your work. + GitHub Workflow --------------- -If you're interested in contributing to mycli, first of all my heart felt -thanks. `Fork the project `_ in github. Then -clone your fork into your computer (``git clone ``). Make -the changes and create the commits in your local machine. Then push those -changes to your fork. Then click on the pull request icon on github and create -a new pull request. Add a description about the change and send it along. I -promise to review the pull request in a reasonable window of time and get back -to you. +1. `Fork the repository `_ on GitHub. +2. Clone your fork locally:: -In order to keep your fork up to date with any changes from mainline, add a new -git remote to your local copy called 'upstream' and point it to the main mycli -repo. + $ git clone -:: +3. Add the official repository (``upstream``) as a remote repository:: - $ git remote add upstream git@github.com:dbcli/mycli.git + $ git remote add upstream git@github.com:dbcli/mycli.git -Once the 'upstream' end point is added you can then periodically do a ``git -pull upstream master`` to update your local copy and then do a ``git push -origin master`` to keep your own fork up to date. +4. Set up a `virtual environment `_ + for development:: -Local Setup ------------ + $ cd mycli + $ pip install virtualenv + $ virtualenv mycli_dev -The installation instructions in the README file are intended for users of -mycli. If you're developing mycli, you'll need to install it in a slightly -different way so you can see the effects of your changes right away without -having to go through the install cycle everytime you change the code. + We've just created a virtual environment that we'll use to install all the dependencies + and tools we need to work on mycli. Whenever you want to work on mycli, you + need to activate the virtual environment:: -It is highly recommended to use virtualenv for development. If you don't know -what a virtualenv is, this `guide `_ -will help you get started. + $ source mycli_dev/bin/activate -Create a virtualenv (let's call it mycli-dev). Activate it: +5. Install the dependencies and development tools:: -:: + $ pip install -r requirements-dev.txt + $ pip install --editable . - source ./mycli-dev/bin/activate +6. Create a branch for your bugfix or feature:: -Once the virtualenv is activated, `cd` into the local clone of mycli folder -and install mycli using pip as follows: + $ git checkout -b -:: +7. While you work on your bugfix or feature, be sure to pull the latest changes from ``upstream``. This ensures that your local codebase is up-to-date:: - $ pip install --editable . + $ git pull upstream master + + +Running the Tests +----------------- - or +While you work on mycli, it's important to run the tests to make sure your code +hasn't broken any existing functionality. To run the tests, just type in:: - $ pip install -e . + $ make test -This will install the necessary dependencies as well as install mycli from the -working folder into the virtualenv. By installing it using `pip install -e` -we've linked the mycli installation with the working copy. So any changes made -to the code is immediately available in the installed version of mycli. This -makes it easy to change something in the code, launch mycli and check the -effects of your change. +Mycli supports Python 2.7 and 3.3+. You can test against multiple versions of +Python by running:: -Building DEB package from scratch --------------------- + $ make test-all -First pip install `make-deb`. Then run make-deb. It will create a debian folder -after asking a few questions like maintainer name, email etc. -$ vagrant up +Coding Style +------------ -PEP8 checks ------------ +Mycli requires code submissions to adhere to +`PEP 8 `_. +It's easy to check the style of your code, just run:: -When you submit a PR, the changeset is checked for pep8 compliance using -`pep8radius `_. If you see a build failing because -of these checks, install pep8radius and apply style fixes: + $ make lint -:: +If you see any PEP 8 style issues, you can automatically fix them by running:: - $ pip install pep8radius - $ pep8radius --docformatter --diff # view a diff of proposed fixes - $ pep8radius --docformatter --in-place # apply the fixes + $ make lint-fix -Then commit and push the fixes. +Be sure to commit and push any PEP 8 fixes. From 615564c34b52b022816d64fe22f254834b099c19 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 14:18:43 -0500 Subject: [PATCH 250/824] Remove outdated release procedure file. --- release_procedure.txt | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 release_procedure.txt diff --git a/release_procedure.txt b/release_procedure.txt deleted file mode 100644 index 1b935b624..000000000 --- a/release_procedure.txt +++ /dev/null @@ -1,14 +0,0 @@ -# vi: ft=vimwiki - -* Bump the version number in mycli/__init__.py -* Commit with message: 'Releasing version X.X.X.' -* Create a tag: git tag vX.X.X -* Register with pypi for new version: python setup.py register -* Fix the image url in PyPI to point to github raw content. https://raw.githubusercontent.com/dbcli/mysql-cli/master/screenshots/image01.png -* Create source dist tar ball: python setup.py sdist -* Test this by installing it in a fresh new virtualenv. Run SanityChecks [./sanity_checks.txt]. -* Upload the source dist to PyPI: https://pypi.python.org/pypi/mycli -* pip install mycli -* Run SanityChecks. -* Push the version back to github: git push --tags origin master -* Done! From 593e68b0d60efc0ead15e782e04594e992b8a245 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 21:09:01 -0500 Subject: [PATCH 251/824] Remove unused files. --- MANIFEST.in | 2 +- TODO | 10 ------ Vagrantfile | 30 ------------------ create_deb.sh | 31 ------------------ debian/changelog | 74 ------------------------------------------- debian/compat | 1 - debian/control | 13 -------- debian/mycli.triggers | 8 ----- debian/postinst | 5 --- debian/postrm | 5 --- debian/rules | 4 --- release_procedure.txt | 14 -------- 12 files changed, 1 insertion(+), 196 deletions(-) delete mode 100644 TODO delete mode 100644 Vagrantfile delete mode 100755 create_deb.sh delete mode 100644 debian/changelog delete mode 100644 debian/compat delete mode 100644 debian/control delete mode 100644 debian/mycli.triggers delete mode 100644 debian/postinst delete mode 100644 debian/postrm delete mode 100644 debian/rules delete mode 100644 release_procedure.txt diff --git a/MANIFEST.in b/MANIFEST.in index a50b73686..d9bc01869 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include LICENSE.txt *.md *.rst TODO requirements-dev.txt screenshots/* +include LICENSE.txt *.md *.rst requirements-dev.txt screenshots/* include conftest.py .coveragerc pytest.ini test tox.ini diff --git a/TODO b/TODO deleted file mode 100644 index 64c9842af..000000000 --- a/TODO +++ /dev/null @@ -1,10 +0,0 @@ -# vi: ft=vimwiki - -* [ ] Check if views are available in mysql. -* [ ] Create waffle.io page. -* [ ] Setup gitter. -* [ ] Setup a landing page for mycli.net. -* [ ] Send out invites to backers, pgcli contributors. -* [ ] Write a blog post on personal blog about the experience of kickstarter. -* [ ] Check mycli against MariaDB and Percona. -* [ ] Use error codes instead of matching error strings for reconnect, auto-password prompt etc. diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index a514d1efd..000000000 --- a/Vagrantfile +++ /dev/null @@ -1,30 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure(2) do |config| - - config.vm.synced_folder ".", "/mycli" - - config.vm.define "debian" do |debian| - debian.vm.box = "debian/jessie64" - debian.vm.provision "shell", inline: <<-SHELL - echo "-> Building DEB" - sudo apt-get update - sudo echo "deb http://ppa.launchpad.net/spotify-jyrki/dh-virtualenv/ubuntu trusty main" >> /etc/apt/sources.list - sudo echo "deb-src http://ppa.launchpad.net/spotify-jyrki/dh-virtualenv/ubuntu trusty main" >> /etc/apt/sources.list - sudo apt-get update - sudo apt-get install -y --force-yes python-virtualenv dh-virtualenv debhelper build-essential python-setuptools python-dev - echo "-> Cleaning up old workspace" - rm -rf build - mkdir -p build - cp -r /mycli build/. - cd build/mycli - - echo "-> Creating mycli deb" - dpkg-buildpackage -us -uc - cp ../*.deb /mycli/. - SHELL - end - -end - diff --git a/create_deb.sh b/create_deb.sh deleted file mode 100755 index 8e75ebb31..000000000 --- a/create_deb.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -set -e - -make-deb -cd debian - -cat > postinst <<- EOM -#!/bin/bash - -echo "Setting up symlink to mycli" -ln -sf /usr/share/python/mycli/bin/mycli /usr/local/bin/mycli -EOM -echo "Created postinst file." - -cat > postrm <<- EOM -#!/bin/bash - -echo "Removing symlink to mycli" -rm /usr/local/bin/mycli -EOM -echo "Created postrm file." - -for f in * -do - echo "" >> $f; -done - -echo "INFO: debian folder is setup and ready." -echo "INFO: 1. Update the changelog with real changes." -echo "INFO: 2. Run:\n\tvagrant provision || vagrant up" diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 5135bf68a..000000000 --- a/debian/changelog +++ /dev/null @@ -1,74 +0,0 @@ -mycli (1.7.0) unstable; urgency=medium - - * Add stdin batch mode. (Thanks: Thomas Roten). - * Add warn/no-warn command-line options. (Thanks: Thomas Roten). - * Upgrade sqlparse dependency to 0.1.19. (Thanks: [Amjith Ramanujam]). - * Update features list in README.md. (Thanks: Matheus Rosa). - * Remove extra \n in features list in README.md. (Thanks: Matheus Rosa). - * Enable history search via . (Thanks: [Amjith Ramanujam]). - * Upgrade prompt_toolkit to 1.0.0. (Thanks: Jonathan Slenders) - - -- Casper Langemeijer Fri, 27 May 2016 12:03:31 +0200 - -mycli (1.6.0) unstable; urgency=medium - - * Change continuation prompt for multi-line mode to match default mysql. - * Add status command to match mysql's status command. (Thanks: Thomas Roten). - * Add SSL support for mycli. (Thanks: Artem Bezsmertnyi). - * Add auto-completion and highlight support for OFFSET keyword. (Thanks: Matheus Rosa). - * Add support for MYSQL_TEST_LOGIN_FILE env variable to specify alternate login file. (Thanks: Thomas Roten). - * Add support for --auto-vertical-output to automatically switch to vertical output if the output doesn't fit in the table format. - * Add support for system-wide config. Now /etc/myclirc will be honored. (Thanks: Thomas Roten). - * Add support for nopager and \n to turn off the pager. (Thanks: Thomas Roten). - * Add support for --local-infile command-line option. (Thanks: Thomas Roten). - * Remove -S from less option which was clobbering the scroll back in history. (Thanks: Thomas Roten). - * Make system command work with Python 3. (Thanks: Thomas Roten). - * Support \G terminator for \f queries. (Thanks: Terseus). - * Upgrade prompt_toolkit to 0.60. - * Add Python 3.5 to test environments. (Thanks: Thomas Roten). - * Remove license meta-data. (Thanks: Thomas Roten). - * Skip binary tests if PyMySQL version does not support it. (Thanks: Thomas Roten). - * Refactor pager handling. (Thanks: Thomas Roten) - * Capture warnings to log file. (Thanks: Mikhail Borisov). - * Make syntax_style a tiny bit more intuitive. (Thanks: Phil Cohen). - - -- Casper Langemeijer Fri, 27 May 2016 12:03:31 +0200 - -mycli (1.5.2) unstable; urgency=low - - * Protect against port number being None when no port is specified in command line. - * Cast the value of port read from my.cnf to int. - * Make a config option to enable `audit_log`. (Thanks: [Matheus Rosa]). - * Add support for reading .mylogin.cnf to get user credentials. (Thanks: [Thomas Roten]). - * Register the special command `prompt` with the `\R` as alias. (Thanks: [Matheus Rosa]). - * Perform completion refresh in a background thread. Now mycli can handle - * Add support for `system` command. (Thanks: [Matheus Rosa]). - * Caught and hexed binary fields in MySQL. (Thanks: [Daniel West]). - * Treat enter key as tab when the suggestion menu is open. (Thanks: [Matheus Rosa]) - * Add "delete" and "truncate" as destructive commands. (Thanks: [Martijn Engler]). - * Change \dt syntax to add an optional table name. (Thanks: [Shoma Suzuki]). - * Add TRANSACTION related keywords. - * Treat DESC and EXPLAIN as DESCRIBE. (Thanks: [spacewander]). - * Fix the removal of whitespace from table output. - * Add ability to make suggestions for compound join clauses. (Thanks: [Matheus Rosa]). - * Fix the incorrect reporting of command time. - * Add type validation for port argument. (Thanks [Matheus Rosa]) - * Make pycrypto optional and only install it in \*nix systems. (Thanks: [Iryna Cherniavska]). - * Add badge for PyPI version to README. (Thanks: [Shoma Suzuki]). - * Updated release script with a --dry-run and --confirm-steps option. (Thanks: [Iryna Cherniavska]). - * Adds support for PyMySQL 0.6.2 and above. This is useful for debian package builders. (Thanks: [Thomas Roten]). - * Disable click warning. - - -- Casper Langemeijer Sun, 15 Nov 2015 10:26:24 +0100 - -mycli (1.4.0) unstable; urgency=low - - * Add `source` command. This allows running sql statement from a file. - * Added a config option to make the warning before destructive commands optional. (Thanks: [Daniel West](https://github.com/danieljwest)) - * Add completion support for CHANGE TO and other master/slave commands. This is still preliminary and it will be enhanced in the future. - * Add custom styles to color the menus and toolbars. - * Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders](https://github.com/jonathanslenders)) - * Fix keyword completion after the `WHERE` clause. - * Add `\g` and `\G` as valid query terminators. Previously in multi-line mode ending a query with a `\G` wouldn't run the query. This is now fixed. - - -- Amjith Ramanujam Sun, 23 Aug 2015 20:14:45 +0000 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144f..000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control deleted file mode 100644 index 418383263..000000000 --- a/debian/control +++ /dev/null @@ -1,13 +0,0 @@ -Source: mycli -Section: python -Priority: extra -Maintainer: Amjith Ramanujam -Build-Depends: debhelper (>= 9), python, dh-virtualenv (>= 0.7), python-setuptools, python-dev -Standards-Version: 3.9.5 - -Package: mycli -Architecture: any -Pre-Depends: dpkg (>= 1.16.1), python2.7-minimal, ${misc:Pre-Depends} -Depends: ${python:Depends}, ${misc:Depends} -Description: CLI for MySQL Database. With auto-completion and syntax highlighting. - CLI for MySQL Database. With auto-completion and syntax highlighting. diff --git a/debian/mycli.triggers b/debian/mycli.triggers deleted file mode 100644 index b0b1d2184..000000000 --- a/debian/mycli.triggers +++ /dev/null @@ -1,8 +0,0 @@ -# Register interest in Python interpreter changes (Python 2 for now); and -# don't make the Python package dependent on the virtualenv package -# processing (noawait) -interest-noawait /usr/bin/python2.7 - -# Also provide a symbolic trigger for all dh-virtualenv packages -interest dh-virtualenv-interpreter-update - diff --git a/debian/postinst b/debian/postinst deleted file mode 100644 index 122ff285d..000000000 --- a/debian/postinst +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -echo "Setting up symlink to mycli" -ln -sf /usr/share/python/mycli/bin/mycli /usr/local/bin/mycli - diff --git a/debian/postrm b/debian/postrm deleted file mode 100644 index 850d6e8d8..000000000 --- a/debian/postrm +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -echo "Removing symlink to mycli" -rm /usr/local/bin/mycli - diff --git a/debian/rules b/debian/rules deleted file mode 100644 index 299e30912..000000000 --- a/debian/rules +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/make -f - -%: - dh $@ --with python-virtualenv diff --git a/release_procedure.txt b/release_procedure.txt deleted file mode 100644 index 1b935b624..000000000 --- a/release_procedure.txt +++ /dev/null @@ -1,14 +0,0 @@ -# vi: ft=vimwiki - -* Bump the version number in mycli/__init__.py -* Commit with message: 'Releasing version X.X.X.' -* Create a tag: git tag vX.X.X -* Register with pypi for new version: python setup.py register -* Fix the image url in PyPI to point to github raw content. https://raw.githubusercontent.com/dbcli/mysql-cli/master/screenshots/image01.png -* Create source dist tar ball: python setup.py sdist -* Test this by installing it in a fresh new virtualenv. Run SanityChecks [./sanity_checks.txt]. -* Upload the source dist to PyPI: https://pypi.python.org/pypi/mycli -* pip install mycli -* Run SanityChecks. -* Push the version back to github: git push --tags origin master -* Done! From 1136a1f88c4fe333348b9031341fc5ed2acbbcc5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 21:13:27 -0500 Subject: [PATCH 252/824] Move coverage config to setup.cfg. --- .coveragerc | 3 --- setup.cfg | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index ae818eefb..000000000 --- a/.coveragerc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -parallel=True -source=mycli diff --git a/setup.cfg b/setup.cfg index 2a9acf13d..c017a54d9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,6 @@ [bdist_wheel] universal = 1 + +[coverage:run] +parallel=True +source=mycli From ffd74e6979acc81c5e21957716d740dc3643e343 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 21:14:36 -0500 Subject: [PATCH 253/824] Move pytest config to setup.cfg. --- pytest.ini | 2 -- setup.cfg | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 7c5b52b77..000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts=--capture=sys --showlocals --doctest-modules diff --git a/setup.cfg b/setup.cfg index c017a54d9..b92934a4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,3 +4,6 @@ universal = 1 [coverage:run] parallel=True source=mycli + +[pytest] +addopts=--capture=sys --showlocals --doctest-modules From ead3a32ae7c370b834e3d7de1e91bcd1f1db9d59 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 21:19:11 -0500 Subject: [PATCH 254/824] Update pytest section to tool:pytest. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b92934a4b..994fc8d84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,5 +5,5 @@ universal = 1 parallel=True source=mycli -[pytest] +[tool:pytest] addopts=--capture=sys --showlocals --doctest-modules From ff7bc198ee6c9311bcf8451654c64992a6d8393c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 22:11:07 -0500 Subject: [PATCH 255/824] Move conftest.py into setup.cfg. --- conftest.py | 13 ------------- setup.cfg | 8 +++++++- 2 files changed, 7 insertions(+), 14 deletions(-) delete mode 100644 conftest.py diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 41e72adae..000000000 --- a/conftest.py +++ /dev/null @@ -1,13 +0,0 @@ -import sys -collect_ignore = [ - "setup.py", - "mycli/magic.py", - "mycli/packages/parseutils.py", - "test/features/environment.py", - "test/features/steps/basic_commands.py", - "test/features/steps/crud_database.py", - "test/features/steps/crud_table.py", - "test/features/steps/iocommands.py", - "test/features/steps/named_queries.py", - "test/features/steps/specials.py", -] diff --git a/setup.cfg b/setup.cfg index 994fc8d84..0854bbe41 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,4 +6,10 @@ parallel=True source=mycli [tool:pytest] -addopts=--capture=sys --showlocals --doctest-modules +addopts = --capture=sys + --showlocals + --doctest-modules + --ignore=setup.py + --ignore=mycli/magic.py + --ignore=mycli/packages/parseutils.py + --ignore=test/features From 0a4675dd65e1b6379d1505379e2e8a776a772eb5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 22:11:24 -0500 Subject: [PATCH 256/824] Standardize spacing in setup.cfg. --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0854bbe41..041a857b1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,8 +2,8 @@ universal = 1 [coverage:run] -parallel=True -source=mycli +parallel = True +source = mycli [tool:pytest] addopts = --capture=sys From befb8a02dd63b98ba2558a8213244a41cc07c75d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 22:21:08 -0500 Subject: [PATCH 257/824] Use setup.cfg for coverage subprocess tests. --- test/features/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/features/environment.py b/test/features/environment.py index 138f88f15..0caf4793d 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -16,7 +16,7 @@ def before_all(context): os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" os.environ['EDITOR'] = 'ex' - os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../.coveragerc" + os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../setup.cfg" context.exit_sent = False From e9179a7a8a2d0ff9b24ce019251d5f571d2cf990 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 23:12:31 -0500 Subject: [PATCH 258/824] Use pytest-cov plugin. --- .travis.yml | 5 +++-- requirements-dev.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index de590fc05..5e2dea07e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,13 @@ python: - "3.6" install: - - pip install PyMySQL . pytest mock codecov pexpect behave + - pip install PyMySQL pytest mock codecov pexpect behave pytest-cov - pip install git+https://github.com/hayd/pep8radius.git + - pip install -e . script: - set -e - - coverage run --source mycli -m py.test + - pytest --cov-report= --cov=mycli - cd test - behave - cd .. diff --git a/requirements-dev.txt b/requirements-dev.txt index cf552f484..c7e7c2c4f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ mock pytest +pytest-cov==2.4.0 tox twine==1.8.1 behave From ec4fad0716952c15a6e081399ea49f23ead83849 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 23:21:38 -0500 Subject: [PATCH 259/824] Travis linter says to drop on_start. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5e2dea07e..7a1023d54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,4 +31,3 @@ notifications: - YOUR_WEBHOOK_URL on_success: change # options: [always|never|change] default: always on_failure: always # options: [always|never|change] default: always - on_start: false # default: false From 3065c0d2bf72c57ca373c800e697324f84e6c4ab Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 23:25:20 -0500 Subject: [PATCH 260/824] pytest isn't found on <3.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7a1023d54..c68e6a9ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ install: script: - set -e - - pytest --cov-report= --cov=mycli + - py.test --cov-report= --cov=mycli - cd test - behave - cd .. From d06ccd48ba0f2c51a262ba8f2e57f3eaab391fdb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 00:02:39 -0500 Subject: [PATCH 261/824] Make behave work from project root as well. --- .travis.yml | 4 +--- setup.cfg | 2 ++ test/features/environment.py | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index c68e6a9ba..49ec67d90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,9 +14,7 @@ install: script: - set -e - py.test --cov-report= --cov=mycli - - cd test - - behave - - cd .. + - behave test/features # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) - set +e diff --git a/setup.cfg b/setup.cfg index 041a857b1..b7286a9bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,6 +9,8 @@ source = mycli addopts = --capture=sys --showlocals --doctest-modules + --cov-report= + --cov=mycli --ignore=setup.py --ignore=mycli/magic.py --ignore=mycli/packages/parseutils.py diff --git a/test/features/environment.py b/test/features/environment.py index 0caf4793d..42ed8df11 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -10,13 +10,16 @@ from steps.wrappers import run_cli, wait_prompt +PACKAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + def before_all(context): """Set env parameters.""" os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" os.environ['EDITOR'] = 'ex' - os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../setup.cfg" + os.environ["COVERAGE_PROCESS_START"] = os.path.join(PACKAGE_ROOT, + 'setup.cfg') context.exit_sent = False @@ -48,7 +51,9 @@ def before_all(context): 'pager_boundary': '---boundary---', } os.environ['PAGER'] = "{0} {1} {2}".format( - sys.executable, "test/features/wrappager.py", context.conf['pager_boundary']) + sys.executable, + os.path.join(PACKAGE_ROOT, '/test/features/wrappager.py'), + context.conf['pager_boundary']) context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], context.conf['pass'], From 88d001ed77935cf2d44fb6f5524a51edfe55e6f8 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 00:03:43 -0500 Subject: [PATCH 262/824] Don't run coverage for every py.test. --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index b7286a9bf..041a857b1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,8 +9,6 @@ source = mycli addopts = --capture=sys --showlocals --doctest-modules - --cov-report= - --cov=mycli --ignore=setup.py --ignore=mycli/magic.py --ignore=mycli/packages/parseutils.py From ea92f9e42e7a62340db21b854553281c2fc4403c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 00:08:22 -0500 Subject: [PATCH 263/824] Fix wrappager path. --- test/features/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/features/environment.py b/test/features/environment.py index 42ed8df11..61a579e53 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -52,7 +52,7 @@ def before_all(context): } os.environ['PAGER'] = "{0} {1} {2}".format( sys.executable, - os.path.join(PACKAGE_ROOT, '/test/features/wrappager.py'), + os.path.join(PACKAGE_ROOT, 'test/features/wrappager.py'), context.conf['pager_boundary']) context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], From adf6b939196d63cd3db44167ddc67cabfcc5753d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 06:44:10 -0500 Subject: [PATCH 264/824] Drop pytest-cov. --- .travis.yml | 4 ++-- requirements-dev.txt | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 49ec67d90..10cb01a8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,13 +7,13 @@ python: - "3.6" install: - - pip install PyMySQL pytest mock codecov pexpect behave pytest-cov + - pip install PyMySQL pytest mock codecov pexpect behave - pip install git+https://github.com/hayd/pep8radius.git - pip install -e . script: - set -e - - py.test --cov-report= --cov=mycli + - coverage run -m py.test - behave test/features # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) diff --git a/requirements-dev.txt b/requirements-dev.txt index c7e7c2c4f..cf552f484 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,5 @@ mock pytest -pytest-cov==2.4.0 tox twine==1.8.1 behave From 47dcafb642adfa9a3853629a1fe0293fea451c07 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 06:44:19 -0500 Subject: [PATCH 265/824] Use absolute path for package root. --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 3eb513c85..ca03900f6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -56,7 +56,7 @@ # Query tuples are used for maintaining history Query = namedtuple('Query', ['query', 'successful', 'mutating']) -PACKAGE_ROOT = os.path.dirname(__file__) +PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__)) # no-op logging handler class NullHandler(logging.Handler): From d4d1f0d88e8a1bd0325bdbbdab658d938371df47 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 08:16:22 -0500 Subject: [PATCH 266/824] Move coverage config back to coveragerc. --- .coveragerc | 3 +++ setup.cfg | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..8d3149f62 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +parallel = True +source = mycli diff --git a/setup.cfg b/setup.cfg index 041a857b1..5d578a99a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,6 @@ [bdist_wheel] universal = 1 -[coverage:run] -parallel = True -source = mycli - [tool:pytest] addopts = --capture=sys --showlocals From b471f5005c5b91963852fdbfca7d4c49697dd397 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 08:16:50 -0500 Subject: [PATCH 267/824] Make behave support running from project/test dir. --- test/features/environment.py | 12 +++++++----- test/features/steps/wrappers.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/features/environment.py b/test/features/environment.py index 61a579e53..f43ff8426 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -10,16 +10,18 @@ from steps.wrappers import run_cli, wait_prompt -PACKAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - def before_all(context): """Set env parameters.""" os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" os.environ['EDITOR'] = 'ex' - os.environ["COVERAGE_PROCESS_START"] = os.path.join(PACKAGE_ROOT, - 'setup.cfg') + + context.package_root = os.path.abspath( + os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + + os.environ["COVERAGE_PROCESS_START"] = os.path.join(context.package_root, + '.coveragerc') context.exit_sent = False @@ -52,7 +54,7 @@ def before_all(context): } os.environ['PAGER'] = "{0} {1} {2}".format( sys.executable, - os.path.join(PACKAGE_ROOT, 'test/features/wrappager.py'), + os.path.join(context.package_root, 'test/features/wrappager.py'), context.conf['pager_boundary']) context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], diff --git a/test/features/steps/wrappers.py b/test/features/steps/wrappers.py index 3855ede07..5070f053c 100644 --- a/test/features/steps/wrappers.py +++ b/test/features/steps/wrappers.py @@ -38,7 +38,7 @@ def run_cli(context): cmd_parts = [cli_cmd] + run_args cmd = ' '.join(cmd_parts) - context.cli = pexpect.spawnu(cmd, cwd='..') + context.cli = pexpect.spawnu(cmd, cwd=context.package_root) context.exit_sent = False context.currentdb = context.conf['dbname'] From 21cd18b556fcb7b6ea891c2f68ae29d128713bdf Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 08:34:11 -0500 Subject: [PATCH 268/824] Make tee files relative to package root. --- test/features/steps/iocommands.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 5de640ffb..6fe36f4a1 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -10,8 +10,8 @@ @when('we start external editor providing a file name') def step_edit_file(context): """Edit file with external editor.""" - context.editor_file_name = '../test_file_{0}.sql'.format( - context.conf['vi']) + context.editor_file_name = os.path.join( + context.package_root, 'test_file_{0}.sql'.format(context.conf['vi'])) if os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) context.cli.sendline('\e {0}'.format( @@ -48,7 +48,8 @@ def step_edit_done_sql(context): @when(u'we tee output') def step_tee_ouptut(context): - context.tee_file_name = '../tee_file_{0}.sql'.format(context.conf['vi']) + context.tee_file_name = os.path.join( + context.package_root, 'tee_file_{0}.sql'.format(context.conf['vi'])) if os.path.exists(context.tee_file_name): os.remove(context.tee_file_name) context.cli.sendline('tee {0}'.format( From ba063c11e38c616ad047f4c02a0cc62dcaf9690d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 09:59:02 -0500 Subject: [PATCH 269/824] Fix test files and config in MANIFEST.in. --- MANIFEST.in | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index d9bc01869..798f07a8a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,6 @@ include LICENSE.txt *.md *.rst requirements-dev.txt screenshots/* -include conftest.py .coveragerc pytest.ini test tox.ini +include .coveragerc tox.ini +recursive-include test *.cnf +recursive-include test *.feature +recursive-include test *.py +recursive-include test *.txt From c606deb3e8e86e73294d92bf714298370d6b1e05 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 17:03:24 -0500 Subject: [PATCH 270/824] Revert "Drop pytest-cov." This reverts commit adf6b939196d63cd3db44167ddc67cabfcc5753d. --- .travis.yml | 4 ++-- requirements-dev.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 10cb01a8a..49ec67d90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,13 +7,13 @@ python: - "3.6" install: - - pip install PyMySQL pytest mock codecov pexpect behave + - pip install PyMySQL pytest mock codecov pexpect behave pytest-cov - pip install git+https://github.com/hayd/pep8radius.git - pip install -e . script: - set -e - - coverage run -m py.test + - py.test --cov-report= --cov=mycli - behave test/features # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) diff --git a/requirements-dev.txt b/requirements-dev.txt index cf552f484..c7e7c2c4f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ mock pytest +pytest-cov==2.4.0 tox twine==1.8.1 behave From 229a38c381f52ff7fc13c33b0ae1952791ed9b5e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 9 May 2017 00:49:35 -0500 Subject: [PATCH 271/824] Move Makefile tasks to setup.py. --- .travis.yml | 3 +- MANIFEST.in | 4 +- Makefile | 24 ------------ setup.py | 14 ++++--- tasks.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 2 +- 6 files changed, 117 insertions(+), 34 deletions(-) delete mode 100644 Makefile mode change 100644 => 100755 setup.py create mode 100644 tasks.py diff --git a/.travis.yml b/.travis.yml index de590fc05..fff93d10e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,7 @@ script: - cd test - behave - cd .. - # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) + - ./setup.py lint - set +e after_success: diff --git a/MANIFEST.in b/MANIFEST.in index f787b9455..af8dd71a8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include LICENSE.txt *.md *.rst TODO requirements-dev.txt Makefile screenshots/* -include conftest.py .coveragerc pytest.ini test tox.ini +include LICENSE.txt *.md *.rst TODO requirements-dev.txt screenshots/* +include tasks.py conftest.py .coveragerc pytest.ini test tox.ini diff --git a/Makefile b/Makefile deleted file mode 100644 index ad6cfbbcc..000000000 --- a/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -.PHONY: clean lint lint-fix test test-all - -help: - @echo "clean - remove all build artifacts" - @echo "lint - check code changes against PEP 8" - @echo "lint-fix - automatically fix PEP 8 violations" - @echo "test - run tests quickly with the current Python" - @echo "test-all - run tests in all environments" - -clean: - rm -rf build dist egg *.egg-info - find . -name '*.py[co]' -exec rm -f {} + - -lint: - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) - -lint-fix: - pep8radius master --docformatter --in-place - -test: - if pytest ; then cd test && behave ; fi - -test-all: - tox diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 5140b7e9b..bc39b29de --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ +#!/usr/bin/env python + import re import ast -import platform from setuptools import setup, find_packages _version_re = re.compile(r'__version__\s+=\s+(.*)') @@ -33,10 +34,13 @@ description=description, long_description=description, install_requires=install_requirements, - entry_points=''' - [console_scripts] - mycli=mycli.main:cli - ''', + entry_points={ + 'console_scripts': ['mycli = mycli.main:cli'], + 'distutils.commands': [ + 'lint = tasks:lint', + 'test = tasks:test', + ], + }, classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', diff --git a/tasks.py b/tasks.py new file mode 100644 index 000000000..b65c019e1 --- /dev/null +++ b/tasks.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +"""Common development tasks for setup.py to use.""" + +import re +import subprocess +import sys + +from setuptools import Command + + +class BaseCommand(Command, object): + """The base command for project tasks.""" + + user_options = [] + + default_cmd_options = ('verbose', 'quiet', 'dry_run') + + def __init__(self, *args, **kwargs): + super(BaseCommand, self).__init__(*args, **kwargs) + self.verbose = False + + def initialize_options(self): + """Override the distutils abstract method.""" + pass + + def finalize_options(self): + """Override the distutils abstract method.""" + # Distutils uses incrementing integers for verbosity. + self.verbose = bool(self.verbose) + + def call_and_exit(self, cmd, shell=True): + """Run the *cmd* and exit with the proper exit code.""" + sys.exit(subprocess.call(cmd, shell=shell)) + + def call_in_sequence(self, cmds, shell=True): + """Run multiple commmands in a row, exiting if one fails.""" + for cmd in cmds: + if subprocess.call(cmd, shell=shell) == 1: + sys.exit(1) + + def apply_options(self, cmd, options=()): + """Apply command-line options.""" + for option in (self.default_cmd_options + options): + cmd = self.apply_option(cmd, option, + active=getattr(self, option, False)) + return cmd + + def apply_option(self, cmd, option, active=True): + """Apply a command-line option.""" + return re.sub(r'{{{}\:(?P