From 9ac202b169f174eebe94c66c5c8ac7fbd00b09f2 Mon Sep 17 00:00:00 2001 From: Daniel West Date: Sat, 15 Aug 2015 10:45:06 -0400 Subject: [PATCH 0001/1025] Caught and hexed certain binary fields coming from pymysql. Resolved #106 and #102 --- mycli/packages/expanded.py | 14 +++++++++++++- mycli/packages/tabulate.py | 17 ++++++++++------- tests/test_expanded.py | 1 + tests/test_sqlexecute.py | 24 ++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 7f082350f..4f8c25915 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,4 +1,5 @@ from .tabulate import _text_type +import codecs def pad(field, total, char=u" "): return field + (char * (total - len(field))) @@ -8,6 +9,16 @@ def get_separator(num, header_len, data_len): sep = u"***************************[ %d. row ]***************************\n" % (num + 1) return sep +def format_field(value): + # Returns the field as a text type, otherwise will hexify the string + try: + if isinstance(value, bytes): + return _text_type(value, "ascii") + else: + return _text_type(value) + except UnicodeDecodeError: + return _text_type('0x' + (codecs.getencoder('hex_codec')(value)[0]).decode('ascii')) + def expanded_table(rows, headers): header_len = max([len(x) for x in headers]) max_row_len = 0 @@ -17,7 +28,8 @@ def expanded_table(rows, headers): header_len += 2 for row in rows: - row_len = max([len(_text_type(x)) for x in row]) + row = [format_field(x) for x in row] + row_len = max([len(x) for x in row]) row_result = [] if row_len > max_row_len: max_row_len = row_len diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index e093f83b9..0c95dab10 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -9,6 +9,7 @@ from platform import python_version_tuple from wcwidth import wcswidth import re +import codecs if python_version_tuple()[0] < "3": @@ -519,6 +520,8 @@ def _format(val, valtype, floatfmt, missingval=""): elif valtype is _binary_type: try: return _text_type(val, "ascii") + except UnicodeDecodeError: + return _text_type('0x' + (codecs.getencoder('hex_codec')(val)[0]).decode('ascii')) except TypeError: return _text_type(val) elif valtype is float: @@ -886,22 +889,22 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", tabular_data = [] list_of_lists, headers = _normalize_tabular_data(tabular_data, headers) + # format rows and columns, convert numeric values to strings + cols = list(zip(*list_of_lists)) + coltypes = list(map(_column_type, cols)) + cols = [[_format(v, ct, floatfmt, missingval) for v in c] + for c,ct in zip(cols, coltypes)] + # optimization: look for ANSI control codes once, # enable smart width functions only if a control code is found plain_text = '\n'.join(['\t'.join(map(_text_type, headers))] + \ - ['\t'.join(map(_text_type, row)) for row in list_of_lists]) + ['\t'.join(map(_text_type, row)) for row in cols]) has_invisible = re.search(_invisible_codes, plain_text) if has_invisible: width_fn = _visible_width else: width_fn = wcswidth - # format rows and columns, convert numeric values to strings - cols = list(zip(*list_of_lists)) - coltypes = list(map(_column_type, cols)) - cols = [[_format(v, ct, floatfmt, missingval) for v in c] - for c,ct in zip(cols, coltypes)] - # align columns aligns = [numalign if ct in [int,float] else stralign for ct in coltypes] minwidths = [width_fn(h) + MIN_PADDING for h in headers] if headers else [0]*len(cols) diff --git a/tests/test_expanded.py b/tests/test_expanded.py index 9b2a6c5cf..a06009a6b 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -11,3 +11,4 @@ def test_expanded_table_renders(): age | 456 """ assert expected == expanded_table(input, ["name", "age"]) + diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index 0904481b0..edc3ecc55 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -31,6 +31,30 @@ def test_bools(executor): +-----+ 1 row in set""") +@dbtest +def test_binary(executor): + run(executor, '''create table bt(geom linestring NOT NULL)''') + run(executor, '''INSERT INTO bt VALUES (GeomFromText('LINESTRING(116.37604 39.73979,116.375 39.73965)'));''') + results = run(executor, '''select * from bt''', join=True) + assert results == dedent("""\ + +----------------------------------------------------------------------------------------------+ + | geom | + |----------------------------------------------------------------------------------------------| + | 0x00000000010200000002000000397f130a11185d4034f44f70b1de43400000000000185d40423ee8d9acde4340 | + +----------------------------------------------------------------------------------------------+ + 1 row in set""") + +@dbtest +def test_binary_expanded(executor): + run(executor, '''create table bt(geom linestring NOT NULL)''') + run(executor, '''INSERT INTO bt VALUES (GeomFromText('LINESTRING(116.37604 39.73979,116.375 39.73965)'));''') + results = run(executor, '''select * from bt\G''', join=True) + assert results == dedent("""\ + ***************************[ 1. row ]*************************** + geom | 0x00000000010200000002000000397f130a11185d4034f44f70b1de43400000000000185d40423ee8d9acde4340 + + 1 row in set""") + @dbtest def test_table_and_columns_query(executor): run(executor, "create table a(x text, y text)") From 15be052d86d3790cc82be20cddbf9b10a5bd24a2 Mon Sep 17 00:00:00 2001 From: Daniel West Date: Sat, 15 Aug 2015 16:28:08 -0400 Subject: [PATCH 0002/1025] Replacing codecs with binascii for hex generation --- mycli/packages/expanded.py | 4 ++-- mycli/packages/tabulate.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 4f8c25915..128e9c690 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,5 +1,5 @@ from .tabulate import _text_type -import codecs +import binascii def pad(field, total, char=u" "): return field + (char * (total - len(field))) @@ -17,7 +17,7 @@ def format_field(value): else: return _text_type(value) except UnicodeDecodeError: - return _text_type('0x' + (codecs.getencoder('hex_codec')(value)[0]).decode('ascii')) + return _text_type('0x' + binascii.hexlify(value).decode('ascii')) def expanded_table(rows, headers): header_len = max([len(x) for x in headers]) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index 0c95dab10..6e92e8f15 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -9,7 +9,7 @@ from platform import python_version_tuple from wcwidth import wcswidth import re -import codecs +import binascii if python_version_tuple()[0] < "3": @@ -521,7 +521,7 @@ def _format(val, valtype, floatfmt, missingval=""): try: return _text_type(val, "ascii") except UnicodeDecodeError: - return _text_type('0x' + (codecs.getencoder('hex_codec')(val)[0]).decode('ascii')) + return _text_type('0x' + binascii.hexlify(val).decode('ascii')) except TypeError: return _text_type(val) elif valtype is float: From 7fdd1b2c025e42454782ebaef13401d12dfc0c53 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 23 Aug 2015 19:29:30 -0700 Subject: [PATCH 0003/1025] Add keywords to completion suggestions for WHERE clause. --- mycli/packages/completion_engine.py | 13 +----- tests/test_completion_engine.py | 44 ++++++++++++++----- ...est_smart_completion_public_schema_only.py | 16 ++++--- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index de81d4ade..ff1386dac 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -157,16 +157,6 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier prev_tok = prev_tok.value.lower() if prev_tok == 'exists': return [{'type': 'keyword'}] - elif prev_tok in ('any', 'some', 'all'): - return column_suggestions + [{'type': 'keyword'}] - elif prev_tok == 'in': - # Technically, we should suggest columns AND keywords, as - # per case 4. However, IN is different from ANY, SOME, ALL - # in that it can accept a *list* of columns, or a subquery. - # But suggesting keywords for , "SELECT * FROM foo WHERE bar IN - # (baz, qux, " would be overwhelming. So we special case 'IN' - # to not suggest keywords. - return column_suggestions else: return column_suggestions @@ -214,7 +204,8 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier {'type': 'function', 'schema': parent}] else: return [{'type': 'column', 'tables': extract_tables(full_text)}, - {'type': 'function', 'schema': []}] + {'type': 'function', 'schema': []}, + {'type': 'keyword'}] elif (token_v.endswith('join') and token.is_keyword) or (token_v in ('copy', 'from', 'update', 'into', 'describe', 'truncate')): schema = (identifier and identifier.get_parent_name()) or [] diff --git a/tests/test_completion_engine.py b/tests/test_completion_engine.py index dbc8ec593..e3c94a970 100644 --- a/tests/test_completion_engine.py +++ b/tests/test_completion_engine.py @@ -9,13 +9,17 @@ def test_select_suggests_cols_with_visible_table_scope(): suggestions = suggest_type('SELECT FROM tabl', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) def test_select_suggests_cols_with_qualified_table_scope(): suggestions = suggest_type('SELECT FROM sch.tabl', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [('sch', 'tabl', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) @pytest.mark.parametrize('expression', [ @@ -34,7 +38,9 @@ def test_where_suggests_columns_functions(expression): suggestions = suggest_type(expression, expression) assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) @pytest.mark.parametrize('expression', [ 'SELECT * FROM tabl WHERE foo IN (', @@ -44,7 +50,9 @@ def test_where_in_suggests_columns(expression): suggestions = suggest_type(expression, expression) assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) def test_where_equals_any_suggests_columns_or_keywords(): text = 'SELECT * FROM tabl WHERE foo = ANY(' @@ -63,7 +71,9 @@ def test_select_suggests_cols_and_funcs(): suggestions = suggest_type('SELECT ', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': []}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) @pytest.mark.parametrize('expression', [ 'SELECT * FROM ', @@ -113,7 +123,9 @@ def test_col_comma_suggests_cols(): suggestions = suggest_type('SELECT a, b, FROM tbl', 'SELECT a, b,') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tbl', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) def test_table_comma_suggests_tables_and_schemas(): suggestions = suggest_type('SELECT a, b FROM tbl1, ', @@ -147,7 +159,9 @@ def test_partially_typed_col_name_suggests_col_names(): 'SELECT * FROM tabl WHERE col_n') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) def test_dot_suggests_cols_of_a_table_or_schema_qualified_table(): suggestions = suggest_type('SELECT tabl. FROM tabl', 'SELECT tabl.') @@ -219,7 +233,9 @@ def test_sub_select_col_name_completion(): 'SELECT * FROM (SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'abc', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) @pytest.mark.xfail def test_sub_select_multiple_col_name_completion(): @@ -312,7 +328,9 @@ def test_2_statements_2nd_current(): 'select * from a; select ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'b', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) # Should work even if first statement is invalid suggestions = suggest_type('select * from; select * from ', @@ -334,7 +352,9 @@ def test_2_statements_1st_current(): 'select ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'a', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) def test_3_statements_2nd_current(): suggestions = suggest_type('select * from a; select * from ; select * from c', @@ -348,7 +368,9 @@ def test_3_statements_2nd_current(): 'select * from a; select ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'b', None)]}, - {'type': 'function', 'schema': []}]) + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) def test_create_db_with_template(): diff --git a/tests/test_smart_completion_public_schema_only.py b/tests/test_smart_completion_public_schema_only.py index f44c1f22a..e99567a07 100644 --- a/tests/test_smart_completion_public_schema_only.py +++ b/tests/test_smart_completion_public_schema_only.py @@ -69,7 +69,9 @@ def test_function_name_completion(completer, complete_event): position = len('SELECT MA') result = completer.get_completions( Document(text=text, cursor_position=position), complete_event) - assert set(result) == set([Completion(text='MAX', start_position=-2)]) + assert set(result) == set([Completion(text='MAX', start_position=-2), + Completion(text='MASTER', start_position=-2), + ]) def test_suggested_column_names(completer, complete_event): """ @@ -89,7 +91,8 @@ def test_suggested_column_names(completer, complete_event): Completion(text='email', start_position=0), Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)] + - list(map(Completion, completer.functions))) + list(map(Completion, completer.functions)) + + list(map(Completion, completer.keywords))) def test_suggested_column_names_in_function(completer, complete_event): """ @@ -168,7 +171,8 @@ def test_suggested_multiple_column_names(completer, complete_event): Completion(text='email', start_position=0), Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)] + - list(map(Completion, completer.functions))) + list(map(Completion, completer.functions)) + + list(map(Completion, completer.keywords))) def test_suggested_multiple_column_names_with_alias(completer, complete_event): """ @@ -274,7 +278,8 @@ def test_auto_escaped_col_names(completer, complete_event): Completion(text='id', start_position=0), Completion(text='`insert`', start_position=0), Completion(text='`ABC`', start_position=0), ] + - list(map(Completion, completer.functions))) + list(map(Completion, completer.functions)) + + list(map(Completion, completer.keywords))) def test_un_escaped_table_names(completer, complete_event): text = 'SELECT from réveillé' @@ -287,4 +292,5 @@ def test_un_escaped_table_names(completer, complete_event): Completion(text='id', start_position=0), Completion(text='`insert`', start_position=0), Completion(text='`ABC`', start_position=0), ] + - list(map(Completion, completer.functions))) + list(map(Completion, completer.functions)) + + list(map(Completion, completer.keywords))) From 8cf3ac9ed53dd267bf5fd53e24dcf020333edef0 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 23 Aug 2015 20:02:12 -0700 Subject: [PATCH 0004/1025] Update changelog for 1.4.0 release. --- changelog.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/changelog.md b/changelog.md index 515ee0a85..ff6ec055e 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,37 @@ +1.4.0: +====== + +Features: +--------- + +* Add `source` command. This allows running sql statement from a file. + + eg: + ``` + mycli> source filename.sql + ``` + +* Added a config option to make the warning before destructive commands optional. (Thanks: [Daniel West](https://github.com/danieljwest)) + + In the config file ~/.myclirc set `destructive_warning = False` which will + disable the warning before running `DROP` commands. + +* Add completion support for CHANGE TO and other master/slave commands. This is + still preliminary and it will be enhanced in the future. + +* Add custom styles to color the menus and toolbars. + +* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders](https://github.com/jonathanslenders)) + + Multi-line queries are automatically indented. + +Bug Fixes: +---------- + +* Fix keyword completion after the `WHERE` clause. +* Add `\g` and `\G` as valid query terminators. Previously in multi-line mode + ending a query with a `\G` wouldn't run the query. This is now fixed. + 1.3.0: ====== From ec8005a423bc915c887f9eaffbc139f7e7473174 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 23 Aug 2015 20:13:45 -0700 Subject: [PATCH 0005/1025] Update debian changelog. --- debian/changelog | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/debian/changelog b/debian/changelog index 59c89cc04..b44bdb458 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,11 @@ -mycli (1.3.0) unstable; urgency=low +mycli (1.4.0) unstable; urgency=low - * Add a new special command (\T) to change the table format on the fly. (Thanks: [Jonathan Bruno](https://github.com/brewneaux)) - * Add `--defaults-group-suffix` to the command line. This lets the user specify a group to use in the my.cnf files. (Thanks: [Iryna Cherniavska](http://github.com/j-bennet)) - * Add `--defaults-file` option to the command line. This allows specifying a `my.cnf` to use at launch. This also makes it play nice with mysql sandbox. - * Make `-p` and `--password` take the password in commandline. This makes mycli a drop in replacement for mysql. + * Add `source` command. This allows running sql statement from a file. + * Added a config option to make the warning before destructive commands optional. (Thanks: [Daniel West](https://github.com/danieljwest)) + * Add completion support for CHANGE TO and other master/slave commands. This is still preliminary and it will be enhanced in the future. + * Add custom styles to color the menus and toolbars. + * Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders](https://github.com/jonathanslenders)) + * Fix keyword completion after the `WHERE` clause. + * Add `\g` and `\G` as valid query terminators. Previously in multi-line mode ending a query with a `\G` wouldn't run the query. This is now fixed. - -- Amjith Ramanujam Sun, 09 Aug 2015 22:25:38 +0000 + -- Amjith Ramanujam Sun, 23 Aug 2015 20:14:45 +0000 From 7f3d5930c09baba1907bdfdd4457837c87739f21 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 23 Aug 2015 22:04:46 -0700 Subject: [PATCH 0006/1025] Update AUTHORS file. --- AUTHORS | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/AUTHORS b/AUTHORS index c95745d90..fa9d0f540 100644 --- a/AUTHORS +++ b/AUTHORS @@ -3,20 +3,21 @@ Many thanks to the following contributors. Contributors: ------------- - * Iryna Cherniavska - * Steve Robbins - * Daniel Black - * Thomas Roten - * Jonathan Bruno - * Heath Naylor - * Abirami P - * jbruno - * Adam Chainz - * Johannes Hoff - * The Gitter Badger - * Tyler Kuipers - * Yasuhiro Matsumoto - * bjarnagin + * Iryna Cherniavska + * Steve Robbins + * Daniel Black + * Thomas Roten + * Jonathan Bruno + * Heath Naylor + * Daniel West + * Abirami P + * jbruno + * Adam Chainz + * Johannes Hoff + * Jonathan Slenders + * Tyler Kuipers + * Yasuhiro Matsumoto + * bjarnagin Creator: -------- From 85c352142b3cf86027c753b69e36ed34e0ab607c Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 23 Aug 2015 22:05:02 -0700 Subject: [PATCH 0007/1025] Releasing version 1.4.0 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index 19b4f1d60..96e3ce8d9 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.3.0' +__version__ = '1.4.0' From 2d2fc237dc3637030f0a15a7fbb7fe59484d9c01 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 26 Aug 2015 13:20:33 -0700 Subject: [PATCH 0008/1025] Add packagecloud url to readme. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 99693d50a..54aed3a09 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ A command line client for MySQL that can do auto-completion and syntax highlight HomePage: [http://mycli.net](http://mycli.net) +Debian Packages via [PackageCloud.io](https://packagecloud.io/amjith/mycli). + ![Completion](screenshots/tables.png) ![CompletionGif](screenshots/main.gif) From f7200071cd7006eac2452bb1a8945266e5026bc8 Mon Sep 17 00:00:00 2001 From: spacewander Date: Fri, 28 Aug 2015 13:21:35 +0800 Subject: [PATCH 0009/1025] treat DESC and EXPLAIN as DESCRIBE --- mycli/packages/completion_engine.py | 3 ++- tests/test_completion_engine.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index ff1386dac..c19cf4e64 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -207,7 +207,8 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier {'type': 'function', 'schema': []}, {'type': 'keyword'}] elif (token_v.endswith('join') and token.is_keyword) or (token_v in - ('copy', 'from', 'update', 'into', 'describe', 'truncate')): + ('copy', 'from', 'update', 'into', 'describe', 'truncate', + 'desc', 'explain')): schema = (identifier and identifier.get_parent_name()) or [] # Suggest tables from either the currently-selected schema or the diff --git a/tests/test_completion_engine.py b/tests/test_completion_engine.py index e3c94a970..3f406bc7e 100644 --- a/tests/test_completion_engine.py +++ b/tests/test_completion_engine.py @@ -81,6 +81,8 @@ def test_select_suggests_cols_and_funcs(): 'COPY ', 'UPDATE ', 'DESCRIBE ', + 'DESC ', + 'EXPLAIN ', 'SELECT * FROM foo JOIN ', ]) def test_expression_suggests_tables_views_and_schemas(expression): @@ -96,6 +98,8 @@ def test_expression_suggests_tables_views_and_schemas(expression): 'COPY sch.', 'UPDATE sch.', 'DESCRIBE sch.', + 'DESC sch.', + 'EXPLAIN sch.', 'SELECT * FROM foo JOIN sch.', ]) def test_expression_suggests_qualified_tables_views_and_schemas(expression): From e9075a05a5cdfbf4bbbabd5d6e817fd959e55400 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 1 Sep 2015 22:05:13 -0700 Subject: [PATCH 0010/1025] Add TRANSACTION related keywords. --- mycli/sqlcompleter.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index acd3219aa..eaeebe0b1 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -18,13 +18,13 @@ class SQLCompleter(Completer): keywords = ['ACCESS', 'ADD', 'ALL', 'ALTER TABLE', 'AND', 'ANY', 'AS', - 'ASC', 'AUDIT', 'BEFORE', 'BETWEEN', 'BINARY', 'BY', 'CASE', - 'CHANGE MASTER TO', 'CHAR', 'CHECK', 'CLUSTER', 'COLUMN', - 'COMMENT', 'COMPRESS', 'CONNECT', 'COPY', 'CREATE', 'CURRENT', - 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', 'DELETE FROM', + 'ASC', 'AUDIT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', + 'CASE', 'CHANGE MASTER TO', 'CHAR', 'CHECK', 'CLUSTER', 'COLUMN', + 'COMMENT', 'COMPRESS', 'COMMIT', 'CONNECT', 'COPY', 'CREATE', + 'CURRENT', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', 'DELETE FROM', 'DELIMITER', 'DESC', 'DESCRIBE', 'DISTINCT', 'DROP', 'ELSE', - 'ENCODING', 'ESCAPE', 'EXCLUSIVE', 'EXISTS', 'EXTENSION', 'FILE', - 'FLOAT', 'FOR', 'FORMAT', 'FORCE_QUOTE', 'FORCE_NOT_NULL', + 'ENCODING', 'END', 'ESCAPE', 'EXCLUSIVE', 'EXISTS', 'EXTENSION', + 'FILE', 'FLOAT', 'FOR', 'FORMAT', 'FORCE_QUOTE', 'FORCE_NOT_NULL', 'FREEZE', 'FROM', 'FULL', 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HEADER', 'HOST', 'IDENTIFIED', 'IMMEDIATE', 'IN', 'INCREMENT', 'INDEX', 'INITIAL', 'INSERT INTO', 'INTEGER', @@ -35,12 +35,13 @@ class SQLCompleter(Completer): 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', 'PCTFREE', 'PORT', 'PRIMARY', 'PRIOR', 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'QUOTE', 'RAW', 'RENAME', 'REPAIR', 'RESOURCE', 'RESET', 'REVOKE', 'RIGHT', - 'ROW', 'ROWID', 'ROWNUM', 'ROWS', 'SELECT', 'SESSION', 'SET', - 'SHARE', 'SHOW', 'SIZE', 'SLAVE', 'SLAVES', 'SMALLINT', 'START', - 'STOP', 'SUCCESSFUL', 'SYNONYM', 'SYSDATE', 'TABLE', 'TEMPLATE', - 'THEN', 'TO', 'TRIGGER', 'TRUNCATE', 'UID', 'UNION', 'UNIQUE', - 'UPDATE', 'USE', 'USER', 'USING', 'VALIDATE', 'VALUES', 'VARCHAR', - 'VARCHAR2', 'VIEW', 'WHEN', 'WHENEVER', 'WHERE', 'WITH'] + 'ROLLBACK', 'ROW', 'ROWID', 'ROWNUM', 'ROWS', 'SELECT', 'SESSION', + 'SET', 'SHARE', 'SHOW', 'SIZE', 'SLAVE', 'SLAVES', 'SMALLINT', + 'START', 'STOP', 'SUCCESSFUL', 'SYNONYM', 'SYSDATE', 'TABLE', + 'TEMPLATE', 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', + 'UID', 'UNION', 'UNIQUE', 'UPDATE', 'USE', 'USER', 'USING', + 'VALIDATE', 'VALUES', 'VARCHAR', 'VARCHAR2', 'VIEW', 'WHEN', + 'WHENEVER', 'WHERE', 'WITH'] functions = ['AVG', 'COUNT', 'DISTINCT', 'FIRST', 'FORMAT', 'LAST', 'LCASE', 'LEN', 'MAX', 'MIN', 'MID', 'NOW', 'ROUND', 'SUM', 'TOP', From 566d0927b2254a425c26f04efd495258cfdb4bff Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 1 Sep 2015 22:08:52 -0700 Subject: [PATCH 0011/1025] Disable click warning. --- mycli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 03b1855f3..db88a5199 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -39,6 +39,8 @@ from .lexer import MyCliLexer from .__init__ import __version__ +click.disable_unicode_literals_warning = True + try: from urlparse import urlparse except ImportError: From 03da5f6b9b0c8cd2fd4fb76d0a363a2d4004af2a Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 10 Sep 2015 11:24:58 -0700 Subject: [PATCH 0012/1025] Add debug logging for charset. --- mycli/sqlexecute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 04c69a9fa..2a3b09534 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -51,7 +51,8 @@ def connect(self, database=None, user=None, password=None, host=None, '\tuser: %r' '\thost: %r' '\tport: %r' - '\tsocket: %r', database, user, host, port, socket) + '\tsocket: %r' + '\tcharset: %r', database, user, host, port, socket, charset) conn = pymysql.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, use_unicode=True, charset=charset, autocommit=True, From 37b0a41f89b214959b98973ca45d8cdf76b7a0f6 Mon Sep 17 00:00:00 2001 From: shoma Date: Mon, 14 Sep 2015 10:15:04 +0900 Subject: [PATCH 0013/1025] Add special command to describe Add a special command `\d` to describe table info with completion for table name. --- mycli/packages/completion_engine.py | 7 +++++++ mycli/packages/special/dbcommands.py | 16 +++++++++++++++- tests/test_dbspecial.py | 8 ++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index c19cf4e64..6d0e002a9 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -102,6 +102,13 @@ def suggest_special(text): if cmd in ['\\f', '\\fs', '\\fd']: return [{'type': 'favoritequery'}] + if cmd in ['\\d']: + return [ + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}, + ] + return [{'type': 'keyword'}, {'type': 'special'}] def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier): diff --git a/mycli/packages/special/dbcommands.py b/mycli/packages/special/dbcommands.py index 140ecda1b..f01c91e35 100644 --- a/mycli/packages/special/dbcommands.py +++ b/mycli/packages/special/dbcommands.py @@ -1,5 +1,5 @@ import logging -from .main import special_command, RAW_QUERY +from .main import special_command, RAW_QUERY, PARSED_QUERY log = logging.getLogger(__name__) @@ -24,3 +24,17 @@ def list_databases(cur, **_): return [(None, cur, headers, '')] else: return [(None, None, None, '')] + + +@special_command('\\d', '\\d [table]', 'Describe table.', case_sensitive=True) +def describe_table(cur, arg=None, arg_type=PARSED_QUERY, **_): + if arg is None: + return list_tables(cur) + query = 'SHOW FIELDS FROM {0}'.format(arg) + log.debug(query) + cur.execute(query) + if cur.description: + headers = [x[0] for x in cur.description] + return [(None, cur, headers, '')] + else: + return [(None, None, None, '')] diff --git a/tests/test_dbspecial.py b/tests/test_dbspecial.py index 5f71e5ccd..6671c1758 100644 --- a/tests/test_dbspecial.py +++ b/tests/test_dbspecial.py @@ -5,3 +5,11 @@ def test_u_suggests_databases(): suggestions = suggest_type('\\u ', '\\u ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'database'}]) + + +def test_describe_table(): + suggestions = suggest_type('\\d', '\\d ') + assert sorted_dicts(suggestions) == sorted_dicts([ + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) From 0f9676d9838a62a0705ac8565814060f4db1a52e Mon Sep 17 00:00:00 2001 From: shoma Date: Mon, 14 Sep 2015 14:10:52 +0900 Subject: [PATCH 0014/1025] Overloading \dt for describe table. Discussion for compatibility, see https://github.com/dbcli/mycli/pull/150 --- mycli/packages/completion_engine.py | 2 +- mycli/packages/special/dbcommands.py | 22 ++++++---------------- tests/test_dbspecial.py | 2 +- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 6d0e002a9..0c480316b 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -102,7 +102,7 @@ def suggest_special(text): if cmd in ['\\f', '\\fs', '\\fd']: return [{'type': 'favoritequery'}] - if cmd in ['\\d']: + if cmd in ['\\dt']: return [ {'type': 'table', 'schema': []}, {'type': 'view', 'schema': []}, diff --git a/mycli/packages/special/dbcommands.py b/mycli/packages/special/dbcommands.py index f01c91e35..5eb1ef1fe 100644 --- a/mycli/packages/special/dbcommands.py +++ b/mycli/packages/special/dbcommands.py @@ -3,9 +3,12 @@ log = logging.getLogger(__name__) -@special_command('\\dt', '\\dt', 'List tables.', arg_type=RAW_QUERY, case_sensitive=True) -def list_tables(cur, **_): - query = 'SHOW TABLES' +@special_command('\\dt', '\\dt', 'List or describe tables.', arg_type=PARSED_QUERY, case_sensitive=True) +def list_tables(cur, arg=None, arg_type=PARSED_QUERY): + if arg: + query = 'SHOW FIELDS FROM {0}'.format(arg) + else: + query = 'SHOW TABLES' log.debug(query) cur.execute(query) if cur.description: @@ -25,16 +28,3 @@ def list_databases(cur, **_): else: return [(None, None, None, '')] - -@special_command('\\d', '\\d [table]', 'Describe table.', case_sensitive=True) -def describe_table(cur, arg=None, arg_type=PARSED_QUERY, **_): - if arg is None: - return list_tables(cur) - query = 'SHOW FIELDS FROM {0}'.format(arg) - log.debug(query) - cur.execute(query) - if cur.description: - headers = [x[0] for x in cur.description] - return [(None, cur, headers, '')] - else: - return [(None, None, None, '')] diff --git a/tests/test_dbspecial.py b/tests/test_dbspecial.py index 6671c1758..f1ee49e51 100644 --- a/tests/test_dbspecial.py +++ b/tests/test_dbspecial.py @@ -8,7 +8,7 @@ def test_u_suggests_databases(): def test_describe_table(): - suggestions = suggest_type('\\d', '\\d ') + suggestions = suggest_type('\\dt', '\\dt ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'table', 'schema': []}, {'type': 'view', 'schema': []}, From 58b60b342947b7378f4bbf5a57ae7915936157bb Mon Sep 17 00:00:00 2001 From: shoma Date: Mon, 14 Sep 2015 18:33:50 +0900 Subject: [PATCH 0015/1025] Add setting for doctest with py.test --- conftest.py | 2 ++ tox.ini | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..cee5a17e4 --- /dev/null +++ b/conftest.py @@ -0,0 +1,2 @@ +# https://pytest.org/latest/example/pythoncollection.html +collect_ignore = ["setup.py"] diff --git a/tox.ini b/tox.ini index b97055f75..96d168014 100644 --- a/tox.ini +++ b/tox.ini @@ -3,4 +3,4 @@ envlist = py26, py27, py33, py34 [testenv] deps = pytest mock -commands = py.test +commands = py.test --doctest-modules --doctest-ignore-import-errors From dfd07f5a8df4fe0f4389690c1f378aabf0cbe99b Mon Sep 17 00:00:00 2001 From: shoma Date: Mon, 14 Sep 2015 18:36:28 +0900 Subject: [PATCH 0016/1025] Fixed doctest for parseutils.last_word. --- mycli/packages/parseutils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mycli/packages/parseutils.py b/mycli/packages/parseutils.py index 2796194e3..3cbf4a05d 100644 --- a/mycli/packages/parseutils.py +++ b/mycli/packages/parseutils.py @@ -7,10 +7,10 @@ cleanup_regex = { # This matches only alphanumerics and underscores. 'alphanum_underscore': re.compile(r'(\w+)$'), - # This matches everything except spaces, parens, and comma - 'many_punctuations': re.compile(r'([^(),\s]+)$'), - # This matches everything except spaces, parens, comma, and period - 'most_punctuations': re.compile(r'([^\.(),\s]+)$'), + # This matches everything except spaces, parens, colon, and comma + 'many_punctuations': re.compile(r'([^():,\s]+)$'), + # This matches everything except spaces, parens, colon, comma, and period + 'most_punctuations': re.compile(r'([^\.():,\s]+)$'), # This matches everything except a space. 'all_punctuations': re.compile('([^\s]+)$'), } @@ -37,12 +37,14 @@ def last_word(text, include='alphanum_underscore'): '' >>> last_word('bac $def') 'def' - >>> last_word('bac $def', True) + >>> last_word('bac $def', include='most_punctuations') '$def' - >>> last_word('bac \def', True) + >>> last_word('bac \def', include='most_punctuations') '\\\\def' - >>> last_word('bac \def;', True) + >>> last_word('bac \def;', include='most_punctuations') '\\\\def;' + >>> last_word('bac::def', include='most_punctuations') + 'def' """ if not text: # Empty string From cec43da19d05db67009b7836073698e32d43e7d3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 16 Sep 2015 15:32:26 -0500 Subject: [PATCH 0017/1025] Adds ability to read multiple config sections. * MyCli.read_my_cnf_files now adds default_suffix to any sections in *sections* list, not just 'client'. * MyCli.read_my_cnf_files.get loops through sections in *cnf*, so that sections that appear later in a cnf file get precedence. This is how mysql loads .mylogin.cnf. --- mycli/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index db88a5199..b515d1768 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -208,12 +208,12 @@ def read_my_cnf_files(self, files, keys): sections = ['client'] if self.defaults_suffix: - sections.append('client{0}'.format(self.defaults_suffix)) + sections.extend([sect + self.defaults_suffix for sect in sections]) def get(key): result = None - for sect in sections: - if sect in cnf and key in cnf[sect]: + for sect in cnf: + if sect in sections and key in cnf[sect]: result = cnf[sect][key] return result From 0406509d2729be0d8d668c5c6dcd956163424ef3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 16 Sep 2015 15:36:25 -0500 Subject: [PATCH 0018/1025] Adds functions to locate and read .mylogin.cnf. * Adds mycli.config.get_mylogin_cnf_path(). Returns the path to the .mylogin.cnf file or None if the file doesn't exist. Looks for %APPDATA%/MySQL directory on Windows and user home directory on other systems. * Adds mycli.config.get_mylogin_cnf_plaintext(). Uses mysql_config_editor algorithm to decrypt and read .mylogin.cnf file. Returns a wrapper (io.TextIOWrapper) around an io.BytesIO buffer. --- mycli/config.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 2 files changed, 62 insertions(+) diff --git a/mycli/config.py b/mycli/config.py index 6f5ff21a4..3bacbab05 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -1,6 +1,10 @@ import shutil +from io import BytesIO, TextIOWrapper +import os from os.path import expanduser, exists +import struct from configobj import ConfigObj +from Crypto.Cipher import AES def load_config(usr_cfg, def_cfg=None): cfg = ConfigObj() @@ -16,3 +20,60 @@ def write_default_config(source, destination, overwrite=False): return shutil.copyfile(source, destination) + +def get_mylogin_cnf_path(): + """Return the path to the .mylogin.cnf file or None if doesn't exist.""" + app_data = os.getenv('APPDATA') + if app_data is None: + mylogin_config_dir = os.path.expanduser('~') + else: + mylogin_config_dir = os.path.join(app_data, 'MySQL') + + mylogin_config_dir = os.path.abspath(mylogin_config_dir) + mylogin_config_path = os.path.join(mylogin_config_dir, '.mylogin.cnf') + + return mylogin_config_path if exists(mylogin_config_path) else None + +def get_mylogin_cnf_plaintext(file_name): + """Return the contents of .mylogin.cnf as a buffered text stream.""" + + # Number of bytes used to store the length of ciphertext. + MAX_CIPHER_STORE_LEN = 4 + LOGIN_KEY_LEN = 20 + + with open(file_name, 'rb') as f: + # Move past the unused buffer. + f.seek(4) + + # Read the login key, a sequence of random non-printable ASCII. + key = f.read(LOGIN_KEY_LEN) + + # Generate the real AES key + rkey = [0] * 16 + for i in range(LOGIN_KEY_LEN): + rkey[i % 16] ^= ord(key[i:i+1]) + rkey = struct.pack('16B', *rkey) + + # Create a cipher object using the key. + aes_cipher = AES.new(rkey, AES.MODE_ECB) + + # Create a bytes buffer to hold the plaintext. + plaintext = BytesIO() + + while True: + # Read the length of the ciphertext. + len_buf = f.read(MAX_CIPHER_STORE_LEN) + if len(len_buf) < MAX_CIPHER_STORE_LEN: + break + cipher_len, = struct.unpack("= 0.6.6', 'sqlparse >= 0.1.16', 'configobj >= 5.0.6', + 'pycrypto >= 2.6.1', ], entry_points=''' [console_scripts] From 016076b4e657fb1553c37ba28ba67470e5c57f3d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 16 Sep 2015 15:54:47 -0500 Subject: [PATCH 0019/1025] Adds --login-path argument support. * Adds --login-path argument to mycli. .mylogin.cnf is always read, even if --defaults-file is specified (per mysql_config_editor documentation). .mylogin.cnf is added to the cnf_files list last, so that it has precedence. --- mycli/main.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index b515d1768..00c7e2eb3 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -33,7 +33,8 @@ from .clistyle import style_factory from .sqlexecute import SQLExecute from .clibuffer import CLIBuffer -from .config import write_default_config, load_config +from .config import (write_default_config, load_config, get_mylogin_cnf_path, + get_mylogin_cnf_plaintext) from .key_bindings import mycli_bindings from .encodingutils import utf8tounicode from .lexer import MyCliLexer @@ -64,14 +65,16 @@ class MyCli(object): '/etc/my.cnf', '/etc/mysql/my.cnf', '/usr/local/etc/my.cnf', - '~/.my.cnf' + os.path.expanduser('~/.my.cnf') ] def __init__(self, sqlexecute=None, prompt=None, - logfile=None, defaults_suffix=None, defaults_file=None): + logfile=None, defaults_suffix=None, defaults_file=None, + login_path=None): self.sqlexecute = sqlexecute self.logfile = logfile self.defaults_suffix = defaults_suffix + self.login_path = login_path # self.cnf_files is a class variable that stores the list of mysql # config files to read in at launch. @@ -111,6 +114,14 @@ def __init__(self, sqlexecute=None, prompt=None, # Register custom special commands. self.register_special_commands() + # Load .mylogin.cnf if it exists. + mylogin_cnf_path = get_mylogin_cnf_path() + if mylogin_cnf_path: + mylogin_cnf = get_mylogin_cnf_plaintext(mylogin_cnf_path) + + # .mylogin.cnf gets read last, even if defaults_file is specified. + self.cnf_files.append(mylogin_cnf) + def register_special_commands(self): special.register_special_command(self.change_db, 'use', '\\u', 'Change to a new database.', aliases=('\\u',)) @@ -198,8 +209,7 @@ def read_my_cnf_files(self, files, keys): cnf = ConfigObj() for _file in files: try: - cnf.merge(ConfigObj(os.path.expanduser(_file), - interpolation=False)) + cnf.merge(ConfigObj(_file, interpolation=False)) except ConfigObjError as e: self.logger.error('Error parsing %r.', _file) self.logger.error('Recovering partially parsed config values.') @@ -207,6 +217,9 @@ def read_my_cnf_files(self, files, keys): pass sections = ['client'] + if self.login_path and self.login_path != 'client': + sections.append(self.login_path) + if self.defaults_suffix: sections.extend([sect + self.defaults_suffix for sect in sections]) @@ -564,16 +577,19 @@ def get_prompt(self, string): help='Read config group with the specified suffix.') @click.option('--defaults-file', type=click.Path(), help='Only read default options from the given file') +@click.option('--login-path', type=str, + help='Read this path from the login file.') @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, - version, prompt, logfile, defaults_group_suffix, defaults_file): + version, prompt, logfile, defaults_group_suffix, defaults_file, + login_path): if version: print('Version:', __version__) sys.exit(0) mycli = MyCli(prompt=prompt, logfile=logfile, defaults_suffix=defaults_group_suffix, - defaults_file=defaults_file) + defaults_file=defaults_file, login_path=login_path) # Choose which ever one has a valid value. database = database or dbname From d221dd5fe1d80d7fddb2e81f903c974654d61797 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 16 Sep 2015 16:00:41 -0500 Subject: [PATCH 0020/1025] Adds missing arguments/options to README.md. --- README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 54aed3a09..9d4e8dbdb 100644 --- a/README.md +++ b/README.md @@ -39,18 +39,21 @@ Check the [detailed install instructions](#Detailed-Install-Instructions) for de Usage: mycli [OPTIONS] [DATABASE] Options: - -h, --host TEXT Host address of the database. - -P, --port TEXT Port number to use for connection. Honors - $MYSQL_TCP_PORT - -u, --user TEXT User name to connect to the database. - -S, --socket TEXT The socket file to use for connection. - -p, --password Force password prompt. - --pass TEXT Password to connect to the database - -v, --version Version of mycli. - -D, --database TEXT Database to use. - -R, --prompt TEXT Prompt format (Default: "\t \u@\h:\d> ") - -l, --logfile FILENAME Log every query and its results to a file. - --help Show this message and exit. + -h, --host TEXT Host address of the database. + -P, --port TEXT Port number to use for connection. Honors + $MYSQL_TCP_PORT + -u, --user TEXT User name to connect to the database. + -S, --socket TEXT The socket file to use for connection. + -p, --password Force password prompt. + --pass TEXT Password to connect to the database + -v, --version Version of mycli. + -D, --database TEXT Database to use. + -R, --prompt TEXT Prompt format (Default: "\t \u@\h:\d> ") + -l, --logfile FILENAME Log every query and its results to a file. + --defaults-group-suffix TEXT Read config group with the specified suffix. + --defaults-file PATH Only read default options from the given file + --login-path TEXT Read this path from the login file. + --help Show this message and exit. ### Examples From d2ad097e08b8c5e9d318968f0a6f859f03f7c07a Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 16 Sep 2015 21:16:15 -0700 Subject: [PATCH 0021/1025] Change \dt syntax to add an optional table name. --- mycli/packages/special/dbcommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/dbcommands.py b/mycli/packages/special/dbcommands.py index 5eb1ef1fe..aa6a7694f 100644 --- a/mycli/packages/special/dbcommands.py +++ b/mycli/packages/special/dbcommands.py @@ -3,7 +3,7 @@ log = logging.getLogger(__name__) -@special_command('\\dt', '\\dt', 'List or describe tables.', arg_type=PARSED_QUERY, case_sensitive=True) +@special_command('\\dt', '\\dt [table]', 'List or describe tables.', arg_type=PARSED_QUERY, case_sensitive=True) def list_tables(cur, arg=None, arg_type=PARSED_QUERY): if arg: query = 'SHOW FIELDS FROM {0}'.format(arg) From f7e3ecb0c4f70746dc4fc0e1c125d118d1e5651b Mon Sep 17 00:00:00 2001 From: shoma Date: Thu, 17 Sep 2015 16:12:55 +0900 Subject: [PATCH 0022/1025] Fix print_function syntax for Py3. --- mycli/packages/counter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/packages/counter.py b/mycli/packages/counter.py index 2f3615339..e74db3116 100644 --- a/mycli/packages/counter.py +++ b/mycli/packages/counter.py @@ -1,3 +1,4 @@ +from __future__ import print_function from operator import itemgetter from heapq import nlargest from itertools import repeat, ifilter @@ -186,4 +187,4 @@ def __and__(self, other): if __name__ == '__main__': import doctest - print doctest.testmod() + print(doctest.testmod()) From 688a600b36049820e75142c7cddfe984a595cdaa Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 17 Sep 2015 20:25:18 -0500 Subject: [PATCH 0023/1025] Splits up login path functions. --- mycli/config.py | 97 ++++++++++++++++++++++++++++++++++--------------- mycli/main.py | 5 ++- 2 files changed, 70 insertions(+), 32 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 3bacbab05..18dd8ae85 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -1,11 +1,14 @@ import shutil from io import BytesIO, TextIOWrapper +import logging import os from os.path import expanduser, exists import struct from configobj import ConfigObj from Crypto.Cipher import AES +logger = logging.getLogger(__name__) + def load_config(usr_cfg, def_cfg=None): cfg = ConfigObj() cfg.merge(ConfigObj(def_cfg, interpolation=False)) @@ -34,46 +37,80 @@ def get_mylogin_cnf_path(): return mylogin_config_path if exists(mylogin_config_path) else None -def get_mylogin_cnf_plaintext(file_name): - """Return the contents of .mylogin.cnf as a buffered text stream.""" +def open_mylogin_cnf(name): + """Open a readable version of .mylogin.cnf. + + Returns the file contents as a TextIOWrapper object. + + :param str name: The pathname of the file to be opened. + :return: the login path file or None + """ + + try: + with open(name, 'rb') as f: + plaintext = read_and_decrypt_mylogin_cnf(f) + except (OSError, IOError): + logger.error("Error: Unable to open '{0}'".format(name)) + return None + + if not isinstance(plaintext, BytesIO): + logger.error("Error: Unable to decrypt '{0}'".format(name)) + return None + + return TextIOWrapper(plaintext) + +def read_and_decrypt_mylogin_cnf(f): + """Read and decrypt the contents of .mylogin.cnf. + + This decryption algorithm mimics the code in MySQL's + mysql_config_editor.cc. + + The login key is 20-bytes of random non-printable ASCII. + It is written to the actual login path file. It is used + to generate the real key used in the AES cipher. + + :param f: an I/O object opened in binary mode + :return: the decrypted login path file + :rtype: io.BytesIO + """ # Number of bytes used to store the length of ciphertext. MAX_CIPHER_STORE_LEN = 4 + LOGIN_KEY_LEN = 20 - with open(file_name, 'rb') as f: - # Move past the unused buffer. - f.seek(4) + # Move past the unused buffer. + f.seek(4) - # Read the login key, a sequence of random non-printable ASCII. - key = f.read(LOGIN_KEY_LEN) + # Read the login key. + key = f.read(LOGIN_KEY_LEN) - # Generate the real AES key - rkey = [0] * 16 - for i in range(LOGIN_KEY_LEN): - rkey[i % 16] ^= ord(key[i:i+1]) - rkey = struct.pack('16B', *rkey) + # Generate the real key. + rkey = [0] * 16 + for i in range(LOGIN_KEY_LEN): + rkey[i % 16] ^= ord(key[i:i+1]) + rkey = struct.pack('16B', *rkey) - # Create a cipher object using the key. - aes_cipher = AES.new(rkey, AES.MODE_ECB) + # Create a cipher object using the key. + aes_cipher = AES.new(rkey, AES.MODE_ECB) - # Create a bytes buffer to hold the plaintext. - plaintext = BytesIO() + # Create a bytes buffer to hold the plaintext. + plaintext = BytesIO() - while True: - # Read the length of the ciphertext. - len_buf = f.read(MAX_CIPHER_STORE_LEN) - if len(len_buf) < MAX_CIPHER_STORE_LEN: - break - cipher_len, = struct.unpack(" Date: Thu, 17 Sep 2015 21:56:56 -0500 Subject: [PATCH 0024/1025] Adds error handling and logging messages to login path file decryption. --- mycli/config.py | 61 +++++++++++++++++++++++++++++++++++++------------ mycli/main.py | 1 + 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 18dd8ae85..4d0c11173 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -28,20 +28,23 @@ def get_mylogin_cnf_path(): """Return the path to the .mylogin.cnf file or None if doesn't exist.""" app_data = os.getenv('APPDATA') if app_data is None: - mylogin_config_dir = os.path.expanduser('~') + mylogin_cnf_dir = os.path.expanduser('~') else: - mylogin_config_dir = os.path.join(app_data, 'MySQL') + mylogin_cnf_dir = os.path.join(app_data, 'MySQL') - mylogin_config_dir = os.path.abspath(mylogin_config_dir) - mylogin_config_path = os.path.join(mylogin_config_dir, '.mylogin.cnf') + mylogin_cnf_dir = os.path.abspath(mylogin_cnf_dir) + mylogin_cnf_path = os.path.join(mylogin_cnf_dir, '.mylogin.cnf') - return mylogin_config_path if exists(mylogin_config_path) else None + if exists(mylogin_cnf_path): + logger.debug("Found login path file at '{0}'".format(mylogin_cnf_path)) + return mylogin_cnf_path + return None def open_mylogin_cnf(name): """Open a readable version of .mylogin.cnf. Returns the file contents as a TextIOWrapper object. - + :param str name: The pathname of the file to be opened. :return: the login path file or None """ @@ -50,11 +53,11 @@ def open_mylogin_cnf(name): with open(name, 'rb') as f: plaintext = read_and_decrypt_mylogin_cnf(f) except (OSError, IOError): - logger.error("Error: Unable to open '{0}'".format(name)) + logger.error('Unable to open login path file.') return None if not isinstance(plaintext, BytesIO): - logger.error("Error: Unable to decrypt '{0}'".format(name)) + logger.error('Unable to decrypt login path file.') return None return TextIOWrapper(plaintext) @@ -71,7 +74,7 @@ def read_and_decrypt_mylogin_cnf(f): :param f: an I/O object opened in binary mode :return: the decrypted login path file - :rtype: io.BytesIO + :rtype: io.BytesIO or None """ # Number of bytes used to store the length of ciphertext. @@ -80,7 +83,12 @@ def read_and_decrypt_mylogin_cnf(f): LOGIN_KEY_LEN = 20 # Move past the unused buffer. - f.seek(4) + buf = f.read(4) + + if not buf or len(buf) != 4: + # File is blank or incomplete. + logger.error('Login path file is blank or incomplete.') + return None # Read the login key. key = f.read(LOGIN_KEY_LEN) @@ -88,7 +96,12 @@ def read_and_decrypt_mylogin_cnf(f): # Generate the real key. rkey = [0] * 16 for i in range(LOGIN_KEY_LEN): - rkey[i % 16] ^= ord(key[i:i+1]) + try: + rkey[i % 16] ^= ord(key[i:i+1]) + except TypeError: + # ord() was unable to get the value of the byte. + logger.error('Unable to generate login path AES key.') + return None rkey = struct.pack('16B', *rkey) # Create a cipher object using the key. @@ -106,11 +119,29 @@ def read_and_decrypt_mylogin_cnf(f): # Read cipher_len bytes from the file and decrypt. cipher = f.read(cipher_len) - plain = aes_cipher.decrypt(cipher) - - # Get rid of pad - plain = plain[:-ord(plain[-1:])] + pplain = aes_cipher.decrypt(cipher) + + try: + # Determine pad length. + pad_len = ord(pplain[-1:]) + except TypeError: + # ord() was unable to get the value of the byte. + logger.warning('Unable to remove pad.') + continue + + if pad_len > len(pplain) or len(set(pplain[-pad_len:])) != 1: + # Pad length should be less than or equal to the length of the + # plaintext. The pad should have a single unqiue byte. + logger.warning('Invalid pad found in login path file.') + continue + + # Get rid of pad. + plain = pplain[:-pad_len] plaintext.write(plain) + if plaintext.tell() == 0: + logger.error('No data successfully decrypted from login path file.') + return None + plaintext.seek(0) return plaintext diff --git a/mycli/main.py b/mycli/main.py index 8cb8b5e7e..3ec76f0b1 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -117,6 +117,7 @@ def __init__(self, sqlexecute=None, prompt=None, # Load .mylogin.cnf if it exists. mylogin_cnf_path = get_mylogin_cnf_path() if mylogin_cnf_path: + self.logger.debug("Found login path file: '{0}'".format(mylogin_cnf_path)) mylogin_cnf = open_mylogin_cnf(mylogin_cnf_path) if mylogin_cnf_path and mylogin_cnf: From 10907a9feb3afeff738c4a26280c9e934a98b8b4 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 17 Sep 2015 21:58:14 -0500 Subject: [PATCH 0025/1025] Removes redundant logging message. --- mycli/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 3ec76f0b1..8cb8b5e7e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -117,7 +117,6 @@ def __init__(self, sqlexecute=None, prompt=None, # Load .mylogin.cnf if it exists. mylogin_cnf_path = get_mylogin_cnf_path() if mylogin_cnf_path: - self.logger.debug("Found login path file: '{0}'".format(mylogin_cnf_path)) mylogin_cnf = open_mylogin_cnf(mylogin_cnf_path) if mylogin_cnf_path and mylogin_cnf: From 384a4dc8b6dd659631c415a92e4926f679b66b60 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 17 Sep 2015 22:02:49 -0500 Subject: [PATCH 0026/1025] Adds user-facing error message for login path. --- mycli/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 8cb8b5e7e..50f3088c2 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -122,6 +122,9 @@ def __init__(self, sqlexecute=None, prompt=None, if mylogin_cnf_path and mylogin_cnf: # .mylogin.cnf gets read last, even if defaults_file is specified. self.cnf_files.append(mylogin_cnf) + elif mylogin_cnf_path and not mylogin_cnf: + # There was an error reading the login path file. + print('Error: Unable to read login path file.') def register_special_commands(self): special.register_special_command(self.change_db, 'use', From df2eaf2b7edef3f9fe0d82d3e70abcff367ed0f5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 17 Sep 2015 22:12:20 -0500 Subject: [PATCH 0027/1025] Changes/removes redundant logs and comments. --- mycli/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 4d0c11173..f22822435 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -57,7 +57,7 @@ def open_mylogin_cnf(name): return None if not isinstance(plaintext, BytesIO): - logger.error('Unable to decrypt login path file.') + logger.error('Unable to read login path file.') return None return TextIOWrapper(plaintext) @@ -86,7 +86,6 @@ def read_and_decrypt_mylogin_cnf(f): buf = f.read(4) if not buf or len(buf) != 4: - # File is blank or incomplete. logger.error('Login path file is blank or incomplete.') return None From 3153fc9348ae6e2b38aea550970e1cbc5df7847b Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 17 Sep 2015 21:56:50 -0700 Subject: [PATCH 0028/1025] Use io module to open files. --- mycli/main.py | 2 +- mycli/packages/special/iocommands.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index db88a5199..bbd380778 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -9,7 +9,7 @@ from time import time from datetime import datetime from random import choice -from codecs import open +from io import open import click import sqlparse diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 68fac4428..a473ecd92 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -1,7 +1,7 @@ import os import re import logging -from codecs import open +from io import open import click import sqlparse From 43fcb4ac70b10cf3a9cb60884dc3d6004cc4784a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 18 Sep 2015 19:10:04 -0500 Subject: [PATCH 0029/1025] Adds unit tests for login path decryption. --- tests/mylogin.cnf | Bin 0 -> 156 bytes tests/test_login_path.py | 72 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 tests/mylogin.cnf create mode 100644 tests/test_login_path.py diff --git a/tests/mylogin.cnf b/tests/mylogin.cnf new file mode 100644 index 0000000000000000000000000000000000000000..1363cc37bd99ac036940096f1f853f86009e56cd GIT binary patch literal 156 zcmZQzU|?Y3la~_~=M)ha73G#@VB})s<`)wHiZ?c$jbh#J^{JLk(?Paj20KWMXV0vL zd#h(I2>R+a|Gyjo06rp_ly5UiI*TX zA}ii+=iac$Chzu_#KOANs9MRwy}g$f@8mmN&0TTgSOmyyTNcR;`&YD`nR8$A$f Date: Sat, 19 Sep 2015 09:40:04 -0500 Subject: [PATCH 0030/1025] Fixes login path file name for tests. --- tests/test_login_path.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_login_path.py b/tests/test_login_path.py index e9666044a..cfdfd276c 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -1,9 +1,12 @@ """Unit tests for mycli.config login path decryption.""" from io import BytesIO, TextIOWrapper +import os import struct from mycli.config import open_mylogin_cnf, read_and_decrypt_mylogin_cnf +LOGIN_PATH_FILE = os.path.join(os.path.dirname(__file__), 'mylogin.cnf') + def open_bmylogin_cnf(name): """Open contents of *name* in a BytesIO buffer.""" @@ -15,7 +18,7 @@ def open_bmylogin_cnf(name): def test_read_mylogin_cnf(): """Tests that a login path file can be read and decrypted.""" - mylogin_cnf = open_mylogin_cnf('./mylogin.cnf') + mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) assert isinstance(mylogin_cnf, TextIOWrapper) @@ -32,7 +35,7 @@ def test_decrypt_blank_mylogin_cnf(): def test_corrupted_login_key(): """Test that a corrupted login path key is handled correctly.""" - buf = open_bmylogin_cnf('./mylogin.cnf') + buf = open_bmylogin_cnf(LOGIN_PATH_FILE) # Skip past the unused bytes buf.seek(4) @@ -48,7 +51,7 @@ def test_corrupted_login_key(): def test_corrupted_pad(): """Tests that a login path file with a corrupted pad is partially read.""" - buf = open_bmylogin_cnf('./mylogin.cnf') + buf = open_bmylogin_cnf(LOGIN_PATH_FILE) # Skip past the login key buf.seek(24) From 369525075f8dd04c2378de478aa46e10542261ca Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 19 Sep 2015 10:46:02 -0400 Subject: [PATCH 0031/1025] Makes test login path file an absolute path. --- tests/test_login_path.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_login_path.py b/tests/test_login_path.py index cfdfd276c..3f02b4df4 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -5,7 +5,8 @@ from mycli.config import open_mylogin_cnf, read_and_decrypt_mylogin_cnf -LOGIN_PATH_FILE = os.path.join(os.path.dirname(__file__), 'mylogin.cnf') +LOGIN_PATH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), + 'mylogin.cnf')) def open_bmylogin_cnf(name): From 1ff8775c07fc63c2f706ad437a36eaa1a9113e53 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 23 Sep 2015 20:37:38 -0500 Subject: [PATCH 0032/1025] Adds custom versions of pymysql's Cursor, Connection, and connect(). --- mycli/packages/connection.py | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 mycli/packages/connection.py diff --git a/mycli/packages/connection.py b/mycli/packages/connection.py new file mode 100644 index 000000000..8edf05c08 --- /dev/null +++ b/mycli/packages/connection.py @@ -0,0 +1,45 @@ +"""Connection and cursor wrappers around PyMySQL. + +This module effectively backports PyMySQL functionality and error handling +so that mycli will support Debian's python-pymysql version (0.6.2). +""" + +import pymysql + +Cursor = pymysql.cursors.Cursor +connect = pymysql.connect + + +if pymysql.VERSION[1] == 6 and pymysql.VERSION[2] < 5: + class Cursor(pymysql.cursors.Cursor): + """Makes pre-0.6.5 Cursor a context manager.""" + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + del exc_info + self.close() + + +if pymysql.VERSION[1] == 6 and pymysql.VERSION[2] < 3: + class Connection(pymysql.connections.Connection): + """Adds error handling to pre-0.6.3 Connection.""" + + def __del__(self): + if self.socket: + try: + self.socket.close() + except: + pass + self.socket = None + self._rfile = None + + def connect(*args, **kwargs): + """Makes connect() use our custom Connection class. + + See pymysql.connections.Connection.__init__() for more information + about calling this function. + """ + + return Connection(*args, **kwargs) From 3fef64c352d842a20b7b614c703161fb36b4af6f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 23 Sep 2015 20:38:23 -0500 Subject: [PATCH 0033/1025] Updates calls to pymysql.connect() to use custom connect() function. --- mycli/sqlexecute.py | 7 ++++--- tests/utils.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 2a3b09534..53918e6c1 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -1,7 +1,7 @@ import logging import pymysql import sqlparse -from .packages import special +from .packages import connection, special _logger = logging.getLogger(__name__) @@ -53,10 +53,11 @@ def connect(self, database=None, user=None, password=None, host=None, '\tport: %r' '\tsocket: %r' '\tcharset: %r', database, user, host, port, socket, charset) - conn = pymysql.connect(database=db, user=user, password=password, + conn = connection.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, use_unicode=True, charset=charset, autocommit=True, - client_flag=pymysql.constants.CLIENT.INTERACTIVE) + client_flag=pymysql.constants.CLIENT.INTERACTIVE, + cursorclass=connection.Cursor) if hasattr(self, 'conn'): self.conn.close() self.conn = conn diff --git a/tests/utils.py b/tests/utils.py index 1ace00b4b..7180fa46f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,6 @@ import pytest -import pymysql from mycli.main import format_output, special +from mycli.packages import connection from os import getenv # TODO: should this be somehow be divined from environment? @@ -8,8 +8,8 @@ PASSWORD = getenv('PASSWORD') def db_connection(dbname=None): - conn = pymysql.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, - charset=CHARSET) + conn = connection.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, + charset=CHARSET) conn.autocommit = True return conn From 133667775e9d22b4593ab994125428c8bf702e99 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 23 Sep 2015 20:40:53 -0500 Subject: [PATCH 0034/1025] Adds support for PyMySQL 0.6.2 and above. Addresses #155. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a6be0e97a..d87c62576 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit==0.46', - 'PyMySQL >= 0.6.6', + 'PyMySQL >= 0.6.2', 'sqlparse >= 0.1.16', 'configobj >= 5.0.6', 'pycrypto >= 2.6.1', From b9364f1e118a5861120ae58dd07686780aebe6c3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 23 Sep 2015 21:06:39 -0500 Subject: [PATCH 0035/1025] Renames the connect password argument to passwd. --- mycli/packages/connection.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mycli/packages/connection.py b/mycli/packages/connection.py index 8edf05c08..a7603c2f0 100644 --- a/mycli/packages/connection.py +++ b/mycli/packages/connection.py @@ -38,8 +38,14 @@ def __del__(self): def connect(*args, **kwargs): """Makes connect() use our custom Connection class. + PyMySQL < 0.6.3 uses the *passwd* argument instead of *password*. This + function renames that keyword or assigns it the default value of '', + which is the same default value PyMySQL gives it. + See pymysql.connections.Connection.__init__() for more information about calling this function. """ + kwargs['passwd'] = kwargs.pop('password', '') + return Connection(*args, **kwargs) From 8805c986dc981ff6097b014c2a3fb5112c1ea056 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 23 Sep 2015 21:19:31 -0500 Subject: [PATCH 0036/1025] Improves docstring grammar. --- mycli/packages/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/packages/connection.py b/mycli/packages/connection.py index a7603c2f0..5ece3f9eb 100644 --- a/mycli/packages/connection.py +++ b/mycli/packages/connection.py @@ -12,7 +12,7 @@ if pymysql.VERSION[1] == 6 and pymysql.VERSION[2] < 5: class Cursor(pymysql.cursors.Cursor): - """Makes pre-0.6.5 Cursor a context manager.""" + """Makes Cursor a context manager in PyMySQL < 0.6.5.""" def __enter__(self): return self @@ -24,7 +24,7 @@ def __exit__(self, *exc_info): if pymysql.VERSION[1] == 6 and pymysql.VERSION[2] < 3: class Connection(pymysql.connections.Connection): - """Adds error handling to pre-0.6.3 Connection.""" + """Adds error handling to Connection in PyMySQL < 0.6.3.""" def __del__(self): if self.socket: From cc7905a18c8715033339aa65df204d06df7c9b27 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 23 Sep 2015 21:33:57 -0500 Subject: [PATCH 0037/1025] Adds custom Cursor to unit tests. --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 7180fa46f..876bd199b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,7 +9,7 @@ def db_connection(dbname=None): conn = connection.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, - charset=CHARSET) + charset=CHARSET, cursorclass=connection.Cursor) conn.autocommit = True return conn From 4d137b5dab1a5ba85821e04c03e052177d7d97ad Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 23 Sep 2015 21:45:09 -0500 Subject: [PATCH 0038/1025] Adds TravisCI tests against two versions of PyMySQL. --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7c5856566..704862ca6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,12 @@ python: - "3.3" - "3.4" +env: + - PYMYSQL_VERSION=0.6.6 + - PYMYSQL_VERSION=0.6.2 + install: - - pip install . pytest mock codecov + - pip install PyMySQL==$PYMYSQL_VERSION . pytest mock codecov script: - coverage run --source mycli -m py.test From 3265662a3745df2e9745b4cc3637cd1ae3cc97b9 Mon Sep 17 00:00:00 2001 From: Martijn Engler Date: Fri, 25 Sep 2015 10:28:12 +0200 Subject: [PATCH 0039/1025] Add "delete" and "truncate" as destructive commands --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 640633c0e..d19c648fc 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -660,7 +660,7 @@ def confirm_destructive_query(queries): True if the query is destructive and the user wants to proceed. False if the query is destructive and the user doesn't want to proceed. """ - destructive = set(['drop', 'shutdown']) + destructive = set(['drop', 'shutdown', 'delete', 'truncate']) queries = queries.strip() for query in sqlparse.split(queries): try: From 3979aeac57bcc4af6a53dea3cf56fe440d151c59 Mon Sep 17 00:00:00 2001 From: Iryna Cherniavska Date: Fri, 25 Sep 2015 09:56:48 -0700 Subject: [PATCH 0040/1025] Updated release script with a --dry-run and --confirm-steps option. --- release.py | 97 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 26 deletions(-) diff --git a/release.py b/release.py index 8f98be5f7..7ced9bd9f 100644 --- a/release.py +++ b/release.py @@ -4,8 +4,44 @@ import ast import subprocess import sys +from optparse import OptionParser DEBUG = False +CONFIRM_STEPS = False +DRY_RUN = False + + +def skip_step(): + """ + Asks for user's response whether to run a step. Default is yes. + :return: boolean + """ + global CONFIRM_STEPS + + if CONFIRM_STEPS: + choice = raw_input("--- Confirm step? (y/N) [y] ") + if choice.lower() == 'n': + return True + return False + + +def run_step(*args): + """ + Prints out the command and asks if it should be run. + If yes (default), runs it. + :param args: list of strings (command and args) + """ + global DRY_RUN + + cmd = args + print(' '.join(cmd)) + if skip_step(): + print('--- Skipping...') + elif DRY_RUN: + print('--- Pretending to run...') + else: + subprocess.check_output(cmd) + def version(version_file): _version_re = re.compile(r'__version__\s+=\s+(.*)') @@ -16,45 +52,36 @@ def version(version_file): return ver + def commit_for_release(version_file, ver): - cmd = ['git', 'reset'] - print(' '.join(cmd)) - subprocess.check_output(cmd) - cmd = ['git', 'add', version_file] - print(' '.join(cmd)) - subprocess.check_output(cmd) - cmd = ['git', 'commit', '--message', 'Releasing version %s' % ver] - print(' '.join(cmd)) - subprocess.check_output(cmd) + run_step('git', 'reset') + run_step('git', 'add', version_file) + run_step('git', 'commit', '--message', 'Releasing version %s' % ver) + def create_git_tag(tag_name): - cmd = ['git', 'tag', tag_name] - print(' '.join(cmd)) - subprocess.check_output(cmd) + run_step('git', 'tag', tag_name) + def register_with_pypi(): - cmd = ['python', 'setup.py', 'register'] - print(' '.join(cmd)) - subprocess.check_output(cmd) + run_step('python', 'setup.py', 'register') + def create_source_tarball(): - cmd = ['python', 'setup.py', 'sdist'] - print(' '.join(cmd)) - subprocess.check_output(cmd) + run_step('python', 'setup.py', 'sdist') + def push_to_github(): - cmd = ['git', 'push', 'origin', 'master'] - print(' '.join(cmd)) - subprocess.check_output(cmd) + run_step('git', 'push', 'origin', 'master') + def push_tags_to_github(): - cmd = ['git', 'push', '--tags', 'origin'] - print(' '.join(cmd)) - subprocess.check_output(cmd) + run_step('git', 'push', '--tags', 'origin') + def checklist(questions): for question in questions: - choice = raw_input(question + ' (y/N)') + choice = raw_input(question + ' (y/N) [n] ') if choice.lower() != 'y': sys.exit(1) @@ -67,11 +94,29 @@ def checklist(questions): 'Have you updated the AUTHORS file?', ] checklist(checks) + ver = version('mycli/__init__.py') print('Releasing Version:', ver) - choice = raw_input('Are you sure? (y/N)') + + parser = OptionParser() + parser.add_option( + "-c", "--confirm-steps", action="store_true", dest="confirm_steps", + default=False, help=("Confirm every step. If the step is not " + "confirmed, it will be skipped.") + ) + parser.add_option( + "-d", "--dry-run", action="store_true", dest="dry_run", + default=False, help="Print out, but not actually run any steps." + ) + + popts, pargs = parser.parse_args() + CONFIRM_STEPS = popts.confirm_steps + DRY_RUN = popts.dry_run + + choice = raw_input('Are you sure? (y/N) [n] ') if choice.lower() != 'y': sys.exit(1) + commit_for_release('mycli/__init__.py', ver) create_git_tag('v%s' % ver) register_with_pypi() From 08c45b9a5a0bf96f1b7eace5139fe5d20622957b Mon Sep 17 00:00:00 2001 From: shoma Date: Tue, 6 Oct 2015 15:53:01 +0900 Subject: [PATCH 0041/1025] Add link to build status badge. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d4e8dbdb..65375e41d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # mycli -![BuildStatus](https://travis-ci.org/dbcli/mycli.svg?branch=master) +[![Build Status](https://travis-ci.org/dbcli/mycli.svg?branch=master)](https://travis-ci.org/dbcli/mycli) [![Join the chat at https://gitter.im/dbcli/mycli](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dbcli/mycli?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) A command line client for MySQL that can do auto-completion and syntax highlighting. From 778f8b1db39886ebd75cbf8739efa5f45db120c5 Mon Sep 17 00:00:00 2001 From: Shoma Suzuki Date: Tue, 6 Oct 2015 15:59:44 +0900 Subject: [PATCH 0042/1025] Add badge of PyPI version and download count --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 65375e41d..1e93ad036 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # mycli [![Build Status](https://travis-ci.org/dbcli/mycli.svg?branch=master)](https://travis-ci.org/dbcli/mycli) +[![PyPI](https://img.shields.io/pypi/v/mycli.svg?style=plastic)](https://pypi.python.org/pypi/mycli) +[![PyPI](https://img.shields.io/pypi/dm/mycli.svg?style=plastic)](https://pypi.python.org/pypi/mycli) [![Join the chat at https://gitter.im/dbcli/mycli](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dbcli/mycli?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) A command line client for MySQL that can do auto-completion and syntax highlighting. From 9d8a0ce2a0a5ec7fa51f0e232712c5375f66e323 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 6 Oct 2015 00:20:07 -0700 Subject: [PATCH 0043/1025] Remove the download count badge. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 1e93ad036..644765aa0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![Build Status](https://travis-ci.org/dbcli/mycli.svg?branch=master)](https://travis-ci.org/dbcli/mycli) [![PyPI](https://img.shields.io/pypi/v/mycli.svg?style=plastic)](https://pypi.python.org/pypi/mycli) -[![PyPI](https://img.shields.io/pypi/dm/mycli.svg?style=plastic)](https://pypi.python.org/pypi/mycli) [![Join the chat at https://gitter.im/dbcli/mycli](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dbcli/mycli?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) A command line client for MySQL that can do auto-completion and syntax highlighting. From 12f48cfe4b100862aa6aa17fe9c3486dafc4bf64 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 6 Oct 2015 00:30:27 -0700 Subject: [PATCH 0044/1025] Add a core dev list to AUTHORS file. --- AUTHORS | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index fa9d0f540..dbbca828b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,12 +1,17 @@ Many thanks to the following contributors. +Core Developers: +---------------- + + * Iryna Cherniavska + * Thomas Roten + * Darik Gamble + Contributors: ------------- - * Iryna Cherniavska * Steve Robbins * Daniel Black - * Thomas Roten * Jonathan Bruno * Heath Naylor * Daniel West From 87cb879dec6a1609ce8da93197b4afea1a0c8999 Mon Sep 17 00:00:00 2001 From: Daniel West Date: Wed, 7 Oct 2015 10:32:39 -0400 Subject: [PATCH 0045/1025] Incrementing PyMySQL version to include bug fixes. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b990b0a05..1d9a26178 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit==0.45', - 'PyMySQL >= 0.6.6', + 'PyMySQL >= 0.6.7', 'sqlparse == 0.1.14', 'configobj >= 5.0.6', ], From 3f6b9035d006fbfc1f3b01ba56a76720b733559c Mon Sep 17 00:00:00 2001 From: Daniel West Date: Wed, 7 Oct 2015 10:59:30 -0400 Subject: [PATCH 0046/1025] Incrementing travis pymysql version number. --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 704862ca6..5a8610ec0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,7 @@ python: - "3.4" env: - - PYMYSQL_VERSION=0.6.6 - - PYMYSQL_VERSION=0.6.2 + - PYMYSQL_VERSION=0.6.7 install: - pip install PyMySQL==$PYMYSQL_VERSION . pytest mock codecov From 1a89f3ce148c9cdab3f5520f174a6a5d560eeb5b Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 9 Oct 2015 14:12:19 -0300 Subject: [PATCH 0047/1025] Add new config to enable enter key to behave like the tab key. --- mycli/key_bindings.py | 15 +++++++++++++++ mycli/myclirc | 3 +++ 2 files changed, 18 insertions(+) diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index 9c2f3e865..44937822e 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -75,4 +75,19 @@ def _(event): else: event.cli.start_completion(select_first=False) + + @key_binding_manager.registry.add_binding(Keys.ControlJ) + def _(event): + """ + This sould only be activated if the config 'use_enter_key_as_tab' + is activated. The enter key's behaviour should be attributed to another + key, for example: C-M. + """ + _logger.debug('Detected key.') + b = event.cli.current_buffer + if b.complete_state: + b.complete_next() + else: + event.cli.start_completion(select_first=True) + return key_binding_manager diff --git a/mycli/myclirc b/mycli/myclirc index 8f8b7d888..3943c2ed7 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -44,6 +44,9 @@ key_bindings = emacs # Enabling this option will show the suggestions in a wider menu. Thus more items are suggested. wider_completion_menu = False +# Enabling this option will make the `enter key` behave like the `tab key`. Default: False. +use_enter_key_as_tab = False + # MySQL prompt # \t - Product type (Percona, MySQL, Mariadb) # \u - Username From cb7699e7099afc90023b06e96f4064046fd72fc4 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 9 Oct 2015 11:51:50 -0700 Subject: [PATCH 0048/1025] Rebind Control-J only when completion menu is showing. --- mycli/key_bindings.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index 44937822e..2675ee45c 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -1,7 +1,7 @@ import logging from prompt_toolkit.keys import Keys from prompt_toolkit.key_binding.manager import KeyBindingManager -from prompt_toolkit.filters import Condition +from prompt_toolkit.filters import Condition, HasCompletions _logger = logging.getLogger(__name__) @@ -75,8 +75,7 @@ def _(event): else: event.cli.start_completion(select_first=False) - - @key_binding_manager.registry.add_binding(Keys.ControlJ) + @key_binding_manager.registry.add_binding(Keys.ControlJ, filter=HasCompletions()) def _(event): """ This sould only be activated if the config 'use_enter_key_as_tab' @@ -84,10 +83,8 @@ def _(event): key, for example: C-M. """ _logger.debug('Detected key.') + event.current_buffer.complete_state = None b = event.cli.current_buffer - if b.complete_state: - b.complete_next() - else: - event.cli.start_completion(select_first=True) + b.complete_state = None return key_binding_manager From 2316dd5cea4f9b7dc6758189c2341aaaadcaf98b Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 9 Oct 2015 20:13:50 -0300 Subject: [PATCH 0049/1025] Remove and updated ControlJ's event binding docstring --- mycli/key_bindings.py | 4 +--- mycli/myclirc | 7 ++----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index 2675ee45c..ac3e5c1c8 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -78,9 +78,7 @@ def _(event): @key_binding_manager.registry.add_binding(Keys.ControlJ, filter=HasCompletions()) def _(event): """ - This sould only be activated if the config 'use_enter_key_as_tab' - is activated. The enter key's behaviour should be attributed to another - key, for example: C-M. + Makes the enter key work as the tab key only when showing the menu. """ _logger.debug('Detected key.') event.current_buffer.complete_state = None diff --git a/mycli/myclirc b/mycli/myclirc index 3943c2ed7..e226a88ce 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -41,12 +41,9 @@ syntax_style = default # When Vi mode is enabled you can use modal editing features offered by Vi in the REPL. key_bindings = emacs -# Enabling this option will show the suggestions in a wider menu. Thus more items are suggested. +# Enabling this option will show the suggestions in a wider menu. Thus more items are suggested. wider_completion_menu = False -# Enabling this option will make the `enter key` behave like the `tab key`. Default: False. -use_enter_key_as_tab = False - # MySQL prompt # \t - Product type (Percona, MySQL, Mariadb) # \u - Username @@ -55,7 +52,7 @@ use_enter_key_as_tab = False # \n - Newline prompt = '\t \u@\h:\d> ' -# Custom colors for the completion menu, toolbar, etc. +# Custom colors for the completion menu, toolbar, etc. [colors] # Completion menus. Token.Menu.Completions.Completion.Current = 'bg:#00aaaa #000000' From 12ac40daddfe7bfbac07d2c770e42c0d8413e2f2 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 10 Oct 2015 00:54:39 -0300 Subject: [PATCH 0050/1025] Add new filter in order to avoid "hitting twice" to execute a single command --- mycli/filters.py | 12 ++++++++++++ mycli/key_bindings.py | 8 ++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 mycli/filters.py diff --git a/mycli/filters.py b/mycli/filters.py new file mode 100644 index 000000000..6a8075ff3 --- /dev/null +++ b/mycli/filters.py @@ -0,0 +1,12 @@ +from prompt_toolkit.filters import Filter + +class HasSelectedCompletion(Filter): + """Enable when the current buffer has a selected completion.""" + + def __call__(self, cli): + complete_state = cli.current_buffer.complete_state + return (complete_state is not None and + complete_state.current_completion is not None) + + def __repr__(self): + return "HasSelectedCompletion()" diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index ac3e5c1c8..4a16b95b2 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -1,16 +1,19 @@ import logging from prompt_toolkit.keys import Keys from prompt_toolkit.key_binding.manager import KeyBindingManager -from prompt_toolkit.filters import Condition, HasCompletions +from prompt_toolkit.filters import Condition +from .filters import HasSelectedCompletion _logger = logging.getLogger(__name__) + def mycli_bindings(get_key_bindings, set_key_bindings): """ Custom key bindings for mycli. """ assert callable(get_key_bindings) assert callable(set_key_bindings) + key_binding_manager = KeyBindingManager( enable_open_in_editor=True, enable_system_bindings=True, @@ -75,12 +78,13 @@ def _(event): else: event.cli.start_completion(select_first=False) - @key_binding_manager.registry.add_binding(Keys.ControlJ, filter=HasCompletions()) + @key_binding_manager.registry.add_binding(Keys.ControlJ, filter=HasSelectedCompletion()) def _(event): """ Makes the enter key work as the tab key only when showing the menu. """ _logger.debug('Detected key.') + event.current_buffer.complete_state = None b = event.cli.current_buffer b.complete_state = None From 6ba5515e062745390eaaa6bcd939cbd88927abb7 Mon Sep 17 00:00:00 2001 From: Kacper Kwapisz Date: Thu, 15 Oct 2015 23:31:12 +0200 Subject: [PATCH 0051/1025] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 644765aa0..e20b209a3 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ or $ brew update && brew install mycli # Only on OS X ``` -Check the [detailed install instructions](#Detailed-Install-Instructions) for debian packages or getting started with pip. +Check the [detailed install instructions](#detailed-install-instructions) for debian packages or getting started with pip. ### Usage From 447808260e274fe6f30879a8f4b624850e858fa9 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 16 Oct 2015 03:41:08 -0700 Subject: [PATCH 0052/1025] Consolidate the timing information. --- mycli/main.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index d19c648fc..5912a1fc7 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -405,7 +405,6 @@ def prompt_tokens(cli): successful = False start = time() res = sqlexecute.run(document.text) - duration = time() - start successful = True output = [] total = 0 @@ -413,7 +412,6 @@ def prompt_tokens(cli): logger.debug("headers: %r", headers) logger.debug("rows: %r", cur) logger.debug("status: %r", status) - start = time() threshold = 1000 if (is_select(status) and cur and cur.rowcount > threshold): @@ -466,8 +464,7 @@ def prompt_tokens(cli): except KeyboardInterrupt: pass if special.is_timing_enabled(): - self.output('Command Time: %0.03fs' % duration) - self.output('Format Time: %0.03fs' % total) + self.output('Time: %0.03fs' % total) # Refresh the table names and column names if necessary. if need_completion_refresh(document.text): From c3131524b930072a916906ed8599db4883a0241c Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 16 Oct 2015 15:13:53 -0300 Subject: [PATCH 0053/1025] Add a handler to execute the system command --- mycli/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 5912a1fc7..c03cb6538 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -6,6 +6,7 @@ import sys import traceback import logging +import subprocess from time import time from datetime import datetime from random import choice @@ -170,6 +171,12 @@ def execute_from_file(self, arg, **_): return self.sqlexecute.run(query) + def execute_system_command(self, arg, **_): + if not arg: + message = 'Missing required argument: command.' + yield(None, None, None, message) + yield(None, None, None, subprocess.call(arg, shell=True)) + def initialize_logging(self): log_file = self.config['main']['log_file'] From 93180a0cc882b647c95615e5c538c56a697a10e9 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 16 Oct 2015 15:14:50 -0300 Subject: [PATCH 0054/1025] Registered the special command --- mycli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index c03cb6538..da121e2ea 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -139,6 +139,8 @@ def register_special_commands(self): '\\T', 'Change Table Type.', aliases=('\\T',), case_sensitive=True) special.register_special_command(self.execute_from_file, 'source', '\\. filename', 'Execute commands from file.', aliases=('\\.',)) + special.register_special_command(self.execute_system_command, 'system', + '\\s', 'Execute a system command.', aliases=('\\s',)) def change_table_format(self, arg, **_): if not arg in table_formats(): From c5308d2ac6e547a19c3ef53ecc2a96482ad7d5b5 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 16 Oct 2015 16:44:14 -0300 Subject: [PATCH 0055/1025] Move system command's implementation to the iocommands file --- mycli/main.py | 9 --------- mycli/packages/special/iocommands.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index da121e2ea..5912a1fc7 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -6,7 +6,6 @@ import sys import traceback import logging -import subprocess from time import time from datetime import datetime from random import choice @@ -139,8 +138,6 @@ def register_special_commands(self): '\\T', 'Change Table Type.', aliases=('\\T',), case_sensitive=True) special.register_special_command(self.execute_from_file, 'source', '\\. filename', 'Execute commands from file.', aliases=('\\.',)) - special.register_special_command(self.execute_system_command, 'system', - '\\s', 'Execute a system command.', aliases=('\\s',)) def change_table_format(self, arg, **_): if not arg in table_formats(): @@ -173,12 +170,6 @@ def execute_from_file(self, arg, **_): return self.sqlexecute.run(query) - def execute_system_command(self, arg, **_): - if not arg: - message = 'Missing required argument: command.' - yield(None, None, None, message) - yield(None, None, None, subprocess.call(arg, shell=True)) - def initialize_logging(self): log_file = self.config['main']['log_file'] diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index a473ecd92..bd314d530 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -1,6 +1,7 @@ import os import re import logging +import subprocess from io import open import click @@ -193,3 +194,12 @@ def delete_favorite_query(arg, **_): return [(None, None, None, status)] +@special_command('system', 'system [command]', 'Execute a system commmand.') +def execute_system_command(arg, **_): + """ + Execute a system command. + """ + usage = "Syntax: system command.\n\n " + if not arg: + return [(None, None, None, usage)] + return [(None, None, None, subprocess.call(arg, shell=True))] From 3986393f76f0b5ada5eb9c3f2d743740ddefb1e9 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 16 Oct 2015 17:56:15 -0300 Subject: [PATCH 0056/1025] Add a utils.py to the "special" package for helper functions --- mycli/packages/special/utils.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 mycli/packages/special/utils.py diff --git a/mycli/packages/special/utils.py b/mycli/packages/special/utils.py new file mode 100644 index 000000000..223e8c1d4 --- /dev/null +++ b/mycli/packages/special/utils.py @@ -0,0 +1,19 @@ +import os + +def handle_cd_command(arg): + """Handles a `cd` shell command by calling python's os.chdir.""" + CD_CMD = 'cd' + command = arg.strip() + directory = '' + + if command == CD_CMD: + # Treat `cd` as a change to the root directory. + # os.path.expanduser does this in a cross platform manner. + directory = os.path.expanduser('~') + else: + tokens = arg.split(CD_CMD + ' ') + directory = tokens[-1] + try: + os.chdir(directory) + except OSError, e: + output = e.strerror From aae8c9068c3abbd1c71e99432dd947b1544edd37 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 16 Oct 2015 17:58:48 -0300 Subject: [PATCH 0057/1025] Improve the "system command" handler to handle errors --- mycli/packages/special/iocommands.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index bd314d530..6e43de98d 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -10,6 +10,7 @@ from . import export from .main import special_command, NO_QUERY, PARSED_QUERY from .favoritequeries import favoritequeries +from .utils import handle_cd_command TIMING_ENABLED = False use_expanded_output = False @@ -199,7 +200,20 @@ def execute_system_command(arg, **_): """ Execute a system command. """ - usage = "Syntax: system command.\n\n " + usage = "Syntax: system [command].\n" + if not arg: return [(None, None, None, usage)] - return [(None, None, None, subprocess.call(arg, shell=True))] + + CD_CMD = 'cd' + output = '' + + try: + command = arg.strip() + if command.startswith(CD_CMD): + output = handle_cd_command(arg) + else: + output = subprocess.check_output(arg, stderr=subprocess.STDOUT, shell=True) + return [(None, None, None, output)] + except subprocess.CalledProcessError, e: + return [(None, None, None, e.output)] From 50e5e7e6f03d2181d97b5630cb3ea434b4d9effa Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 16 Oct 2015 18:37:58 -0300 Subject: [PATCH 0058/1025] Remove useless constant CD_CMD from handler --- mycli/packages/special/iocommands.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 6e43de98d..72f67ef5e 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -205,13 +205,12 @@ def execute_system_command(arg, **_): if not arg: return [(None, None, None, usage)] - CD_CMD = 'cd' output = '' try: command = arg.strip() - if command.startswith(CD_CMD): - output = handle_cd_command(arg) + if command.startswith('cd'): + output = handle_cd_command(arg) else: output = subprocess.check_output(arg, stderr=subprocess.STDOUT, shell=True) return [(None, None, None, output)] From aa17c9b61188cc2acb6859e15cac5cdc51ca9b82 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 16 Oct 2015 18:40:54 -0300 Subject: [PATCH 0059/1025] Format handle_cd_command's output message --- mycli/packages/special/utils.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/mycli/packages/special/utils.py b/mycli/packages/special/utils.py index 223e8c1d4..308d74353 100644 --- a/mycli/packages/special/utils.py +++ b/mycli/packages/special/utils.py @@ -1,19 +1,26 @@ import os +import subprocess def handle_cd_command(arg): """Handles a `cd` shell command by calling python's os.chdir.""" CD_CMD = 'cd' command = arg.strip() directory = '' + error = False + + tokens = arg.split(CD_CMD + ' ') + directory = tokens[-1] - if command == CD_CMD: - # Treat `cd` as a change to the root directory. - # os.path.expanduser does this in a cross platform manner. - directory = os.path.expanduser('~') - else: - tokens = arg.split(CD_CMD + ' ') - directory = tokens[-1] try: os.chdir(directory) + output = subprocess.check_output('pwd', stderr=subprocess.STDOUT, shell=True) except OSError, e: - output = e.strerror + output, error = e.strerror, True + + # formatting a nice output + if error: + output = "Error: {}".format(output) + else: + output = "Current directory: {}".format(output) + + return output From a13cc3fa6f27f1e0a1fce4e6d6061d87a890a30f Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 16 Oct 2015 18:56:26 -0300 Subject: [PATCH 0060/1025] Changed the expected line in test_special_command --- tests/test_sqlexecute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index 0904481b0..d8af89d8b 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -153,7 +153,7 @@ def test_favorite_query_multiple_statement(executor): @dbtest def test_special_command(executor): results = run(executor, '\\?') - expected_line = u'| help | \\? | Show this help. |\n' + expected_line = u'| help | \\? | Show this help. |\n' assert len(results) == 1 assert expected_line in results[0] From 084e7ae769cde2b2cdae9a166be8c39b148b7125 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 16 Oct 2015 19:16:42 -0300 Subject: [PATCH 0061/1025] Fix syntax error for python3 --- mycli/packages/special/iocommands.py | 2 +- mycli/packages/special/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 72f67ef5e..e8a9369d9 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -214,5 +214,5 @@ def execute_system_command(arg, **_): else: output = subprocess.check_output(arg, stderr=subprocess.STDOUT, shell=True) return [(None, None, None, output)] - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: return [(None, None, None, e.output)] diff --git a/mycli/packages/special/utils.py b/mycli/packages/special/utils.py index 308d74353..b8dc05f6d 100644 --- a/mycli/packages/special/utils.py +++ b/mycli/packages/special/utils.py @@ -14,7 +14,7 @@ def handle_cd_command(arg): try: os.chdir(directory) output = subprocess.check_output('pwd', stderr=subprocess.STDOUT, shell=True) - except OSError, e: + except OSError as e: output, error = e.strerror, True # formatting a nice output From c191b7a7b89b0c6df2dcc281a4db5481ff2e63df Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 16 Oct 2015 20:49:57 -0700 Subject: [PATCH 0062/1025] Handle decoding errors in older pymysql. --- mycli/main.py | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 5912a1fc7..9a7879313 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -425,6 +425,13 @@ def prompt_tokens(cli): end = time() total += end - start mutating = mutating or is_mutating(status) + except UnicodeDecodeError: + import pymysql + ver = pymysql.__version__.split('.') + if ver < ('0', '6', '7'): + message = ('You are running an older version of pymysql.\n' + 'Please upgrade to 0.6.7 or above to view binary data.') + self.output(message) except KeyboardInterrupt: # Restart connection to the database sqlexecute.connect() diff --git a/setup.py b/setup.py index 4fb1bf1d6..d87c62576 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit==0.46', - 'PyMySQL >= 0.6.7', + 'PyMySQL >= 0.6.2', 'sqlparse >= 0.1.16', 'configobj >= 5.0.6', 'pycrypto >= 2.6.1', From 4ab53bc73406396206ead375dd7b5e656fdc41b7 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 17 Oct 2015 07:11:12 -0300 Subject: [PATCH 0063/1025] Remove unused variable from `handle_cd_command` --- mycli/packages/special/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mycli/packages/special/utils.py b/mycli/packages/special/utils.py index b8dc05f6d..85d1993bb 100644 --- a/mycli/packages/special/utils.py +++ b/mycli/packages/special/utils.py @@ -4,7 +4,6 @@ def handle_cd_command(arg): """Handles a `cd` shell command by calling python's os.chdir.""" CD_CMD = 'cd' - command = arg.strip() directory = '' error = False From ed23e06513444efa90d838f3ed697ffd19790497 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 17 Oct 2015 23:30:50 -0700 Subject: [PATCH 0064/1025] Reraise exception if pymysql version is above 0.6.7. --- mycli/main.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 9a7879313..db7d09305 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -425,13 +425,15 @@ def prompt_tokens(cli): end = time() total += end - start mutating = mutating or is_mutating(status) - except UnicodeDecodeError: + except UnicodeDecodeError as e: import pymysql - ver = pymysql.__version__.split('.') - if ver < ('0', '6', '7'): + if pymysql.VERSION < ('0', '6', '7'): message = ('You are running an older version of pymysql.\n' - 'Please upgrade to 0.6.7 or above to view binary data.') + 'Please upgrade to 0.6.7 or above to view binary data.\n' + 'Try \'pip install -U pymysql\'.') self.output(message) + else: + raise e except KeyboardInterrupt: # Restart connection to the database sqlexecute.connect() From af7b3174ec52c7fe0feadf624a89310a4e420b9c Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 18 Oct 2015 19:44:52 -0700 Subject: [PATCH 0065/1025] Perform completion refresh in a background thread. --- mycli/clitoolbar.py | 7 +- mycli/completion_refresher.py | 119 +++++++++++++++++++++++++++ mycli/main.py | 124 ++++++++++++++++------------- mycli/sqlcompleter.py | 2 +- tests/test_completion_refresher.py | 85 ++++++++++++++++++++ 5 files changed, 278 insertions(+), 59 deletions(-) create mode 100644 mycli/completion_refresher.py create mode 100644 tests/test_completion_refresher.py diff --git a/mycli/clitoolbar.py b/mycli/clitoolbar.py index d6afaed92..66bdc8028 100644 --- a/mycli/clitoolbar.py +++ b/mycli/clitoolbar.py @@ -1,12 +1,12 @@ from pygments.token import Token -def create_toolbar_tokens_func(get_key_bindings, token=None): +def create_toolbar_tokens_func(get_key_bindings, get_is_refreshing): """ Return a function that generates the toolbar tokens. """ assert callable(get_key_bindings) - token = token or Token.Toolbar + token = Token.Toolbar def get_toolbar_tokens(cli): result = [] @@ -31,5 +31,8 @@ def get_toolbar_tokens(cli): else: result.append((token.On, '[F4] Emacs-mode')) + if get_is_refreshing(): + result.append((token, ' Refreshing completions...')) + return result return get_toolbar_tokens diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py new file mode 100644 index 000000000..4e05ac121 --- /dev/null +++ b/mycli/completion_refresher.py @@ -0,0 +1,119 @@ +import threading +from .packages.special.main import COMMANDS +try: + from collections import OrderedDict +except ImportError: + from .packages.ordereddict import OrderedDict + +from .sqlcompleter import SQLCompleter +from .sqlexecute import SQLExecute + +class CompletionRefresher(object): + + refreshers = OrderedDict() + + def __init__(self): + self._completer_thread = None + self._restart_refresh = threading.Event() + + def refresh(self, executor, callbacks): + """ + Creates a SQLCompleter object and populates it with the relevant + completion suggestions in a background thread. + + executor - SQLExecute object, used to extract the credentials to connect + to the database. + callbacks - A function or a list of functions to call after the thread + has completed the refresh. The newly created completion + object will be passed in as an argument to each callback. + """ + if self.is_refreshing(): + self._restart_refresh.set() + return [(None, None, None, 'Auto-completion refresh restarted.')] + else: + self._completer_thread = threading.Thread(target=self._bg_refresh, + args=(executor, callbacks), + name='completion_refresh') + self._completer_thread.setDaemon(True) + self._completer_thread.start() + return [(None, None, None, + 'Auto-completion refresh started in the background.')] + + def is_refreshing(self): + return self._completer_thread and self._completer_thread.is_alive() + + def _bg_refresh(self, sqlexecute, callbacks): + completer = SQLCompleter(smart_completion=True) + + # Create a new pgexecute method to popoulate the completions. + e = sqlexecute + executor = SQLExecute(e.dbname, e.user, e.password, e.host, e.port, + e.socket, e.charset) + + # If callbacks is a single function then push it into a list. + if callable(callbacks): + callbacks = [callbacks] + + while 1: + for refresher in self.refreshers.values(): + refresher(completer, executor) + if self._restart_refresh.is_set(): + self._restart_refresh.clear() + break + else: + # Break out of while loop if the for loop finishes natually + # without hitting the break statement. + break + + # Start over the refresh from the beginning if the for loop hit the + # break statement. + continue + + for callback in callbacks: + callback(completer) + +def refresher(name, refreshers=CompletionRefresher.refreshers): + """Decorator to add the decorated function to the dictionary of + refreshers. Any function decorated with a @refresher will be executed as + part of the completion refresh routine.""" + def wrapper(wrapped): + refreshers[name] = wrapped + return wrapped + return wrapper + +@refresher('databases') +def refresh_databases(completer, executor): + completer.extend_database_names(executor.databases()) + +@refresher('schemata') +def refresh_schemata(completer, executor): + # schemata - In MySQL Schema is the same as database. But for mycli + # schemata will be the name of the current database. + completer.extend_schemata(executor.dbname) + completer.set_dbname(executor.dbname) + +@refresher('tables') +def refresh_tables(completer, executor): + completer.extend_relations(executor.tables(), kind='tables') + completer.extend_columns(executor.table_columns(), kind='tables') + +@refresher('users') +def refresh_users(completer, executor): + completer.extend_users(executor.users()) + +# @refresher('views') +# def refresh_views(completer, executor): +# completer.extend_relations(executor.views(), kind='views') +# completer.extend_columns(executor.view_columns(), kind='views') + +@refresher('functions') +def refresh_functions(completer, executor): + completer.extend_functions(executor.functions()) + +@refresher('special_commands') +def refresh_special(completer, executor): + completer.extend_special_commands(COMMANDS.keys()) + +@refresher('show_commands') +def refresh_show_commands(completer, executor): + completer.extend_show_items(executor.show_candidates()) diff --git a/mycli/main.py b/mycli/main.py index 9a7879313..4e66386d2 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -6,6 +6,7 @@ import sys import traceback import logging +import threading from time import time from datetime import datetime from random import choice @@ -33,6 +34,7 @@ from .clistyle import style_factory from .sqlexecute import SQLExecute from .clibuffer import CLIBuffer +from .completion_refresher import CompletionRefresher from .config import (write_default_config, load_config, get_mylogin_cnf_path, open_mylogin_cnf) from .key_bindings import mycli_bindings @@ -97,6 +99,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') + self.completion_refresher = CompletionRefresher() + self.logger = logging.getLogger(__name__) self.initialize_logging() @@ -108,8 +112,8 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. smart_completion = c['main'].as_bool('smart_completion') - completer = SQLCompleter(smart_completion) - self.completer = completer + self.completer = SQLCompleter(smart_completion) + self._completer_lock = threading.Lock() # Register custom special commands. self.register_special_commands() @@ -126,13 +130,15 @@ def __init__(self, sqlexecute=None, prompt=None, # There was an error reading the login path file. print('Error: Unable to read login path file.') + self.cli = None + def register_special_commands(self): special.register_special_command(self.change_db, 'use', '\\u', 'Change to a new database.', aliases=('\\u',)) special.register_special_command(self.change_db, 'connect', '\\r', 'Reconnect to the database. Optional database argument.', aliases=('\\r', )) - special.register_special_command(self.refresh_dynamic_completions, 'rehash', + special.register_special_command(self.refresh_completions, 'rehash', '\\#', 'Refresh auto-completions.', arg_type=NO_QUERY, aliases=('\\#',)) special.register_special_command(self.change_table_format, 'tableformat', '\\T', 'Change Table Type.', aliases=('\\T',), case_sensitive=True) @@ -315,8 +321,7 @@ def run_cli(self): original_less_opts = self.adjust_less_opts() self.set_pager_from_config() - self.initialize_completions() - completer = self.completer + self.refresh_completions() def set_key_bindings(value): if value not in ('emacs', 'vi'): @@ -338,7 +343,9 @@ def set_key_bindings(value): def prompt_tokens(cli): return [(Token.Prompt, self.get_prompt(self.prompt_format))] - get_toolbar_tokens = create_toolbar_tokens_func(lambda: self.key_bindings) + get_toolbar_tokens = create_toolbar_tokens_func(lambda: self.key_bindings, + self.completion_refresher.is_refreshing) + layout = create_default_layout(lexer=MyCliLexer, reserve_space_for_menu=True, multiline=True, @@ -350,20 +357,22 @@ def prompt_tokens(cli): processor=HighlightMatchingBracketProcessor(chars='[](){}'), filter=HasFocus(DEFAULT_BUFFER) & ~IsDone()), ]) - buf = CLIBuffer(always_multiline=self.multi_line, completer=completer, - history=FileHistory(os.path.expanduser('~/.mycli-history')), - complete_while_typing=Always()) - - application = Application(style=style_factory(self.syntax_style, self.cli_style), - layout=layout, buffer=buf, - key_bindings_registry=key_binding_manager.registry, - on_exit=AbortAction.RAISE_EXCEPTION, - ignore_case=True) - cli = CommandLineInterface(application=application, eventloop=create_eventloop()) + with self._completer_lock: + buf = CLIBuffer(always_multiline=self.multi_line, completer=self.completer, + history=FileHistory(os.path.expanduser('~/.mycli-history')), + complete_while_typing=Always()) + + application = Application(style=style_factory(self.syntax_style, self.cli_style), + layout=layout, buffer=buf, + key_bindings_registry=key_binding_manager.registry, + on_exit=AbortAction.RAISE_EXCEPTION, + ignore_case=True) + self.cli = CommandLineInterface(application=application, + eventloop=create_eventloop()) try: while True: - document = cli.run() + document = self.cli.run() special.set_expanded_output(False) @@ -375,7 +384,7 @@ def prompt_tokens(cli): raise EOFError try: - document = self.handle_editor_command(cli, document) + document = self.handle_editor_command(self.cli, document) except RuntimeError as e: logger.error("sql: %r, error: %r", document.text, e) logger.error("traceback: %r", traceback.format_exc()) @@ -475,7 +484,7 @@ def prompt_tokens(cli): # Refresh the table names and column names if necessary. if need_completion_refresh(document.text): - self.refresh_dynamic_completions() + self.refresh_completions(reset=need_completion_reset(document.text)) query = Query(document.text, successful, mutating) self.query_history.append(query) @@ -511,49 +520,39 @@ def set_pager_from_config(self): if cnf['pager']: special.set_pager(cnf['pager']) - def initialize_completions(self): - completer = self.completer - - # special_commands - completer.extend_special_commands(COMMANDS.keys()) - - # Items to complete after the SHOW command. - completer.extend_show_items(self.sqlexecute.show_candidates()) - - return self.refresh_dynamic_completions() - - def refresh_dynamic_completions(self): - sqlexecute = self.sqlexecute - - completer = self.completer - completer.reset_completions() - - # databases - completer.extend_database_names(sqlexecute.databases()) + def refresh_completions(self, reset=False): + if reset: + with self._completer_lock: + self.completer.reset_completions() + self.completion_refresher.refresh(self.sqlexecute, + self._on_completions_refreshed) - # schemata - In MySQL Schema is the same as database. But for mycli - # schemata will be the name of the current database. - completer.extend_schemata(self.sqlexecute.dbname) - completer.set_dbname(self.sqlexecute.dbname) + return [(None, None, None, + 'Auto-completion refresh started in the background.')] - # tables - completer.extend_relations(sqlexecute.tables(), kind='tables') - completer.extend_columns(sqlexecute.table_columns(), kind='tables') + def _on_completions_refreshed(self, new_completer): + self._swap_completer_objects(new_completer) - # users - completer.extend_users(sqlexecute.users()) + if self.cli: + # After refreshing, redraw the CLI to clear the statusbar + # "Refreshing completions..." indicator + self.cli.request_redraw() - # views - #completer.extend_relations(sqlexecute.views(), kind='views') - #completer.extend_columns(sqlexecute.view_columns(), kind='views') - - # functions - completer.extend_functions(sqlexecute.functions()) - return [(None, None, None, 'Auto-completion refreshed.')] + def _swap_completer_objects(self, new_completer): + """Swap the completer object in cli with the newly created completer. + """ + with self._completer_lock: + self.completer = new_completer + # When mycli is first launched we call refresh_completions before + # instantiating the cli object. So it is necessary to check if cli + # exists before trying the replace the completer object in cli. + if self.cli: + self.cli.current_buffer.completer = new_completer def get_completions(self, text, cursor_positition): - return self.completer.get_completions( - Document(text=text, cursor_position=cursor_positition), None) + with self._completer_lock: + return self.completer.get_completions( + Document(text=text, cursor_position=cursor_positition), None) def get_prompt(self, string): sqlexecute = self.sqlexecute @@ -642,6 +641,19 @@ def need_completion_refresh(queries): except Exception: return False +def need_completion_reset(queries): + """Determines if the statement is a database switch such as 'use' or '\\u'. + When a database is changed the existing completions must be reset before we + start the completion refresh for the new database. + """ + for query in sqlparse.split(queries): + try: + first_token = query.split()[0] + return first_token.lower() in ('use', '\\u') + except Exception: + return False + + def is_mutating(status): """Determines if the statement is mutating based on the status.""" if not status: diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index eaeebe0b1..20a2b3c89 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -150,7 +150,7 @@ def extend_relations(self, data, kind): for relname in data: try: metadata[self.dbname][relname[0]] = ['*'] - except AttributeError: + except KeyError: _logger.error('%r %r listed in unrecognized schema %r', kind, relname[0], self.dbname) self.all_completions.add(relname[0]) diff --git a/tests/test_completion_refresher.py b/tests/test_completion_refresher.py new file mode 100644 index 000000000..a50ad7492 --- /dev/null +++ b/tests/test_completion_refresher.py @@ -0,0 +1,85 @@ +import time +import pytest +from mock import Mock, patch + + +@pytest.fixture +def refresher(): + from mycli.completion_refresher import CompletionRefresher + return CompletionRefresher() + + +def test_ctor(refresher): + """ + Refresher object should contain a few handlers + :param refresher: + :return: + """ + assert len(refresher.refreshers) > 0 + actual_handlers = list(refresher.refreshers.keys()) + expected_handlers = ['databases', 'schemata', 'tables', 'users', 'functions', + 'special_commands', 'show_commands'] + assert expected_handlers == actual_handlers + + +def test_refresh_called_once(refresher): + """ + + :param refresher: + :return: + """ + callbacks = Mock() + sqlexecute = Mock() + + with patch.object(refresher, '_bg_refresh') as bg_refresh: + actual = refresher.refresh(sqlexecute, callbacks) + time.sleep(1) # Wait for the thread to work. + assert len(actual) == 1 + assert len(actual[0]) == 4 + assert actual[0][3] == 'Auto-completion refresh started in the background.' + bg_refresh.assert_called_with(sqlexecute, callbacks) + + +def test_refresh_called_twice(refresher): + """ + If refresh is called a second time, it should be restarted + :param refresher: + :return: + """ + callbacks = Mock() + + sqlexecute = Mock() + + def dummy_bg_refresh(*args): + time.sleep(3) # seconds + + refresher._bg_refresh = dummy_bg_refresh + + actual1 = refresher.refresh(sqlexecute, callbacks) + time.sleep(1) # Wait for the thread to work. + assert len(actual1) == 1 + assert len(actual1[0]) == 4 + assert actual1[0][3] == 'Auto-completion refresh started in the background.' + + actual2 = refresher.refresh(sqlexecute, callbacks) + time.sleep(1) # Wait for the thread to work. + assert len(actual2) == 1 + assert len(actual2[0]) == 4 + assert actual2[0][3] == 'Auto-completion refresh restarted.' + + +def test_refresh_with_callbacks(refresher): + """ + Callbacks must be called + :param refresher: + """ + callbacks = [Mock()] + sqlexecute_class = Mock() + sqlexecute = Mock() + + with patch('mycli.completion_refresher.SQLExecute', sqlexecute_class): + # Set refreshers to 0: we're not testing refresh logic here + refresher.refreshers = {} + refresher.refresh(sqlexecute, callbacks) + time.sleep(1) # Wait for the thread to work. + assert (callbacks[0].call_count == 1) From f13fa3b21c364b8879475273a8440a8cdc569787 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 18 Oct 2015 19:55:39 -0700 Subject: [PATCH 0066/1025] Fix refresh completion to check each query. --- mycli/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 4e66386d2..1b82e9091 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -635,9 +635,9 @@ def need_completion_refresh(queries): for query in sqlparse.split(queries): try: first_token = query.split()[0] - res = first_token.lower() in ('alter', 'create', 'use', '\\r', - '\\u', 'connect', 'drop') - return res + if first_token.lower() in ('alter', 'create', 'use', '\\r', + '\\u', 'connect', 'drop'): + return True except Exception: return False @@ -649,7 +649,8 @@ def need_completion_reset(queries): for query in sqlparse.split(queries): try: first_token = query.split()[0] - return first_token.lower() in ('use', '\\u') + if first_token.lower() in ('use', '\\u'): + return True except Exception: return False From b0aa05230cf5624059849224ab164f1766f7dc4d Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 18 Oct 2015 20:23:57 -0700 Subject: [PATCH 0067/1025] Add ordereddict for python 2.6 compatibility. --- mycli/packages/ordereddict.py | 127 ++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 mycli/packages/ordereddict.py diff --git a/mycli/packages/ordereddict.py b/mycli/packages/ordereddict.py new file mode 100644 index 000000000..5b0303f5a --- /dev/null +++ b/mycli/packages/ordereddict.py @@ -0,0 +1,127 @@ +# Copyright (c) 2009 Raymond Hettinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from UserDict import DictMixin + +class OrderedDict(dict, DictMixin): + + def __init__(self, *args, **kwds): + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__end + except AttributeError: + self.clear() + self.update(*args, **kwds) + + def clear(self): + self.__end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.__map = {} # key --> [key, prev, next] + dict.clear(self) + + def __setitem__(self, key, value): + if key not in self: + end = self.__end + curr = end[1] + curr[2] = end[1] = self.__map[key] = [key, curr, end] + dict.__setitem__(self, key, value) + + def __delitem__(self, key): + dict.__delitem__(self, key) + key, prev, next = self.__map.pop(key) + prev[2] = next + next[1] = prev + + def __iter__(self): + end = self.__end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.__end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + def popitem(self, last=True): + if not self: + raise KeyError('dictionary is empty') + if last: + key = reversed(self).next() + else: + key = iter(self).next() + value = self.pop(key) + return key, value + + def __reduce__(self): + items = [[k, self[k]] for k in self] + tmp = self.__map, self.__end + del self.__map, self.__end + inst_dict = vars(self).copy() + self.__map, self.__end = tmp + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def keys(self): + return list(self) + + setdefault = DictMixin.setdefault + update = DictMixin.update + pop = DictMixin.pop + values = DictMixin.values + items = DictMixin.items + iterkeys = DictMixin.iterkeys + itervalues = DictMixin.itervalues + iteritems = DictMixin.iteritems + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + + def copy(self): + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + if isinstance(other, OrderedDict): + if len(self) != len(other): + return False + for p, q in zip(self.items(), other.items()): + if p != q: + return False + return True + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other From ef577743c1ce4582d484fc5a523c5a2b95687b30 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 19 Oct 2015 15:02:35 -0200 Subject: [PATCH 0068/1025] Refactored system command handler --- mycli/packages/special/iocommands.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index e8a9369d9..27586a6fb 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -205,14 +205,18 @@ def execute_system_command(arg, **_): if not arg: return [(None, None, None, usage)] - output = '' - try: command = arg.strip() if command.startswith('cd'): - output = handle_cd_command(arg) - else: - output = subprocess.check_output(arg, stderr=subprocess.STDOUT, shell=True) - return [(None, None, None, output)] - except subprocess.CalledProcessError as e: - return [(None, None, None, e.output)] + result, error_message = handle_cd_command(arg) + if not result: + return [(None, None, None, error_message)] + + args = arg.split(' ') + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + response = output if not error else error + + return [(None, None, None, response)] + except OSError as e: + return [(None, None, None, 'OSError: %s' % e.strerror)] From f27eddc2b2f22575fb7bf0ef1d40167456ffd569 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 19 Oct 2015 15:15:44 -0200 Subject: [PATCH 0069/1025] Remove formatting in handler_cd_command --- mycli/packages/special/utils.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/mycli/packages/special/utils.py b/mycli/packages/special/utils.py index 85d1993bb..7812976ec 100644 --- a/mycli/packages/special/utils.py +++ b/mycli/packages/special/utils.py @@ -14,12 +14,4 @@ def handle_cd_command(arg): os.chdir(directory) output = subprocess.check_output('pwd', stderr=subprocess.STDOUT, shell=True) except OSError as e: - output, error = e.strerror, True - - # formatting a nice output - if error: - output = "Error: {}".format(output) - else: - output = "Current directory: {}".format(output) - - return output + return False, e.strerror From 60a44ce1fe2fda130ec1cf416accfffa270fcd2e Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 19 Oct 2015 15:18:32 -0200 Subject: [PATCH 0070/1025] Stop using 'check_output' method and start using 'call' method in handler_cd_command --- mycli/packages/special/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/packages/special/utils.py b/mycli/packages/special/utils.py index 7812976ec..8a0ce20e6 100644 --- a/mycli/packages/special/utils.py +++ b/mycli/packages/special/utils.py @@ -12,6 +12,7 @@ def handle_cd_command(arg): try: os.chdir(directory) - output = subprocess.check_output('pwd', stderr=subprocess.STDOUT, shell=True) + subprocess.call(['pwd']) + return True, None except OSError as e: return False, e.strerror From 713b91c4d7dc3737223bc70aa329ec9de2c48fb8 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 19 Oct 2015 15:19:24 -0200 Subject: [PATCH 0071/1025] Add validation for 'cd' command argument --- mycli/packages/special/utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mycli/packages/special/utils.py b/mycli/packages/special/utils.py index 8a0ce20e6..e1a160acb 100644 --- a/mycli/packages/special/utils.py +++ b/mycli/packages/special/utils.py @@ -4,12 +4,10 @@ def handle_cd_command(arg): """Handles a `cd` shell command by calling python's os.chdir.""" CD_CMD = 'cd' - directory = '' - error = False - tokens = arg.split(CD_CMD + ' ') - directory = tokens[-1] - + directory = tokens[-1] if len(tokens) > 1 else None + if not directory: + return False, "No folder name was provided." try: os.chdir(directory) subprocess.call(['pwd']) From a447b2999d7988087d47a476daa0d27249d54260 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 19 Oct 2015 15:20:26 -0200 Subject: [PATCH 0072/1025] Add tests for system command --- tests/test.txt | 1 + tests/test_sqlexecute.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 tests/test.txt diff --git a/tests/test.txt b/tests/test.txt new file mode 100644 index 000000000..8d8b211e5 --- /dev/null +++ b/tests/test.txt @@ -0,0 +1 @@ +mycli rocks! diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index d8af89d8b..62da9cc51 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -2,6 +2,7 @@ import pytest import pymysql +import os from textwrap import dedent from utils import run, dbtest, set_expanded_output @@ -157,6 +158,35 @@ def test_special_command(executor): assert len(results) == 1 assert expected_line in results[0] +@dbtest +def test_cd_command_without_a_folder_name(executor): + results = run(executor, 'system cd') + expected_line = 'No folder name was provided.' + assert len(results) == 1 + assert expected_line in results[0] + +@dbtest +def test_system_command_not_found(executor): + results = run(executor, 'system xyz') + assert len(results) == 1 + expected_line = 'OSError:' + assert expected_line in results[0] + +@dbtest +def test_system_command_output(executor): + test_file_path = os.path.join(os.path.abspath('.'), 'tests/test.txt') + results = run(executor, 'system cat {0}'.format(test_file_path)) + assert len(results) == 1 + expected_line = u'mycli rocks!\n' + result_str = results[0].decode('utf-8') # python3 returns a bytes-string + assert expected_line == result_str + +@dbtest +def test_cd_command_current_dir(executor): + tests_path = os.path.join(os.path.abspath('.'), 'tests') + results = run(executor, 'system cd {0}'.format(tests_path)) + assert os.getcwd() == tests_path + @dbtest def test_unicode_support(executor): assert u'日本語' in run(executor, "SELECT '日本語' AS japanese;", join=True) From 37cc73d08be905ebd407fe04839568d00390799d Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 19 Oct 2015 17:19:14 -0200 Subject: [PATCH 0073/1025] Add missing statement when handling the command --- mycli/packages/special/iocommands.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 27586a6fb..307a27906 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -208,9 +208,10 @@ def execute_system_command(arg, **_): try: command = arg.strip() if command.startswith('cd'): - result, error_message = handle_cd_command(arg) - if not result: + ok, error_message = handle_cd_command(arg) + if not ok: return [(None, None, None, error_message)] + return [(None, None, None, '')] args = arg.split(' ') process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 78e5fad09ba7de65f3206306653fdc6ecf2cf9d2 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 20 Oct 2015 03:47:47 -0700 Subject: [PATCH 0074/1025] Refresh completions only when a command is successful. --- mycli/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 1b82e9091..43a55a4a4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -482,9 +482,10 @@ def prompt_tokens(cli): if special.is_timing_enabled(): self.output('Time: %0.03fs' % total) - # Refresh the table names and column names if necessary. - if need_completion_refresh(document.text): - self.refresh_completions(reset=need_completion_reset(document.text)) + # Refresh the table names and column names if necessary. + if need_completion_refresh(document.text): + self.refresh_completions( + reset=need_completion_reset(document.text)) query = Query(document.text, successful, mutating) self.query_history.append(query) From efeb8ce91e9a4c375475fbb6b36586eccdd718da Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Tue, 20 Oct 2015 16:48:18 -0200 Subject: [PATCH 0075/1025] Make the `connect` alias command to be case_sensitive in order to use the `\R` --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index db7d09305..b15778f49 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -131,7 +131,7 @@ def register_special_commands(self): '\\u', 'Change to a new database.', aliases=('\\u',)) special.register_special_command(self.change_db, 'connect', '\\r', 'Reconnect to the database. Optional database argument.', - aliases=('\\r', )) + aliases=('\\r', ), case_sensitive=True) special.register_special_command(self.refresh_dynamic_completions, 'rehash', '\\#', 'Refresh auto-completions.', arg_type=NO_QUERY, aliases=('\\#',)) special.register_special_command(self.change_table_format, 'tableformat', From eb4f0e27fb62923ff5f5b75bb7152b29e99918cb Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Tue, 20 Oct 2015 16:49:32 -0200 Subject: [PATCH 0076/1025] Add a handler for the `\R` command --- mycli/main.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index b15778f49..9ed067dad 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -170,6 +170,17 @@ def execute_from_file(self, arg, **_): return self.sqlexecute.run(query) + def change_prompt_format(self, arg, **_): + """ + Change the prompt format. + """ + if not arg: + message = 'Missing required argument, format.' + return [(None, None, None, message)] + + self.prompt_format = self.get_prompt(arg) + return [(None, None, None, "Changed prompt format to %s" % arg)] + def initialize_logging(self): log_file = self.config['main']['log_file'] From 19463453fe96073f9c2e0b3bce206274fbbf1ca9 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Tue, 20 Oct 2015 16:51:15 -0200 Subject: [PATCH 0077/1025] Register the special command `prompt` with the `\R` as alias --- mycli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 9ed067dad..837ba430a 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -138,6 +138,8 @@ def register_special_commands(self): '\\T', 'Change Table Type.', aliases=('\\T',), case_sensitive=True) special.register_special_command(self.execute_from_file, 'source', '\\. filename', 'Execute commands from file.', aliases=('\\.',)) + special.register_special_command(self.change_prompt_format, 'prompt', + '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): if not arg in table_formats(): From 386030e985467f932ebbf8e9a11b2bd7bbe63d4a Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Thu, 22 Oct 2015 22:02:42 -0200 Subject: [PATCH 0078/1025] Fix typo in MyCli's method name when registering a special command --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index e24399daa..3bcef18b5 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -138,7 +138,7 @@ def register_special_commands(self): special.register_special_command(self.change_db, 'connect', '\\r', 'Reconnect to the database. Optional database argument.', aliases=('\\r', ), case_sensitive=True) - special.register_special_command(self.refresh_dynamic_completions, 'rehash', + special.register_special_command(self.refresh_completions, 'rehash', '\\#', 'Refresh auto-completions.', arg_type=NO_QUERY, aliases=('\\#',)) special.register_special_command(self.change_table_format, 'tableformat', '\\T', 'Change Table Type.', aliases=('\\T',), case_sensitive=True) From da33c56683bc8babc8756ddd0cd8a6b06957187a Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 23 Oct 2015 11:46:30 -0200 Subject: [PATCH 0079/1025] Add ability to make suggestions for compound join clauses --- mycli/packages/completion_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 0c480316b..9f6a2c321 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -270,7 +270,7 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier return [{'type': 'database'}] elif token_v == 'tableformat': return [{'type': 'table_format'}] - elif token_v.endswith(',') or token_v == '=': + elif token_v.endswith(',') or token_v in ['=', 'and', 'or']: prev_keyword, text_before_cursor = find_prev_keyword(text_before_cursor) if prev_keyword: return suggest_based_on_last_token( From 17ad0d4394672ecccdc274de80521e10d24d1bdc Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 23 Oct 2015 11:47:59 -0200 Subject: [PATCH 0080/1025] Refactor tests to check for compound statements --- tests/test_completion_engine.py | 60 ++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/tests/test_completion_engine.py b/tests/test_completion_engine.py index 3f406bc7e..ee1c46e2d 100644 --- a/tests/test_completion_engine.py +++ b/tests/test_completion_engine.py @@ -268,46 +268,60 @@ def test_join_suggests_tables_and_schemas(tbl_alias, join_type): {'type': 'view', 'schema': []}, {'type': 'schema'}]) -def test_join_alias_dot_suggests_cols1(): - suggestions = suggest_type('SELECT * FROM abc a JOIN def d ON a.', - 'SELECT * FROM abc a JOIN def d ON a.') +@pytest.mark.parametrize('sql', [ + 'SELECT * FROM abc a JOIN def d ON a.', + 'SELECT * FROM abc a JOIN def d ON a.id = d.id AND a.', +]) +def test_join_alias_dot_suggests_cols1(sql): + suggestions = suggest_type(sql, sql) assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'abc', 'a')]}, {'type': 'table', 'schema': 'a'}, {'type': 'view', 'schema': 'a'}, {'type': 'function', 'schema': 'a'}]) -def test_join_alias_dot_suggests_cols2(): - suggestion = suggest_type('SELECT * FROM abc a JOIN def d ON a.', - 'SELECT * FROM abc a JOIN def d ON a.id = d.') - assert sorted_dicts(suggestion) == sorted_dicts([ +@pytest.mark.parametrize('sql', [ + 'SELECT * FROM abc a JOIN def d ON a.id = d.', + 'SELECT * FROM abc a JOIN def d ON a.id = d.id AND a.id2 = d.', +]) +def test_join_alias_dot_suggests_cols2(sql): + suggestions = suggest_type(sql, sql) + assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'def', 'd')]}, {'type': 'table', 'schema': 'd'}, {'type': 'view', 'schema': 'd'}, {'type': 'function', 'schema': 'd'}]) -def test_on_suggests_aliases(): - suggestions = suggest_type( - 'select a.x, b.y from abc a join bcd b on ', - 'select a.x, b.y from abc a join bcd b on ') +@pytest.mark.parametrize('sql', [ + 'select a.x, b.y from abc a join bcd b on ', + 'select a.x, b.y from abc a join bcd b on a.id = b.id OR ', +]) +def test_on_suggests_aliases(sql): + suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['a', 'b']}] -def test_on_suggests_tables(): - suggestions = suggest_type( - 'select abc.x, bcd.y from abc join bcd on ', - 'select abc.x, bcd.y from abc join bcd on ') +@pytest.mark.parametrize('sql', [ + 'select abc.x, bcd.y from abc join bcd on ', + 'select abc.x, bcd.y from abc join bcd on abc.id = bcd.id AND ', +]) +def test_on_suggests_tables(sql): + suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['abc', 'bcd']}] -def test_on_suggests_aliases_right_side(): - suggestions = suggest_type( - 'select a.x, b.y from abc a join bcd b on a.id = ', - 'select a.x, b.y from abc a join bcd b on a.id = ') +@pytest.mark.parametrize('sql', [ + 'select a.x, b.y from abc a join bcd b on a.id = ', + 'select a.x, b.y from abc a join bcd b on a.id = b.id AND a.id2 = ', +]) +def test_on_suggests_aliases_right_side(sql): + suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['a', 'b']}] -def test_on_suggests_tables_right_side(): - suggestions = suggest_type( - 'select abc.x, bcd.y from abc join bcd on ', - 'select abc.x, bcd.y from abc join bcd on ') +@pytest.mark.parametrize('sql', [ + 'select abc.x, bcd.y from abc join bcd on ', + 'select abc.x, bcd.y from abc join bcd on abc.id = bcd.id and ', +]) +def test_on_suggests_tables_right_side(sql): + suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['abc', 'bcd']}] From 16f9ba97e4c1dd5577f7c3476fa443ee9ce2cb01 Mon Sep 17 00:00:00 2001 From: Iryna Cherniavska Date: Sat, 24 Oct 2015 20:29:01 -0700 Subject: [PATCH 0081/1025] Make pycrypto optional. Connect #169. --- mycli/config.py | 14 +++++++++++++- mycli/main.py | 21 ++++++++++++--------- setup.py | 4 +++- tests/test_login_path.py | 15 ++++++++++++++- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index f22822435..8e9d9d627 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -5,7 +5,17 @@ from os.path import expanduser, exists import struct from configobj import ConfigObj -from Crypto.Cipher import AES +try: + from Crypto.Cipher import AES +except ImportError: + AES = None + + +class CryptoError(Exception): + """ + Exception to signal about pycrypto not available. + """ + pass logger = logging.getLogger(__name__) @@ -76,6 +86,8 @@ def read_and_decrypt_mylogin_cnf(f): :return: the decrypted login path file :rtype: io.BytesIO or None """ + if AES is None: + raise CryptoError('pycrypto is not available.') # Number of bytes used to store the length of ciphertext. MAX_CIPHER_STORE_LEN = 4 diff --git a/mycli/main.py b/mycli/main.py index 3bcef18b5..e91420c4f 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -36,7 +36,7 @@ from .clibuffer import CLIBuffer from .completion_refresher import CompletionRefresher from .config import (write_default_config, load_config, get_mylogin_cnf_path, - open_mylogin_cnf) + open_mylogin_cnf, CryptoError) from .key_bindings import mycli_bindings from .encodingutils import utf8tounicode from .lexer import MyCliLexer @@ -121,14 +121,17 @@ def __init__(self, sqlexecute=None, prompt=None, # Load .mylogin.cnf if it exists. mylogin_cnf_path = get_mylogin_cnf_path() if mylogin_cnf_path: - mylogin_cnf = open_mylogin_cnf(mylogin_cnf_path) - - if mylogin_cnf_path and mylogin_cnf: - # .mylogin.cnf gets read last, even if defaults_file is specified. - self.cnf_files.append(mylogin_cnf) - elif mylogin_cnf_path and not mylogin_cnf: - # There was an error reading the login path file. - print('Error: Unable to read login path file.') + try: + mylogin_cnf = open_mylogin_cnf(mylogin_cnf_path) + if mylogin_cnf_path and mylogin_cnf: + # .mylogin.cnf gets read last, even if defaults_file is specified. + self.cnf_files.append(mylogin_cnf) + elif mylogin_cnf_path and not mylogin_cnf: + # There was an error reading the login path file. + print('Error: Unable to read login path file.') + except CryptoError: + click.secho('Warning: .mylogin.cnf was not read: pycrypto ' + 'module is not available.') self.cli = None diff --git a/setup.py b/setup.py index d87c62576..f72485d29 100644 --- a/setup.py +++ b/setup.py @@ -29,8 +29,10 @@ 'PyMySQL >= 0.6.2', 'sqlparse >= 0.1.16', 'configobj >= 5.0.6', - 'pycrypto >= 2.6.1', ], + extras_require={ + 'parse_mylogin_cnf': ['pycrypto >= 2.6.1'] + }, entry_points=''' [console_scripts] mycli=mycli.main:cli diff --git a/tests/test_login_path.py b/tests/test_login_path.py index 3f02b4df4..b00d616d6 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -1,9 +1,12 @@ """Unit tests for mycli.config login path decryption.""" from io import BytesIO, TextIOWrapper import os +import sys import struct +import pytest -from mycli.config import open_mylogin_cnf, read_and_decrypt_mylogin_cnf +from mycli.config import open_mylogin_cnf, read_and_decrypt_mylogin_cnf, \ + CryptoError LOGIN_PATH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'mylogin.cnf')) @@ -17,6 +20,13 @@ def open_bmylogin_cnf(name): return buf +@pytest.mark.skipif('pycrypto' in sys.modules, reason='requires pycrypto missing') +def test_read_mylogin_cnf_without_crypto(): + with pytest.raises(CryptoError): + mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) + + +@pytest.mark.skipif('pycrypto' not in sys.modules, reason='requires pycrypto') def test_read_mylogin_cnf(): """Tests that a login path file can be read and decrypted.""" mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) @@ -28,12 +38,14 @@ def test_read_mylogin_cnf(): assert word in contents +@pytest.mark.skipif('pycrypto' not in sys.modules, reason='requires pycrypto') def test_decrypt_blank_mylogin_cnf(): """Test that a blank login path file is handled correctly.""" mylogin_cnf = read_and_decrypt_mylogin_cnf(BytesIO()) assert mylogin_cnf is None +@pytest.mark.skipif('pycrypto' not in sys.modules, reason='requires pycrypto') def test_corrupted_login_key(): """Test that a corrupted login path key is handled correctly.""" buf = open_bmylogin_cnf(LOGIN_PATH_FILE) @@ -50,6 +62,7 @@ def test_corrupted_login_key(): assert mylogin_cnf is None +@pytest.mark.skipif('pycrypto' not in sys.modules, reason='requires pycrypto') def test_corrupted_pad(): """Tests that a login path file with a corrupted pad is partially read.""" buf = open_bmylogin_cnf(LOGIN_PATH_FILE) From 13beaf400331a53fbc0354b3d25dfbd331049122 Mon Sep 17 00:00:00 2001 From: Iryna Cherniavska Date: Mon, 26 Oct 2015 11:59:58 -0700 Subject: [PATCH 0082/1025] Install pycrypto by default, but only if we're not on windows. --- setup.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index f72485d29..e503e2c4d 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ import re import ast +import platform from setuptools import setup, find_packages _version_re = re.compile(r'__version__\s+=\s+(.*)') @@ -10,6 +11,20 @@ description = 'CLI for MySQL Database. With auto-completion and syntax highlighting.' +install_requirements = [ + 'click >= 4.1', + 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? + 'prompt_toolkit==0.46', + 'PyMySQL >= 0.6.2', + 'sqlparse >= 0.1.16', + 'configobj >= 5.0.6', +] + +# pycrypto is a hard package to install on Windows, so we make it an optional +# dependency. When it's installed, we can read mylogin.cnf, when it is not +# available, we skip reading mylogin.cnf and print a warning message. +if platform.system() != 'Windows': + install_requirements.append('pycrypto >= 2.6.1') setup( name='mycli', @@ -22,17 +37,7 @@ package_data={'mycli': ['myclirc', '../AUTHORS', '../SPONSORS']}, description=description, long_description=description, - install_requires=[ - 'click >= 4.1', - 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? - 'prompt_toolkit==0.46', - 'PyMySQL >= 0.6.2', - 'sqlparse >= 0.1.16', - 'configobj >= 5.0.6', - ], - extras_require={ - 'parse_mylogin_cnf': ['pycrypto >= 2.6.1'] - }, + install_requires=install_requirements, entry_points=''' [console_scripts] mycli=mycli.main:cli From bc04403e8a1312fad32ecc0c5dafa232826a297e Mon Sep 17 00:00:00 2001 From: Iryna Cherniavska Date: Mon, 26 Oct 2015 12:18:07 -0700 Subject: [PATCH 0083/1025] Added pip to check for pycrypto (checking sys.modules worked on Mac but not on Linux, oh well). --- tests/test_login_path.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_login_path.py b/tests/test_login_path.py index b00d616d6..642751531 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -1,13 +1,16 @@ """Unit tests for mycli.config login path decryption.""" from io import BytesIO, TextIOWrapper import os -import sys +import pip import struct import pytest from mycli.config import open_mylogin_cnf, read_and_decrypt_mylogin_cnf, \ CryptoError +with_pycrypto = ['pycrypto' in set([package.project_name for package in + pip.get_installed_distributions()])] + LOGIN_PATH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'mylogin.cnf')) @@ -20,13 +23,13 @@ def open_bmylogin_cnf(name): return buf -@pytest.mark.skipif('pycrypto' in sys.modules, reason='requires pycrypto missing') +@pytest.mark.skipif(with_pycrypto, reason='requires pycrypto missing') def test_read_mylogin_cnf_without_crypto(): with pytest.raises(CryptoError): mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) -@pytest.mark.skipif('pycrypto' not in sys.modules, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') def test_read_mylogin_cnf(): """Tests that a login path file can be read and decrypted.""" mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) @@ -38,14 +41,14 @@ def test_read_mylogin_cnf(): assert word in contents -@pytest.mark.skipif('pycrypto' not in sys.modules, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') def test_decrypt_blank_mylogin_cnf(): """Test that a blank login path file is handled correctly.""" mylogin_cnf = read_and_decrypt_mylogin_cnf(BytesIO()) assert mylogin_cnf is None -@pytest.mark.skipif('pycrypto' not in sys.modules, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') def test_corrupted_login_key(): """Test that a corrupted login path key is handled correctly.""" buf = open_bmylogin_cnf(LOGIN_PATH_FILE) @@ -62,7 +65,7 @@ def test_corrupted_login_key(): assert mylogin_cnf is None -@pytest.mark.skipif('pycrypto' not in sys.modules, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') def test_corrupted_pad(): """Tests that a login path file with a corrupted pad is partially read.""" buf = open_bmylogin_cnf(LOGIN_PATH_FILE) From 954206df3ef858f046769ae6aab9ad9161c86fac Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Wed, 28 Oct 2015 10:38:26 -0200 Subject: [PATCH 0084/1025] Renamed argument "logfile" to "auditlog" --- mycli/main.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index e91420c4f..1a38fd96e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -71,10 +71,10 @@ class MyCli(object): ] def __init__(self, sqlexecute=None, prompt=None, - logfile=None, defaults_suffix=None, defaults_file=None, + auditlog=None, defaults_suffix=None, defaults_file=None, login_path=None): self.sqlexecute = sqlexecute - self.logfile = logfile + self.auditlog = auditlog self.defaults_suffix = defaults_suffix self.login_path = login_path @@ -423,10 +423,10 @@ def prompt_tokens(cli): try: logger.debug('sql: %r', document.text) - if self.logfile: - self.logfile.write('\n# %s\n' % datetime.now()) - self.logfile.write(document.text) - self.logfile.write('\n') + if self.auditlog: + self.auditlog.write('\n# %s\n' % datetime.now()) + self.auditlog.write(document.text) + self.auditlog.write('\n') successful = False start = time() res = sqlexecute.run(document.text) @@ -516,15 +516,15 @@ def prompt_tokens(cli): os.environ['PAGER'] = special.get_original_pager() def output(self, text, **kwargs): - if self.logfile: - self.logfile.write(utf8tounicode(text)) - self.logfile.write('\n') + if self.auditlog: + self.auditlog.write(utf8tounicode(text)) + self.auditlog.write('\n') click.secho(text, **kwargs) def output_via_pager(self, text): - if self.logfile: - self.logfile.write(text) - self.logfile.write('\n') + if self.auditlog: + self.auditlog.write(text) + self.auditlog.write('\n') click.echo_via_pager(text) def adjust_less_opts(self): @@ -597,7 +597,7 @@ def get_prompt(self, string): @click.option('-R', '--prompt', 'prompt', help='Prompt format (Default: "{0}")'.format( MyCli.default_prompt)) -@click.option('-l', '--logfile', type=click.File(mode='a', encoding='utf-8'), +@click.option('-a', '--auditlog', type=click.File(mode='a', encoding='utf-8'), help='Log every query and its results to a file.') @click.option('--defaults-group-suffix', type=str, help='Read config group with the specified suffix.') @@ -607,13 +607,13 @@ def get_prompt(self, string): help='Read this path from the login file.') @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, - version, prompt, logfile, defaults_group_suffix, defaults_file, + version, prompt, auditlog, defaults_group_suffix, defaults_file, login_path): if version: print('Version:', __version__) sys.exit(0) - mycli = MyCli(prompt=prompt, logfile=logfile, + mycli = MyCli(prompt=prompt, auditlog=auditlog, defaults_suffix=defaults_group_suffix, defaults_file=defaults_file, login_path=login_path) From ded2d49020cc55f2a6d31a118f27247dff2872f7 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Wed, 28 Oct 2015 10:39:11 -0200 Subject: [PATCH 0085/1025] Add audit_log config to default myclirc --- mycli/myclirc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mycli/myclirc b/mycli/myclirc index e226a88ce..a23c3aa68 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -23,6 +23,9 @@ log_file = ~/.mycli.log # and "DEBUG". log_level = INFO +# Log every query and its results to a file. +audit_log = ~/.mycli-audit.log + # Timing of sql statments and table rendering. timing = True From f422432a083d90b46fc13c87cbef6910593323f9 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Wed, 28 Oct 2015 10:43:04 -0200 Subject: [PATCH 0086/1025] Add validation for auditlog parameter --- mycli/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 1a38fd96e..c0a456e47 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -88,6 +88,7 @@ def __init__(self, sqlexecute=None, prompt=None, default_config = os.path.join(PACKAGE_ROOT, 'myclirc') write_default_config(default_config, '~/.myclirc') + # Load config. c = self.config = load_config('~/.myclirc', default_config) self.multi_line = c['main'].as_bool('multi_line') @@ -99,6 +100,10 @@ def __init__(self, sqlexecute=None, prompt=None, self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') + # audit log + if self.auditlog is None: + self.auditlog = open(os.path.expanduser(c['main']['audit_log']), 'a') + self.completion_refresher = CompletionRefresher() self.logger = logging.getLogger(__name__) From 9057ff1e64cf94a54d3194e8600c1605a71ba00d Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Wed, 28 Oct 2015 13:06:03 -0200 Subject: [PATCH 0087/1025] Revert "Renamed argument "logfile" to "auditlog"" This reverts commit 954206df3ef858f046769ae6aab9ad9161c86fac. --- mycli/main.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index c0a456e47..33687e3dc 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -71,10 +71,10 @@ class MyCli(object): ] def __init__(self, sqlexecute=None, prompt=None, - auditlog=None, defaults_suffix=None, defaults_file=None, + logfile=None, defaults_suffix=None, defaults_file=None, login_path=None): self.sqlexecute = sqlexecute - self.auditlog = auditlog + self.logfile = logfile self.defaults_suffix = defaults_suffix self.login_path = login_path @@ -428,10 +428,10 @@ def prompt_tokens(cli): try: logger.debug('sql: %r', document.text) - if self.auditlog: - self.auditlog.write('\n# %s\n' % datetime.now()) - self.auditlog.write(document.text) - self.auditlog.write('\n') + if self.logfile: + self.logfile.write('\n# %s\n' % datetime.now()) + self.logfile.write(document.text) + self.logfile.write('\n') successful = False start = time() res = sqlexecute.run(document.text) @@ -521,15 +521,15 @@ def prompt_tokens(cli): os.environ['PAGER'] = special.get_original_pager() def output(self, text, **kwargs): - if self.auditlog: - self.auditlog.write(utf8tounicode(text)) - self.auditlog.write('\n') + if self.logfile: + self.logfile.write(utf8tounicode(text)) + self.logfile.write('\n') click.secho(text, **kwargs) def output_via_pager(self, text): - if self.auditlog: - self.auditlog.write(text) - self.auditlog.write('\n') + if self.logfile: + self.logfile.write(text) + self.logfile.write('\n') click.echo_via_pager(text) def adjust_less_opts(self): @@ -602,7 +602,7 @@ def get_prompt(self, string): @click.option('-R', '--prompt', 'prompt', help='Prompt format (Default: "{0}")'.format( MyCli.default_prompt)) -@click.option('-a', '--auditlog', type=click.File(mode='a', encoding='utf-8'), +@click.option('-l', '--logfile', type=click.File(mode='a', encoding='utf-8'), help='Log every query and its results to a file.') @click.option('--defaults-group-suffix', type=str, help='Read config group with the specified suffix.') @@ -612,13 +612,13 @@ def get_prompt(self, string): help='Read this path from the login file.') @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, - version, prompt, auditlog, defaults_group_suffix, defaults_file, + version, prompt, logfile, defaults_group_suffix, defaults_file, login_path): if version: print('Version:', __version__) sys.exit(0) - mycli = MyCli(prompt=prompt, auditlog=auditlog, + mycli = MyCli(prompt=prompt, logfile=logfile, defaults_suffix=defaults_group_suffix, defaults_file=defaults_file, login_path=login_path) From 49f7f44aeb6c12f849060ff1ab1e231207b0c0a2 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Wed, 28 Oct 2015 13:10:11 -0200 Subject: [PATCH 0088/1025] Changed validation to use the `logfile` attribute --- mycli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 33687e3dc..d597048b2 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -101,8 +101,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') # audit log - if self.auditlog is None: - self.auditlog = open(os.path.expanduser(c['main']['audit_log']), 'a') + if self.logfile is None: + self.logfile = open(os.path.expanduser(c['main']['audit_log']), 'a') self.completion_refresher = CompletionRefresher() From 17b6f83e93e31d56829c31fd9e155a1968549f76 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 28 Oct 2015 21:34:52 -0700 Subject: [PATCH 0089/1025] Remove the whitespace removal from tabulate. --- mycli/packages/tabulate.py | 3 --- tests/test_tabulate.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 tests/test_tabulate.py diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index 6e92e8f15..a5911992f 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -442,10 +442,8 @@ def _align_column(strings, alignment, minwidth=0, has_invisible=True): """ if alignment == "right": - strings = [s.strip() for s in strings] padfn = _padleft elif alignment == "center": - strings = [s.strip() for s in strings] padfn = _padboth elif alignment == "decimal": decimals = [_afterpoint(s) for s in strings] @@ -456,7 +454,6 @@ def _align_column(strings, alignment, minwidth=0, has_invisible=True): elif not alignment: return strings else: - strings = [s.strip() for s in strings] padfn = _padright if has_invisible: diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py new file mode 100644 index 000000000..528255d1c --- /dev/null +++ b/tests/test_tabulate.py @@ -0,0 +1,14 @@ +from pgcli.packages.tabulate import tabulate +from textwrap import dedent + + +def test_dont_strip_leading_whitespace(): + data = [[' abc']] + headers = ['xyz'] + tbl, _ = tabulate(data, headers, tablefmt='psql') + assert tbl == dedent(''' + +---------+ + | xyz | + |---------| + | abc | + +---------+ ''').strip() From 3c6c242fd42bd9acf9866f458fa70536d56f3ccd Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 28 Oct 2015 21:38:14 -0700 Subject: [PATCH 0090/1025] Change the pgcli import to mycli. --- tests/test_tabulate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py index 528255d1c..351261971 100644 --- a/tests/test_tabulate.py +++ b/tests/test_tabulate.py @@ -1,4 +1,4 @@ -from pgcli.packages.tabulate import tabulate +from mycli.packages.tabulate import tabulate from textwrap import dedent From 1e393735cb81654a90caaa1b5e594c3ddb2192b4 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 29 Oct 2015 01:20:20 -0700 Subject: [PATCH 0091/1025] Fix the failing tests for tabulate. --- tests/test_tabulate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py index 351261971..0af232b6c 100644 --- a/tests/test_tabulate.py +++ b/tests/test_tabulate.py @@ -5,7 +5,7 @@ def test_dont_strip_leading_whitespace(): data = [[' abc']] headers = ['xyz'] - tbl, _ = tabulate(data, headers, tablefmt='psql') + tbl = tabulate(data, headers, tablefmt='psql') assert tbl == dedent(''' +---------+ | xyz | From 533a9b734b0155a3255df0dd4e4d116067d242da Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Thu, 29 Oct 2015 20:05:44 -0200 Subject: [PATCH 0092/1025] Make `audit_log` config optional --- mycli/main.py | 2 +- mycli/myclirc | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index d597048b2..73e605f7b 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -101,7 +101,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') # audit log - if self.logfile is None: + if self.logfile is None and 'audit_log' in c['main']: self.logfile = open(os.path.expanduser(c['main']['audit_log']), 'a') self.completion_refresher = CompletionRefresher() diff --git a/mycli/myclirc b/mycli/myclirc index a23c3aa68..9abef8818 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -23,8 +23,9 @@ log_file = ~/.mycli.log # and "DEBUG". log_level = INFO -# Log every query and its results to a file. -audit_log = ~/.mycli-audit.log +# Log every query and its results to a file. Enable this by uncommenting the +# line below. +# audit_log = ~/.mycli-audit.log # Timing of sql statments and table rendering. timing = True From 529dff66551170ca6e9f3e40ca72ed0f56d82518 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 30 Oct 2015 19:42:30 -0200 Subject: [PATCH 0093/1025] Add try/except in order to validate the existence of the auditlog file --- mycli/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 73e605f7b..860f227b4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -102,7 +102,10 @@ def __init__(self, sqlexecute=None, prompt=None, # audit log if self.logfile is None and 'audit_log' in c['main']: - self.logfile = open(os.path.expanduser(c['main']['audit_log']), 'a') + try: + self.logfile = open(os.path.expanduser(c['main']['audit_log']), 'a') + except (IOError, OSError) as e: + self.logfile = False self.completion_refresher = CompletionRefresher() From faa2202dc4db15dfa3f8bc0576c29caee5dc3e69 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Fri, 30 Oct 2015 19:46:48 -0200 Subject: [PATCH 0094/1025] Add error message for auditlog's validation --- mycli/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 860f227b4..c248f21f8 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -431,10 +431,12 @@ def prompt_tokens(cli): try: logger.debug('sql: %r', document.text) - if self.logfile: + if self.logfile is not False: self.logfile.write('\n# %s\n' % datetime.now()) self.logfile.write(document.text) self.logfile.write('\n') + else: + self.output("Error: Unable to load the audit log file.", err=True, fg='red') successful = False start = time() res = sqlexecute.run(document.text) From 3b0ec53de7dbe7cbb099d70ce7fefc478df8156c Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 31 Oct 2015 00:34:08 -0200 Subject: [PATCH 0095/1025] Add a error message when it fails to open the audit log file --- mycli/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/main.py b/mycli/main.py index c248f21f8..8a10efbd0 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -105,6 +105,7 @@ def __init__(self, sqlexecute=None, prompt=None, try: self.logfile = open(os.path.expanduser(c['main']['audit_log']), 'a') except (IOError, OSError) as e: + self.output('Error: Unable to open the audit log file.', err=True, fg='red') self.logfile = False self.completion_refresher = CompletionRefresher() From 59be492b12bda27f53410e187969dd0e731901a0 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 31 Oct 2015 00:38:28 -0200 Subject: [PATCH 0096/1025] Change if/else condition for `logfile` (when writing) and also the error message --- mycli/main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 8a10efbd0..6ac03559e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -432,12 +432,14 @@ def prompt_tokens(cli): try: logger.debug('sql: %r', document.text) - if self.logfile is not False: + + if self.logfile: self.logfile.write('\n# %s\n' % datetime.now()) self.logfile.write(document.text) self.logfile.write('\n') - else: - self.output("Error: Unable to load the audit log file.", err=True, fg='red') + elif self.logfile is False: + self.output("Error: Unable to write to the audit log file.", err=True, fg='red') + successful = False start = time() res = sqlexecute.run(document.text) From c827f98d4afec7eb8ede08e69b71ea238bce9440 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 1 Nov 2015 11:31:08 -0200 Subject: [PATCH 0097/1025] Improved error message for when audit log file could not be opened --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 6ac03559e..f50e7672f 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -105,7 +105,7 @@ def __init__(self, sqlexecute=None, prompt=None, try: self.logfile = open(os.path.expanduser(c['main']['audit_log']), 'a') except (IOError, OSError) as e: - self.output('Error: Unable to open the audit log file.', err=True, fg='red') + self.output('Error: Unable to open the audit log file. Your queries will not be logged.', err=True, fg='red') self.logfile = False self.completion_refresher = CompletionRefresher() From 494608861e25615ac8ec873de6069a3da359d33b Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 1 Nov 2015 11:42:21 -0200 Subject: [PATCH 0098/1025] Show auditlog warning message after each query --- mycli/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index f50e7672f..993853d5a 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -437,8 +437,6 @@ def prompt_tokens(cli): self.logfile.write('\n# %s\n' % datetime.now()) self.logfile.write(document.text) self.logfile.write('\n') - elif self.logfile is False: - self.output("Error: Unable to write to the audit log file.", err=True, fg='red') successful = False start = time() @@ -517,7 +515,9 @@ def prompt_tokens(cli): if need_completion_refresh(document.text): self.refresh_completions( reset=need_completion_reset(document.text)) - + finally: + if self.logfile is False: + self.output("Warning: This query was not logged.", err=True, fg='red') query = Query(document.text, successful, mutating) self.query_history.append(query) From ebecf2eaab0a4cb3fcc987d13bfddb3da7ee0683 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 2 Nov 2015 06:17:50 -0800 Subject: [PATCH 0099/1025] Add Matheus to the core dev list. --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index dbbca828b..895b6edf3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -6,6 +6,7 @@ Core Developers: * Iryna Cherniavska * Thomas Roten * Darik Gamble + * Matheus Rosa Contributors: ------------- From 6ac7b12e53ee3baf14f4a636594599cbd9af1181 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 3 Nov 2015 06:16:51 -0800 Subject: [PATCH 0100/1025] Update AUTHORS file. --- AUTHORS | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 895b6edf3..b5b6e6a2f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -12,18 +12,24 @@ Contributors: ------------- * Steve Robbins + * Daniel West + * shoma * Daniel Black * Jonathan Bruno * Heath Naylor - * Daniel West - * Abirami P + * bjarnagin + * darikg * jbruno + * Abirami P + * spacewander * Adam Chainz * Johannes Hoff * Jonathan Slenders + * Kacper Kwapisz + * Martijn Engler + * Shoma Suzuki * Tyler Kuipers * Yasuhiro Matsumoto - * bjarnagin Creator: -------- From eda57dbb680095d690c84d64ba0f4f76a6cae258 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 3 Nov 2015 08:15:21 -0800 Subject: [PATCH 0101/1025] Update changelog for release 1.5.0. --- AUTHORS | 1 - changelog.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b5b6e6a2f..6d86f1f86 100644 --- a/AUTHORS +++ b/AUTHORS @@ -18,7 +18,6 @@ Contributors: * Jonathan Bruno * Heath Naylor * bjarnagin - * darikg * jbruno * Abirami P * spacewander diff --git a/changelog.md b/changelog.md index ff6ec055e..9bfd28e12 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,54 @@ +1.5.0: +====== + +Features: +--------- + +* Make a config option to enable `audit_log`. (Thanks: [Matheus Rosa]). +* Add support for reading .mylogin.cnf to get user credentials. (Thanks: [Thomas Roten]). + This feature is only available when `pycrypto` package is installed. +* Register the special command `prompt` with the `\R` as alias. (Thanks: [Matheus Rosa]). + Users can now change the mysql prompt at runtime using `prompt` command. + eg: + ``` + mycli> prompt \u@\h> + Changed prompt format to \u@\h> + Time: 0.001s + amjith@localhost> + ``` +* Perform completion refresh in a background thread. Now mycli can handle + databases with thousands of tables without blocking. +* Add support for `system` command. (Thanks: [Matheus Rosa]). + Users can now run a system command from within mycli as follows: + ``` + amjith@localhost:(none)>system cat tmp.sql + select 1; + select * from django_migrations; + ``` +* Caught and hexed binary fields in MySQL. (Thanks: [Daniel West]). + Geometric fields stored in a database will be displayed as hexed strings. +* Treat enter key as tab when the suggestion menu is open. (Thanks: [Matheus Rosa]) +* Add "delete" and "truncate" as destructive commands. (Thanks: [Martijn Engler]). +* Change \dt syntax to add an optional table name. (Thanks: [shoma]). + `\dt [tablename]` will describe the columns in a table. +* Add TRANSACTION related keywords. +* Treat DESC and EXPLAIN as DESCRIBE. (Thanks: [spacewander]). + +Bug Fixes: +---------- + +* Fix the removal of whitespace from table output. (Thanks: [Amjith Ramanujam]). +* Add ability to make suggestions for compound join clauses. (Thanks: [Matheus Rosa]). +* Fix the incorrect reporting of command time. + +Internal Changes: +----------------- +* Make pycrypto optional and only install it in \*nix systems. (Thanks: [Iryna Cherniavska]). +* Add badge for PyPI version to README. (Thanks: [Shoma Suzuki]). +* Updated release script with a --dry-run and --confirm-steps option. (Thanks: [Iryna Cherniavska]). +* Adds support for PyMySQL 0.6.2 and above. This is useful for debian package builders. (Thanks: [Thomas Roten]). +* Disable click warning. + 1.4.0: ====== @@ -175,3 +226,12 @@ Features: Bug Fixes: ---------- * Fixed the installation issues with PyMySQL dependency on case-sensitive file systems. + +[Daniel West]: http://github.com/danieljwest +[Iryna Cherniavska]: https://github.com/j-bennet +[Kacper Kwapisz]: https://github.com/KKKas +[Martijn Engler]: https://github.com/martijnengler +[Matheus Rosa]: https://github.com/mdsrosa +[Shoma Suzuki]: https://github.com/shoma +[spacewander]: https://github.com/spacewander +[Thomas Roten]: https://github.com/tsroten From 9b4d48a83bb85b8bb0012a41b0b9735c278117d6 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 9 Nov 2015 09:57:05 -0200 Subject: [PATCH 0102/1025] Add type validation for `port` argument --- mycli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 993853d5a..d9b34acc0 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -289,7 +289,7 @@ def connect(self, database='', user='', passwd='', host='', port='', socket = socket or cnf['socket'] user = user or cnf['user'] or os.getenv('USER') host = host or cnf['host'] or 'localhost' - port = int(port or cnf['port'] or 3306) + port = port or cnf['port'] or 3306 passwd = passwd or cnf['password'] charset = charset or cnf['default-character-set'] or 'utf8' @@ -597,7 +597,7 @@ def get_prompt(self, string): @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') -@click.option('-P', '--port', envvar='MYSQL_TCP_PORT', help='Port number to use for connection. Honors ' +@click.option('-P', '--port', envvar='MYSQL_TCP_PORT', type=int, help='Port number to use for connection. Honors ' '$MYSQL_TCP_PORT') @click.option('-u', '--user', help='User name to connect to the database.') @click.option('-S', '--socket', envvar='MYSQL_UNIX_PORT', help='The socket file to use for connection.') From 0f19a1811e118f29724d9d4ecfd97d46734e1d78 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 9 Nov 2015 08:07:17 -0800 Subject: [PATCH 0103/1025] Add port number validation to changelog. --- changelog.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 9bfd28e12..dd1a0c5b4 100644 --- a/changelog.md +++ b/changelog.md @@ -37,9 +37,10 @@ Features: Bug Fixes: ---------- -* Fix the removal of whitespace from table output. (Thanks: [Amjith Ramanujam]). +* Fix the removal of whitespace from table output. * Add ability to make suggestions for compound join clauses. (Thanks: [Matheus Rosa]). * Fix the incorrect reporting of command time. +* Add type validation for port argument. (Thanks [Matheus Rosa]) Internal Changes: ----------------- From 2b236c334626d7563cd103d8fb2eba7772d9ed96 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 9 Nov 2015 08:07:47 -0800 Subject: [PATCH 0104/1025] Releasing version 1.5.0 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index 96e3ce8d9..77f1c8e63 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.4.0' +__version__ = '1.5.0' From 0657518dec6c5a40f520760a4bf0155f58ee8237 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 9 Nov 2015 08:17:59 -0800 Subject: [PATCH 0105/1025] Fix a broken link in changelog. --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index dd1a0c5b4..b93693f2e 100644 --- a/changelog.md +++ b/changelog.md @@ -29,7 +29,7 @@ Features: Geometric fields stored in a database will be displayed as hexed strings. * Treat enter key as tab when the suggestion menu is open. (Thanks: [Matheus Rosa]) * Add "delete" and "truncate" as destructive commands. (Thanks: [Martijn Engler]). -* Change \dt syntax to add an optional table name. (Thanks: [shoma]). +* Change \dt syntax to add an optional table name. (Thanks: [Shoma Suzuki]). `\dt [tablename]` will describe the columns in a table. * Add TRANSACTION related keywords. * Treat DESC and EXPLAIN as DESCRIBE. (Thanks: [spacewander]). From 940cd5835f555e90ac7f0f0444c2b27b1f9aa450 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 10 Nov 2015 20:31:47 -0800 Subject: [PATCH 0106/1025] Cast the value of port read from my.cnf to int. --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index d9b34acc0..fd176520e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -289,7 +289,7 @@ def connect(self, database='', user='', passwd='', host='', port='', socket = socket or cnf['socket'] user = user or cnf['user'] or os.getenv('USER') host = host or cnf['host'] or 'localhost' - port = port or cnf['port'] or 3306 + port = int(port or cnf['port']) or 3306 passwd = passwd or cnf['password'] charset = charset or cnf['default-character-set'] or 'utf8' From c3b03ae82559f18bf1b0758eb396cb9ee0dddd5a Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 11 Nov 2015 09:05:36 -0800 Subject: [PATCH 0107/1025] Update changelog for version 1.5.1 --- changelog.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/changelog.md b/changelog.md index b93693f2e..986547574 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,11 @@ +1.5.1: +====== + +Bug Fixes: +---------- + +* Cast the value of port read from my.cnf to int. + 1.5.0: ====== From 85f21e097aa77225e75cb11d4e501acd18a1de32 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 12 Nov 2015 20:13:13 -0800 Subject: [PATCH 0108/1025] Fix the port number casting bug. --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index fd176520e..815d3439e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -289,7 +289,7 @@ def connect(self, database='', user='', passwd='', host='', port='', socket = socket or cnf['socket'] user = user or cnf['user'] or os.getenv('USER') host = host or cnf['host'] or 'localhost' - port = int(port or cnf['port']) or 3306 + port = int(port or cnf['port'] or 3306) passwd = passwd or cnf['password'] charset = charset or cnf['default-character-set'] or 'utf8' From 6c1d9a9c175e4c0e58b99d89cb7c625db5b6d5cc Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 12 Nov 2015 20:17:01 -0800 Subject: [PATCH 0109/1025] Update changelog for release 1.5.2. --- changelog.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/changelog.md b/changelog.md index 986547574..7e3446b75 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,11 @@ +1.5.2: +====== + +Bug Fixes: +---------- + +* Protect against port number being None when no port is specified in command line. + 1.5.1: ====== From 29253c69a5aec598b66baa5379cea0659078aeed Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 13 Nov 2015 07:23:00 -0800 Subject: [PATCH 0110/1025] Handle invalid port validation exception. --- mycli/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 815d3439e..1436c8ed1 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -289,7 +289,12 @@ def connect(self, database='', user='', passwd='', host='', port='', socket = socket or cnf['socket'] user = user or cnf['user'] or os.getenv('USER') host = host or cnf['host'] or 'localhost' - port = int(port or cnf['port'] or 3306) + try: + port = int(port or cnf['port'] or 3306) + except ValueError as e: + self.output('Invalid port number. ' + str(e), err=True, fg='red') + exit(1) + passwd = passwd or cnf['password'] charset = charset or cnf['default-character-set'] or 'utf8' From 18d8f552da6c99586735843048e3e2587ac2ecfc Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 13 Nov 2015 07:24:48 -0800 Subject: [PATCH 0111/1025] Releasing version 1.5.2 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index 77f1c8e63..c3b384154 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.5.0' +__version__ = '1.5.2' From cba203b379db36712c89fde7b05a460cae3a8020 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 13 Nov 2015 21:40:26 -0600 Subject: [PATCH 0112/1025] Adds system-wide mycli config file. --- mycli/config.py | 35 +++++++++++++++++------ mycli/main.py | 24 +++++++++++----- mycli/packages/special/favoritequeries.py | 5 ++-- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 8e9d9d627..8429cc963 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -2,9 +2,9 @@ from io import BytesIO, TextIOWrapper import logging import os -from os.path import expanduser, exists +from os.path import exists import struct -from configobj import ConfigObj +from configobj import ConfigObj, ConfigObjError try: from Crypto.Cipher import AES except ImportError: @@ -19,16 +19,33 @@ class CryptoError(Exception): logger = logging.getLogger(__name__) -def load_config(usr_cfg, def_cfg=None): - cfg = ConfigObj() - cfg.merge(ConfigObj(def_cfg, interpolation=False)) - cfg.merge(ConfigObj(expanduser(usr_cfg), interpolation=False)) - cfg.filename = expanduser(usr_cfg) +def read_config_files(files, base_config=None): + """Read and merge a string or list of config files. - return cfg + If a file is read successfully, the config object takes on that + filename. + """ + + config = base_config or ConfigObj() + files = [files] if isinstance(files, str) else files + + for _file in files: + try: + _config = ConfigObj(_file, interpolation=False) + if bool(_config) is True: + config.filename = _file + config.merge(_config) + except ConfigObjError as e: + logger.error("Error parsing config file '{0}'.".format(_file)) + logger.error('Recovering partially parsed config values.') + config.merge(e.config) + except (IOError, PermissionError) as e: + logger.warning("You don't have permission to read config " + "file' {0}'.".format(e.filename)) + + return config def write_default_config(source, destination, overwrite=False): - destination = expanduser(destination) if not overwrite and exists(destination): return diff --git a/mycli/main.py b/mycli/main.py index 815d3439e..698b0b940 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -35,8 +35,8 @@ from .sqlexecute import SQLExecute from .clibuffer import CLIBuffer from .completion_refresher import CompletionRefresher -from .config import (write_default_config, load_config, get_mylogin_cnf_path, - open_mylogin_cnf, CryptoError) +from .config import (write_default_config, get_mylogin_cnf_path, + open_mylogin_cnf, CryptoError, read_config_files) from .key_bindings import mycli_bindings from .encodingutils import utf8tounicode from .lexer import MyCliLexer @@ -70,6 +70,14 @@ class MyCli(object): os.path.expanduser('~/.my.cnf') ] + system_config_files = [ + '/etc/myclirc', + ] + + default_config_file = os.path.join(PACKAGE_ROOT, 'myclirc') + user_config_file = os.path.expanduser('~/.myclirc') + + def __init__(self, sqlexecute=None, prompt=None, logfile=None, defaults_suffix=None, defaults_file=None, login_path=None): @@ -85,12 +93,10 @@ def __init__(self, sqlexecute=None, prompt=None, if defaults_file: self.cnf_files = [defaults_file] - default_config = os.path.join(PACKAGE_ROOT, 'myclirc') - write_default_config(default_config, '~/.myclirc') - - # Load config. - c = self.config = load_config('~/.myclirc', default_config) + c = self.config = ConfigObj(self.default_config_file) + read_config_files(self.system_config_files, base_config=c) + read_config_files(self.user_config_file, base_config=c) self.multi_line = c['main'].as_bool('multi_line') self.destructive_warning = c['main'].as_bool('destructive_warning') self.key_bindings = c['main']['key_bindings'] @@ -100,6 +106,10 @@ def __init__(self, sqlexecute=None, prompt=None, self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') + # Write user config if system config wasn't the last config loaded. + if c.filename not in self.system_config_files: + write_default_config(self.default_config_file, self.user_config_file) + # audit log if self.logfile is None and 'audit_log' in c['main']: try: diff --git a/mycli/packages/special/favoritequeries.py b/mycli/packages/special/favoritequeries.py index addd1fe72..3e88c7cae 100644 --- a/mycli/packages/special/favoritequeries.py +++ b/mycli/packages/special/favoritequeries.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +from os.path import expanduser class FavoriteQueries(object): @@ -57,5 +58,5 @@ def delete(self, name): self.config.write() return '%s: Deleted' % name -from ...config import load_config -favoritequeries = FavoriteQueries(load_config('~/.myclirc')) +from ...config import read_config_files +favoritequeries = FavoriteQueries(read_config_files(expanduser('~/.myclirc'))) From 8864857a0bf0496129db6cb1d1b497b1ccddd260 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 13 Nov 2015 21:46:46 -0600 Subject: [PATCH 0113/1025] Removes redendant cnf reading code. --- mycli/main.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 698b0b940..38400564f 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -251,15 +251,7 @@ def read_my_cnf_files(self, files, keys): :param keys: list of keys to retrieve :returns: tuple, with None for missing keys. """ - cnf = ConfigObj() - for _file in files: - try: - cnf.merge(ConfigObj(_file, interpolation=False)) - except ConfigObjError as e: - self.logger.error('Error parsing %r.', _file) - self.logger.error('Recovering partially parsed config values.') - cnf.merge(e.config) - pass + cnf = read_config_files(files) sections = ['client'] if self.login_path and self.login_path != 'client': From d9d64ce78e8fad358cf686f5cf486c33c3ab2f33 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 13 Nov 2015 22:15:32 -0600 Subject: [PATCH 0114/1025] Creates a read_config_file function. --- mycli/config.py | 43 +++++++++++++---------- mycli/main.py | 5 +-- mycli/packages/special/favoritequeries.py | 4 +-- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 8429cc963..931b27118 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -19,29 +19,36 @@ class CryptoError(Exception): logger = logging.getLogger(__name__) -def read_config_files(files, base_config=None): - """Read and merge a string or list of config files. +def read_config_file(f, base_config=None): + """Read and merge a config file. - If a file is read successfully, the config object takes on that - filename. + If the file is read successfully, the config object takes + on that filename. """ - config = base_config or ConfigObj() - files = [files] if isinstance(files, str) else files + config = ConfigObj() if base_config is None else base_config + try: + _config = ConfigObj(f, interpolation=False) + config.merge(_config) + if bool(_config) is True: + config.filename = f + except ConfigObjError as e: + logger.error("Error parsing config file '{0}'.".format(f)) + logger.error('Recovering partially parsed config values.') + config.merge(e.config) + except (IOError, PermissionError) as e: + logger.warning("You don't have permission to read config " + "file '{0}'.".format(e.filename)) + + return config + +def read_config_files(files, base_config=None): + """Read and merge a list of config files.""" + + config = ConfigObj() if base_config is None else base_config for _file in files: - try: - _config = ConfigObj(_file, interpolation=False) - if bool(_config) is True: - config.filename = _file - config.merge(_config) - except ConfigObjError as e: - logger.error("Error parsing config file '{0}'.".format(_file)) - logger.error('Recovering partially parsed config values.') - config.merge(e.config) - except (IOError, PermissionError) as e: - logger.warning("You don't have permission to read config " - "file' {0}'.".format(e.filename)) + read_config_file(_file, base_config=config) return config diff --git a/mycli/main.py b/mycli/main.py index 38400564f..19d51a9e1 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -36,7 +36,8 @@ from .clibuffer import CLIBuffer from .completion_refresher import CompletionRefresher from .config import (write_default_config, get_mylogin_cnf_path, - open_mylogin_cnf, CryptoError, read_config_files) + open_mylogin_cnf, CryptoError, read_config_file, + read_config_files) from .key_bindings import mycli_bindings from .encodingutils import utf8tounicode from .lexer import MyCliLexer @@ -96,7 +97,7 @@ def __init__(self, sqlexecute=None, prompt=None, # Load config. c = self.config = ConfigObj(self.default_config_file) read_config_files(self.system_config_files, base_config=c) - read_config_files(self.user_config_file, base_config=c) + read_config_file(self.user_config_file, base_config=c) self.multi_line = c['main'].as_bool('multi_line') self.destructive_warning = c['main'].as_bool('destructive_warning') self.key_bindings = c['main']['key_bindings'] diff --git a/mycli/packages/special/favoritequeries.py b/mycli/packages/special/favoritequeries.py index 3e88c7cae..a8bd6d416 100644 --- a/mycli/packages/special/favoritequeries.py +++ b/mycli/packages/special/favoritequeries.py @@ -58,5 +58,5 @@ def delete(self, name): self.config.write() return '%s: Deleted' % name -from ...config import read_config_files -favoritequeries = FavoriteQueries(read_config_files(expanduser('~/.myclirc'))) +from ...config import read_config_file +favoritequeries = FavoriteQueries(read_config_file(expanduser('~/.myclirc'))) From 68aad93dcc8aad47ff4aa9978cd734631b83e47f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 13 Nov 2015 22:17:09 -0600 Subject: [PATCH 0115/1025] Makes reading default config use read_config_file function. --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 19d51a9e1..fecf28053 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -95,7 +95,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.cnf_files = [defaults_file] # Load config. - c = self.config = ConfigObj(self.default_config_file) + c = self.config = read_config_file(self.default_config_file) read_config_files(self.system_config_files, base_config=c) read_config_file(self.user_config_file, base_config=c) self.multi_line = c['main'].as_bool('multi_line') From d5a8bab886d246b5da218c44d2ff5914129100ed Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 13 Nov 2015 23:40:50 -0600 Subject: [PATCH 0116/1025] Updates invalid port error message. --- mycli/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 1436c8ed1..af12f2a15 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -289,10 +289,12 @@ def connect(self, database='', user='', passwd='', host='', port='', socket = socket or cnf['socket'] user = user or cnf['user'] or os.getenv('USER') host = host or cnf['host'] or 'localhost' + port = port or cnf['port'] or 3306 try: - port = int(port or cnf['port'] or 3306) + port = int(port) except ValueError as e: - self.output('Invalid port number. ' + str(e), err=True, fg='red') + self.output("Error: Invalid port number: '{0}'.".format(port), + err=True, fg='red') exit(1) passwd = passwd or cnf['password'] From 0f843c0b0eced191f2caee00531bc8a570bc2559 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 14 Nov 2015 14:07:13 -0600 Subject: [PATCH 0117/1025] Removes the passed ConfigObj object from the config file functions. --- mycli/config.py | 28 ++++++++++++---------------- mycli/main.py | 6 +++--- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 931b27118..4f57d76db 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -19,36 +19,32 @@ class CryptoError(Exception): logger = logging.getLogger(__name__) -def read_config_file(f, base_config=None): - """Read and merge a config file. +def read_config_file(f): + """Read a config file.""" - If the file is read successfully, the config object takes - on that filename. - """ - - config = ConfigObj() if base_config is None else base_config try: - _config = ConfigObj(f, interpolation=False) - config.merge(_config) - if bool(_config) is True: - config.filename = f + config = ConfigObj(f, interpolation=False) except ConfigObjError as e: logger.error("Error parsing config file '{0}'.".format(f)) logger.error('Recovering partially parsed config values.') - config.merge(e.config) - except (IOError, PermissionError) as e: + return e.config + except (IOError, OSError) as e: logger.warning("You don't have permission to read config " "file '{0}'.".format(e.filename)) + return None return config -def read_config_files(files, base_config=None): +def read_config_files(files): """Read and merge a list of config files.""" - config = ConfigObj() if base_config is None else base_config + config = ConfigObj() for _file in files: - read_config_file(_file, base_config=config) + _config = read_config_file(_file) + if bool(_config) is True: + config.merge(_config) + config.filename = _file return config diff --git a/mycli/main.py b/mycli/main.py index fecf28053..768d94f2c 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -95,9 +95,9 @@ def __init__(self, sqlexecute=None, prompt=None, self.cnf_files = [defaults_file] # Load config. - c = self.config = read_config_file(self.default_config_file) - read_config_files(self.system_config_files, base_config=c) - read_config_file(self.user_config_file, base_config=c) + config_files = ([self.default_config_file] + self.system_config_files + + [self.user_config_file]) + c = self.config = read_config_files(config_files) self.multi_line = c['main'].as_bool('multi_line') self.destructive_warning = c['main'].as_bool('destructive_warning') self.key_bindings = c['main']['key_bindings'] From b5819ca91d4a6be0073bbd3f52e042fe4632c748 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 14 Nov 2015 14:39:32 -0600 Subject: [PATCH 0118/1025] Makes config errors print to stderr. --- mycli/config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 4f57d76db..fe61aabc5 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -1,9 +1,11 @@ +from __future__ import print_function import shutil from io import BytesIO, TextIOWrapper import logging import os from os.path import exists import struct +import sys from configobj import ConfigObj, ConfigObjError try: from Crypto.Cipher import AES @@ -25,12 +27,12 @@ def read_config_file(f): try: config = ConfigObj(f, interpolation=False) except ConfigObjError as e: - logger.error("Error parsing config file '{0}'.".format(f)) - logger.error('Recovering partially parsed config values.') + print("Error parsing config file '{0}'.".format(f), file=sys.stderr) + print('Recovering partially parsed config values.', file=sys.stderr) return e.config except (IOError, OSError) as e: - logger.warning("You don't have permission to read config " - "file '{0}'.".format(e.filename)) + print("You don't have permission to read config file " + "'{0}'.".format(e.filename), file=sys.stderr) return None return config From 4cc4a92d4659b52182a09b2b7940519b6b516702 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 14 Nov 2015 14:46:44 -0600 Subject: [PATCH 0119/1025] Adds line number to broken config error message. --- mycli/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/config.py b/mycli/config.py index fe61aabc5..f650a9fdd 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -27,7 +27,8 @@ def read_config_file(f): try: config = ConfigObj(f, interpolation=False) except ConfigObjError as e: - print("Error parsing config file '{0}'.".format(f), file=sys.stderr) + print("Error parsing line {0} of config file '{1}'.".format( + e.line_number, f), file=sys.stderr) print('Recovering partially parsed config values.', file=sys.stderr) return e.config except (IOError, OSError) as e: From e7838a4f91c073789fe203e3bfc159daedae469d Mon Sep 17 00:00:00 2001 From: Casper Langemeijer Date: Sun, 15 Nov 2015 10:50:58 +0100 Subject: [PATCH 0120/1025] New debian release (1.5.2) --- debian/changelog | 27 +++++++++++++++++++++++++++ debian/control | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index b44bdb458..18da86c59 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,30 @@ +mycli (1.5.2) unstable; urgency=low + + * Protect against port number being None when no port is specified in command line. + * Cast the value of port read from my.cnf to int. + * Make a config option to enable `audit_log`. (Thanks: [Matheus Rosa]). + * Add support for reading .mylogin.cnf to get user credentials. (Thanks: [Thomas Roten]). + * Register the special command `prompt` with the `\R` as alias. (Thanks: [Matheus Rosa]). + * Perform completion refresh in a background thread. Now mycli can handle + * Add support for `system` command. (Thanks: [Matheus Rosa]). + * Caught and hexed binary fields in MySQL. (Thanks: [Daniel West]). + * Treat enter key as tab when the suggestion menu is open. (Thanks: [Matheus Rosa]) + * Add "delete" and "truncate" as destructive commands. (Thanks: [Martijn Engler]). + * Change \dt syntax to add an optional table name. (Thanks: [Shoma Suzuki]). + * Add TRANSACTION related keywords. + * Treat DESC and EXPLAIN as DESCRIBE. (Thanks: [spacewander]). + * Fix the removal of whitespace from table output. + * Add ability to make suggestions for compound join clauses. (Thanks: [Matheus Rosa]). + * Fix the incorrect reporting of command time. + * Add type validation for port argument. (Thanks [Matheus Rosa]) + * Make pycrypto optional and only install it in \*nix systems. (Thanks: [Iryna Cherniavska]). + * Add badge for PyPI version to README. (Thanks: [Shoma Suzuki]). + * Updated release script with a --dry-run and --confirm-steps option. (Thanks: [Iryna Cherniavska]). + * Adds support for PyMySQL 0.6.2 and above. This is useful for debian package builders. (Thanks: [Thomas Roten]). + * Disable click warning. + + -- Casper Langemeijer Sun, 15 Nov 2015 10:26:24 +0100 + mycli (1.4.0) unstable; urgency=low * Add `source` command. This allows running sql statement from a file. diff --git a/debian/control b/debian/control index f34b48981..f1602021a 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: mycli Section: python Priority: extra Maintainer: Amjith Ramanujam -Build-Depends: debhelper (>= 9), python, dh-virtualenv (>= 0.7) +Build-Depends: debhelper (>= 9), python, dh-virtualenv (>= 0.7), python-setuptools, python-dev Standards-Version: 3.9.5 Package: mycli From 136c2b4b85e8ae897b0272e0eb13adf96eef80d8 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 15 Nov 2015 18:36:39 -0600 Subject: [PATCH 0121/1025] Adds function for outputting config-related errors. --- mycli/config.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index f650a9fdd..0fc438c10 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -21,19 +21,27 @@ class CryptoError(Exception): logger = logging.getLogger(__name__) +def log(logger, level, message): + """Logs message to stderr if logging isn't initialized.""" + + if logger.parent.name != 'root': + logger.log(level, message) + else: + print(message, file=sys.stderr) + def read_config_file(f): """Read a config file.""" try: config = ConfigObj(f, interpolation=False) except ConfigObjError as e: - print("Error parsing line {0} of config file '{1}'.".format( - e.line_number, f), file=sys.stderr) - print('Recovering partially parsed config values.', file=sys.stderr) + log(logger, logging.ERROR, "Unable to parse line {0} of config file " + "'{1}'.".format(e.line_number, f)) + log(logger, logging.ERROR, "Using successfully parsed config values.") return e.config except (IOError, OSError) as e: - print("You don't have permission to read config file " - "'{0}'.".format(e.filename), file=sys.stderr) + log(logger, logging.WARNING, "You don't have permission to read " + "config file '{0}'.".format(e.filename)) return None return config From b1a5df139c3564cd7f237eb1ae0fa50a3c68b674 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 15 Nov 2015 21:56:05 -0600 Subject: [PATCH 0122/1025] Makes config file functions call expanduser. --- mycli/config.py | 10 +++++++++- mycli/main.py | 4 ++-- mycli/packages/special/favoritequeries.py | 3 +-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 0fc438c10..0d0dc2e9c 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -7,6 +7,10 @@ import struct import sys from configobj import ConfigObj, ConfigObjError +try: + basestring +except NameError: + basestring = str try: from Crypto.Cipher import AES except ImportError: @@ -32,6 +36,9 @@ def log(logger, level, message): def read_config_file(f): """Read a config file.""" + if isinstance(f, basestring): + f = os.path.expanduser(f) + try: config = ConfigObj(f, interpolation=False) except ConfigObjError as e: @@ -55,11 +62,12 @@ def read_config_files(files): _config = read_config_file(_file) if bool(_config) is True: config.merge(_config) - config.filename = _file + config.filename = _config.filename return config def write_default_config(source, destination, overwrite=False): + destination = os.path.expanduser(destination) if not overwrite and exists(destination): return diff --git a/mycli/main.py b/mycli/main.py index 768d94f2c..ec8e6a829 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -68,7 +68,7 @@ class MyCli(object): '/etc/my.cnf', '/etc/mysql/my.cnf', '/usr/local/etc/my.cnf', - os.path.expanduser('~/.my.cnf') + '~/.my.cnf' ] system_config_files = [ @@ -76,7 +76,7 @@ class MyCli(object): ] default_config_file = os.path.join(PACKAGE_ROOT, 'myclirc') - user_config_file = os.path.expanduser('~/.myclirc') + user_config_file = '~/.myclirc' def __init__(self, sqlexecute=None, prompt=None, diff --git a/mycli/packages/special/favoritequeries.py b/mycli/packages/special/favoritequeries.py index a8bd6d416..bec14ecb8 100644 --- a/mycli/packages/special/favoritequeries.py +++ b/mycli/packages/special/favoritequeries.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from os.path import expanduser class FavoriteQueries(object): @@ -59,4 +58,4 @@ def delete(self, name): return '%s: Deleted' % name from ...config import read_config_file -favoritequeries = FavoriteQueries(read_config_file(expanduser('~/.myclirc'))) +favoritequeries = FavoriteQueries(read_config_file('~/.myclirc')) From 1299f1476e788b7e9c35e20f5171380aed94ce14 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 25 Nov 2015 23:48:03 -0800 Subject: [PATCH 0123/1025] Add --auto-vertical-output option. --- mycli/main.py | 43 ++++++++++++++++++++++++++++++-------- mycli/packages/tabulate.py | 4 +++- tests/test_main.py | 21 +++++++++++++++++++ 3 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 tests/test_main.py diff --git a/mycli/main.py b/mycli/main.py index c32147937..31b8eea44 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -81,11 +81,12 @@ class MyCli(object): def __init__(self, sqlexecute=None, prompt=None, logfile=None, defaults_suffix=None, defaults_file=None, - login_path=None): + login_path=None, auto_vertical_output=False): self.sqlexecute = sqlexecute self.logfile = logfile self.defaults_suffix = defaults_suffix self.login_path = login_path + self.auto_vertical_output = auto_vertical_output # self.cnf_files is a class variable that stores the list of mysql # config files to read in at launch. @@ -466,8 +467,17 @@ def prompt_tokens(cli): if not click.confirm('Do you want to continue?'): self.output("Aborted!", err=True, fg='red') break - output.extend(format_output(title, cur, headers, - status, self.table_format)) + + if self.auto_vertical_output: + max_width = self.cli.output.get_size().columns + else: + max_width = None + + formatted = format_output(title, cur, headers, + status, self.table_format, + special.is_expanded_output(), max_width) + + output.extend(formatted) end = time() total += end - start mutating = mutating or is_mutating(status) @@ -607,6 +617,7 @@ def get_prompt(self, string): @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') +@click.option('--auto-vertical-output', is_flag=True, help='Automatically switch to vertical output mode if the result is wider than the terminal width.') @click.option('-P', '--port', envvar='MYSQL_TCP_PORT', type=int, help='Port number to use for connection. Honors ' '$MYSQL_TCP_PORT') @click.option('-u', '--user', help='User name to connect to the database.') @@ -631,14 +642,15 @@ def get_prompt(self, string): @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, - login_path): + login_path, auto_vertical_output): if version: print('Version:', __version__) sys.exit(0) mycli = MyCli(prompt=prompt, logfile=logfile, defaults_suffix=defaults_group_suffix, - defaults_file=defaults_file, login_path=login_path) + defaults_file=defaults_file, login_path=login_path, + auto_vertical_output=auto_vertical_output) # Choose which ever one has a valid value. database = database or dbname @@ -656,21 +668,34 @@ def cli(database, user, host, port, socket, password, dbname, mycli.run_cli() -def format_output(title, cur, headers, status, table_format): +def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): output = [] if title: # Only print the title if it's not None. output.append(title) if cur: headers = [utf8tounicode(x) for x in headers] - if special.is_expanded_output(): + if expanded: output.append(expanded_table(cur, headers)) else: - output.append(tabulate(cur, headers, tablefmt=table_format, - missingval='')) + tabulated, rows = tabulate(cur, headers, tablefmt=table_format, + missingval='') + if (max_width and rows and + content_exceeds_width(rows[0], max_width) and + headers): + output.append(expanded_table(rows, headers)) + else: + output.append(tabulated) if status: # Only print the status if it's not None. output.append(status) return output +def content_exceeds_width(row, width): + # Account for 3 characters between each column + separator_space = (len(row)*3) + # Add 2 columns for a bit of buffer + line_len = sum([len(str(x)) for x in row]) + separator_space + 2 + return line_len > width + def need_completion_refresh(queries): """Determines if the completion needs a refresh by checking if the sql statement is an alter, create, drop or change db.""" diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index a5911992f..f0874d525 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -881,6 +881,8 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", eggs & 451 \\\\ \\bottomrule \end{tabular} + + Also returns a tuple of the raw rows pulled from tabular_data """ if tabular_data is None: tabular_data = [] @@ -923,7 +925,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", if not isinstance(tablefmt, TableFormat): tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) - return _format_table(tablefmt, headers, rows, minwidths, aligns) + return _format_table(tablefmt, headers, rows, minwidths, aligns), rows def _build_simple_row(padded_cells, rowfmt): diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 000000000..247c863d8 --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,21 @@ +import pytest +from mycli.main import format_output + +def test_format_output(): + results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], + 'test status', 'psql') + expected = ['Title', '+---------+---------+\n| head1 | head2 |\n|---------+---------|\n| abc | def |\n+---------+---------+', 'test status'] + assert results == expected + +def test_format_output_auto_expand(): + table_results = format_output('Title', [('abc', 'def')], + ['head1', 'head2'], 'test status', 'psql', + max_width=100) + table = ['Title', '+---------+---------+\n| head1 | head2 |\n|---------+---------|\n| abc | def |\n+---------+---------+', 'test status'] + assert table_results == table + + expanded_results = format_output('Title', [('abc', 'def')], + ['head1', 'head2'], 'test status', 'psql', + max_width=1) + expanded = ['Title', u'***************************[ 1. row ]***************************\nhead1 | abc \nhead2 | def \n', 'test status'] + assert expanded_results == expanded From a24c8e605bfc3ddcf657d04118502946f39da61d Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 26 Nov 2015 03:07:47 -0800 Subject: [PATCH 0124/1025] Update tests to accomodate for new format_output signature. --- tests/test_expanded.py | 1 - tests/test_tabulate.py | 2 +- tests/utils.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_expanded.py b/tests/test_expanded.py index a06009a6b..9b2a6c5cf 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -11,4 +11,3 @@ def test_expanded_table_renders(): age | 456 """ assert expected == expanded_table(input, ["name", "age"]) - diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py index 0af232b6c..351261971 100644 --- a/tests/test_tabulate.py +++ b/tests/test_tabulate.py @@ -5,7 +5,7 @@ def test_dont_strip_leading_whitespace(): data = [[' abc']] headers = ['xyz'] - tbl = tabulate(data, headers, tablefmt='psql') + tbl, _ = tabulate(data, headers, tablefmt='psql') assert tbl == dedent(''' +---------+ | xyz | diff --git a/tests/utils.py b/tests/utils.py index 876bd199b..5d0ccbec6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -35,7 +35,7 @@ def run(executor, sql, join=False): " Return string output for the sql to be run " result = [] for title, rows, headers, status in executor.run(sql): - result.extend(format_output(title, rows, headers, status, 'psql')) + result.extend(format_output(title, rows, headers, status, 'psql', special.is_expanded_output())) if join: result = '\n'.join(result) return result From 8801c2a21d6da64b27b631e378e4b911917b0e8f Mon Sep 17 00:00:00 2001 From: William GARCIA Date: Sat, 28 Nov 2015 09:20:12 +0100 Subject: [PATCH 0125/1025] Fix #206 - Debian build: fixing missing dependencies, .gitignore --- .gitignore | 4 +++- Vagrantfile | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 619cd103b..0cab00de1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ /mycli.egg-info /src -*.pyc \ No newline at end of file +.vagrant +*.pyc +*.deb diff --git a/Vagrantfile b/Vagrantfile index e873c2fd5..a514d1efd 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,7 +13,7 @@ Vagrant.configure(2) do |config| sudo echo "deb http://ppa.launchpad.net/spotify-jyrki/dh-virtualenv/ubuntu trusty main" >> /etc/apt/sources.list sudo echo "deb-src http://ppa.launchpad.net/spotify-jyrki/dh-virtualenv/ubuntu trusty main" >> /etc/apt/sources.list sudo apt-get update - sudo apt-get install -y --force-yes python-virtualenv dh-virtualenv debhelper build-essential + sudo apt-get install -y --force-yes python-virtualenv dh-virtualenv debhelper build-essential python-setuptools python-dev echo "-> Cleaning up old workspace" rm -rf build mkdir -p build From e685639f053d45db4e741d6999e50726ced7bf6c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 29 Nov 2015 21:49:47 -0600 Subject: [PATCH 0126/1025] Makes auto-expand use unformatted rows for output. --- mycli/main.py | 5 +++-- tests/test_main.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 31b8eea44..6c6089bcf 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -677,10 +677,11 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max if expanded: output.append(expanded_table(cur, headers)) else: - tabulated, rows = tabulate(cur, headers, tablefmt=table_format, + rows = list(cur) + tabulated, frows = tabulate(rows, headers, tablefmt=table_format, missingval='') if (max_width and rows and - content_exceeds_width(rows[0], max_width) and + content_exceeds_width(frows[0], max_width) and headers): output.append(expanded_table(rows, headers)) else: diff --git a/tests/test_main.py b/tests/test_main.py index 247c863d8..de7737644 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -17,5 +17,5 @@ def test_format_output_auto_expand(): expanded_results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], 'test status', 'psql', max_width=1) - expanded = ['Title', u'***************************[ 1. row ]***************************\nhead1 | abc \nhead2 | def \n', 'test status'] + expanded = ['Title', u'***************************[ 1. row ]***************************\nhead1 | abc\nhead2 | def\n', 'test status'] assert expanded_results == expanded From 31ebe841aaf34fb557a78923ea3f2ed8d7effa6e Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 30 Nov 2015 04:55:46 -0800 Subject: [PATCH 0127/1025] Move the --auto-vertical-output option down the list in help message. --- mycli/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 31b8eea44..fdf20e7d4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -617,7 +617,6 @@ def get_prompt(self, string): @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') -@click.option('--auto-vertical-output', is_flag=True, help='Automatically switch to vertical output mode if the result is wider than the terminal width.') @click.option('-P', '--port', envvar='MYSQL_TCP_PORT', type=int, help='Port number to use for connection. Honors ' '$MYSQL_TCP_PORT') @click.option('-u', '--user', help='User name to connect to the database.') @@ -637,6 +636,8 @@ def get_prompt(self, string): help='Read config group with the specified suffix.') @click.option('--defaults-file', type=click.Path(), help='Only read default options from the given file') +@click.option('--auto-vertical-output', is_flag=True, + help='Automatically switch to vertical output mode if the result is wider than the terminal width.') @click.option('--login-path', type=str, help='Read this path from the login file.') @click.argument('database', default='', nargs=1) From 1288031f2771ea126890e3d687f540337609b93b Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 30 Nov 2015 15:23:34 -0200 Subject: [PATCH 0128/1025] Update README with new auto-vertical-output option --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e20b209a3..9d5190c85 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,9 @@ Check the [detailed install instructions](#detailed-install-instructions) for de -l, --logfile FILENAME Log every query and its results to a file. --defaults-group-suffix TEXT Read config group with the specified suffix. --defaults-file PATH Only read default options from the given file + --auto-vertical-output Automatically switch to vertical output mode + if the result is wider than the terminal + width. --login-path TEXT Read this path from the login file. --help Show this message and exit. From 6b94eb9ab047fbd9f52e232dcc45c9e4f8f5ec0b Mon Sep 17 00:00:00 2001 From: Terseus Date: Wed, 9 Dec 2015 22:15:58 +0100 Subject: [PATCH 0129/1025] Fix #203 - Support \G terminator for \f queries --- mycli/sqlexecute.py | 17 ++++++++--------- tests/test_sqlexecute.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 53918e6c1..d7b1bf53c 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -98,15 +98,14 @@ def run(self, statement): # and then proceed to execute the sql as normal. if sql.endswith('\\G'): special.set_expanded_output(True) - yield self.execute_normal_sql(sql.rsplit('\\G', 1)[0]) - else: - try: # Special command - _logger.debug('Trying a dbspecial command. sql: %r', sql) - cur = self.conn.cursor() - for result in special.execute(cur, sql): - yield result - except special.CommandNotFound: # Regular SQL - yield self.execute_normal_sql(sql) + sql = sql[:-2].strip() + try: # Special command + _logger.debug('Trying a dbspecial command. sql: %r', sql) + cur = self.conn.cursor() + for result in special.execute(cur, sql): + yield result + except special.CommandNotFound: # Regular SQL + yield self.execute_normal_sql(sql) def execute_normal_sql(self, split_sql): _logger.debug('Regular sql statement. sql: %r', split_sql) diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index d57c22edc..286e172db 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -175,6 +175,36 @@ def test_favorite_query_multiple_statement(executor): results = run(executor, "\\fd test-ad") assert results == ['test-ad: Deleted'] +@dbtest +def test_favorite_query_expanded_output(executor): + set_expanded_output(False) + run(executor, '''create table test(a text)''') + run(executor, '''insert into test values('abc')''') + + results = run(executor, "\\fs test-ae select * from test") + assert results == ['Saved.'] + + results = run(executor, "\\f test-ae \G", join=True) + + expected_results = set([ + dedent("""\ + > select * from test + -[ RECORD 0 ] + a | abc + """), + dedent("""\ + > select * from test + ***************************[ 1. row ]*************************** + a | abc + """), + ]) + set_expanded_output(False) + + assert results in expected_results + + results = run(executor, "\\fd test-ae") + assert results == ['test-ae: Deleted'] + @dbtest def test_special_command(executor): results = run(executor, '\\?') From de792680f1fab092c6da2ca11ab915d93121b34c Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Wed, 9 Dec 2015 23:18:13 -0200 Subject: [PATCH 0130/1025] Add more features to `Features` section in README --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d5190c85..af6c67498 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Features `mycli` is written using [prompt_toolkit](https://github.com/jonathanslenders/python-prompt-toolkit/). -* Auto-completion as you type for SQL keywords as well as tables and +* Auto-completion as you type for SQL keywords as well as tables, views and columns in the database. * Syntax highlighting using Pygments. * Smart-completion (enabled by default) will suggest context-sensitive completion. @@ -80,7 +80,16 @@ Features - `SELECT * FROM ` will only show table names. - `SELECT * FROM users WHERE ` will only show column names. +* Support for multiline queries. + +* Favorite queries. Save a query using `\fs alias query` and execute it with `\f alias` whenever you need. + +* Timing of sql statments and table rendering. + * Config file is automatically created at ``~/.myclirc`` at first launch. + +* Log every query and its results to a file (disabled by default). + * Pretty prints tabular data. Contributions: From 7d8556821d7ee1ea99261e42f82f2f94d98751a8 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 11 Dec 2015 05:27:42 -0800 Subject: [PATCH 0131/1025] Remove the obselete comment. --- mycli/sqlexecute.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index d7b1bf53c..2ae56d8e9 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -94,8 +94,7 @@ def run(self, statement): # Remove spaces, eol and semi-colons. sql = sql.rstrip(';') - # \G is treated specially since we have to set the expanded output - # and then proceed to execute the sql as normal. + # \G is treated specially since we have to set the expanded output. if sql.endswith('\\G'): special.set_expanded_output(True) sql = sql[:-2].strip() From 4e39bbc0eba5232e169f84e87e70ffcecbc8bbd9 Mon Sep 17 00:00:00 2001 From: shoma Date: Thu, 17 Dec 2015 10:24:27 +0900 Subject: [PATCH 0132/1025] Remove duplicated my name from AUTHORS I'm listed as `Shoma Suzuki` --- AUTHORS | 1 - 1 file changed, 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 6d86f1f86..0c566da43 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,7 +13,6 @@ Contributors: * Steve Robbins * Daniel West - * shoma * Daniel Black * Jonathan Bruno * Heath Naylor From a8c57b511f2bbc15647dadb4b4e92a3b5ec950ac Mon Sep 17 00:00:00 2001 From: Phil Cohen Date: Tue, 29 Dec 2015 11:52:08 -0800 Subject: [PATCH 0133/1025] Make `syntax_style` a tiny bit more intuitive --- mycli/myclirc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mycli/myclirc b/mycli/myclirc index 9abef8818..6212fca72 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -35,9 +35,11 @@ timing = True # Recommended: psql, fancy_grid and grid. table_format = psql -# Syntax Style. Possible values: manni, igor, xcode, vim, autumn, vs, rrt, -# native, perldoc, borland, tango, emacs, friendly, monokai, paraiso-dark, -# colorful, murphy, bw, pastie, paraiso-light, trac, default, fruity +# Syntax coloring style. Possible values (many support the "-dark" suffix): +# manni, igor, xcode, vim, autumn, vs, rrt, native, perldoc, borland, tango, emacs, +# friendly, monokai, paraiso, colorful, murphy, bw, pastie, paraiso-light, trac, +# default, fruity. +# Screenshots at http://mycli.net/syntax syntax_style = default # Keybindings: Possible values: emacs, vi. From f919e512dda771c43cabadcc6e88892b9e3ad865 Mon Sep 17 00:00:00 2001 From: Phil Cohen Date: Tue, 29 Dec 2015 11:53:31 -0800 Subject: [PATCH 0134/1025] oops --- mycli/myclirc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/myclirc b/mycli/myclirc index 6212fca72..c59e621c7 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -37,8 +37,8 @@ table_format = psql # Syntax coloring style. Possible values (many support the "-dark" suffix): # manni, igor, xcode, vim, autumn, vs, rrt, native, perldoc, borland, tango, emacs, -# friendly, monokai, paraiso, colorful, murphy, bw, pastie, paraiso-light, trac, -# default, fruity. +# friendly, monokai, paraiso, colorful, murphy, bw, pastie, paraiso, trac, default, +# fruity. # Screenshots at http://mycli.net/syntax syntax_style = default From 9c0b65ebb9d8b423c7e7701116dc20bda206296a Mon Sep 17 00:00:00 2001 From: Lennart Weller Date: Fri, 27 Nov 2015 14:19:17 +0100 Subject: [PATCH 0135/1025] Respect the new default actions in prompt_toolkit which were added in 0.51 --- mycli/key_bindings.py | 1 + mycli/main.py | 4 +++- setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index 4a16b95b2..94de541a9 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -17,6 +17,7 @@ def mycli_bindings(get_key_bindings, set_key_bindings): key_binding_manager = KeyBindingManager( enable_open_in_editor=True, enable_system_bindings=True, + enable_abort_and_exit_bindings=True, enable_vi_mode=Condition(lambda cli: get_key_bindings() == 'vi')) @key_binding_manager.registry.add_binding(Keys.F2) diff --git a/mycli/main.py b/mycli/main.py index 1cfc57acc..1b60f01c9 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -15,6 +15,7 @@ import click import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction +from prompt_toolkit.interface import AcceptAction from prompt_toolkit.enums import DEFAULT_BUFFER from prompt_toolkit.shortcuts import create_default_layout, create_eventloop from prompt_toolkit.document import Document @@ -396,12 +397,13 @@ def prompt_tokens(cli): with self._completer_lock: buf = CLIBuffer(always_multiline=self.multi_line, completer=self.completer, history=FileHistory(os.path.expanduser('~/.mycli-history')), - complete_while_typing=Always()) + complete_while_typing=Always(), accept_action=AcceptAction.RETURN_DOCUMENT) application = Application(style=style_factory(self.syntax_style, self.cli_style), layout=layout, buffer=buf, key_bindings_registry=key_binding_manager.registry, on_exit=AbortAction.RAISE_EXCEPTION, + on_abort=AbortAction.RETRY, ignore_case=True) self.cli = CommandLineInterface(application=application, eventloop=create_eventloop()) diff --git a/setup.py b/setup.py index e503e2c4d..ac9717e82 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ install_requirements = [ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? - 'prompt_toolkit==0.46', + 'prompt_toolkit==0.51', 'PyMySQL >= 0.6.2', 'sqlparse >= 0.1.16', 'configobj >= 5.0.6', From 87ec3b41c2a48ae212f282c5eaf55d637bc96b89 Mon Sep 17 00:00:00 2001 From: Mikhail Borisov Date: Tue, 5 Jan 2016 02:50:14 +0300 Subject: [PATCH 0136/1025] Upgrade to prompt_toolkit 0.56 --- mycli/clistyle.py | 4 ++-- mycli/clitoolbar.py | 7 ++++--- mycli/main.py | 23 +++++++++++------------ setup.py | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/mycli/clistyle.py b/mycli/clistyle.py index 45f20e347..a74df4506 100644 --- a/mycli/clistyle.py +++ b/mycli/clistyle.py @@ -1,7 +1,7 @@ from pygments.token import string_to_tokentype from pygments.style import Style from pygments.util import ClassNotFound -from prompt_toolkit.styles import default_style_extensions +from prompt_toolkit.styles import default_style_extensions, PygmentsStyle import pygments.styles @@ -19,4 +19,4 @@ class CLIStyle(Style): custom_styles = dict([(string_to_tokentype(x), y) for x, y in cli_style.items()]) styles.update(custom_styles) - return CLIStyle + return PygmentsStyle(CLIStyle) diff --git a/mycli/clitoolbar.py b/mycli/clitoolbar.py index 66bdc8028..8abe948ab 100644 --- a/mycli/clitoolbar.py +++ b/mycli/clitoolbar.py @@ -1,4 +1,5 @@ from pygments.token import Token +from prompt_toolkit.enums import DEFAULT_BUFFER def create_toolbar_tokens_func(get_key_bindings, get_is_refreshing): """ @@ -12,17 +13,17 @@ def get_toolbar_tokens(cli): result = [] result.append((token, ' ')) - if cli.buffers['default'].completer.smart_completion: + if cli.buffers[DEFAULT_BUFFER].completer.smart_completion: result.append((token.On, '[F2] Smart Completion: ON ')) else: result.append((token.Off, '[F2] Smart Completion: OFF ')) - if cli.buffers['default'].always_multiline: + if cli.buffers[DEFAULT_BUFFER].always_multiline: result.append((token.On, '[F3] Multiline: ON ')) else: result.append((token.Off, '[F3] Multiline: OFF ')) - if cli.buffers['default'].always_multiline: + if cli.buffers[DEFAULT_BUFFER].always_multiline: result.append((token, ' (Semi-colon [;] will end the line)')) diff --git a/mycli/main.py b/mycli/main.py index 1b60f01c9..b38a83ebf 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -17,7 +17,7 @@ from prompt_toolkit import CommandLineInterface, Application, AbortAction from prompt_toolkit.interface import AcceptAction from prompt_toolkit.enums import DEFAULT_BUFFER -from prompt_toolkit.shortcuts import create_default_layout, create_eventloop +from prompt_toolkit.shortcuts import create_prompt_layout, create_eventloop from prompt_toolkit.document import Document from prompt_toolkit.filters import Always, HasFocus, IsDone from prompt_toolkit.layout.processors import (HighlightMatchingBracketProcessor, @@ -383,17 +383,16 @@ def prompt_tokens(cli): get_toolbar_tokens = create_toolbar_tokens_func(lambda: self.key_bindings, self.completion_refresher.is_refreshing) - layout = create_default_layout(lexer=MyCliLexer, - reserve_space_for_menu=True, - multiline=True, - get_prompt_tokens=prompt_tokens, - get_bottom_toolbar_tokens=get_toolbar_tokens, - display_completions_in_columns=self.wider_completion_menu, - extra_input_processors=[ - ConditionalProcessor( - processor=HighlightMatchingBracketProcessor(chars='[](){}'), - filter=HasFocus(DEFAULT_BUFFER) & ~IsDone()), - ]) + layout = create_prompt_layout(lexer=MyCliLexer, + multiline=True, + get_prompt_tokens=prompt_tokens, + get_bottom_toolbar_tokens=get_toolbar_tokens, + display_completions_in_columns=self.wider_completion_menu, + extra_input_processors=[ + ConditionalProcessor( + processor=HighlightMatchingBracketProcessor(chars='[](){}'), + filter=HasFocus(DEFAULT_BUFFER) & ~IsDone()), + ]) with self._completer_lock: buf = CLIBuffer(always_multiline=self.multi_line, completer=self.completer, history=FileHistory(os.path.expanduser('~/.mycli-history')), diff --git a/setup.py b/setup.py index ac9717e82..a18ad9405 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ install_requirements = [ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? - 'prompt_toolkit==0.51', + 'prompt_toolkit==0.56', 'PyMySQL >= 0.6.2', 'sqlparse >= 0.1.16', 'configobj >= 5.0.6', From ba96b11674ddd8c110f8d391c353d1d2d6d7199f Mon Sep 17 00:00:00 2001 From: Mikhail Borisov Date: Thu, 7 Jan 2016 03:11:54 +0300 Subject: [PATCH 0137/1025] Capture warnings to log file Prevents runtime warnings in pymysql from littering terminal output. --- mycli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 1cfc57acc..f9db3ea01 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -237,6 +237,8 @@ def initialize_logging(self): root_logger.addHandler(handler) root_logger.setLevel(level_map[log_level.upper()]) + logging.captureWarnings(True) + root_logger.debug('Initializing mycli logging.') root_logger.debug('Log file %r.', log_file) From 32e923c590d46b90e3a4ff2660354f803315867c Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 6 Jan 2016 21:09:12 -0800 Subject: [PATCH 0138/1025] Update prompt_toolkit dependency to 0.57 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a18ad9405..92b11a219 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ install_requirements = [ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? - 'prompt_toolkit==0.56', + 'prompt_toolkit==0.57', 'PyMySQL >= 0.6.2', 'sqlparse >= 0.1.16', 'configobj >= 5.0.6', From 3873a27349c8b07d31b4f669387f833e1f36b9e7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 9 Jan 2016 15:13:45 -0600 Subject: [PATCH 0139/1025] Adds nopager command and skip-pager config. --- mycli/main.py | 9 +++++++-- mycli/packages/special/iocommands.py | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 1cfc57acc..f9babae52 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -525,7 +525,10 @@ def prompt_tokens(cli): self.output(str(e), err=True, fg='red') else: try: - self.output_via_pager('\n'.join(output)) + if special.is_pager_enabled(): + self.output_via_pager('\n'.join(output)) + else: + self.output('\n'.join(output)) except KeyboardInterrupt: pass if special.is_timing_enabled(): @@ -568,9 +571,11 @@ def adjust_less_opts(self): return less_opts def set_pager_from_config(self): - cnf = self.read_my_cnf_files(self.cnf_files, ['pager']) + cnf = self.read_my_cnf_files(self.cnf_files, ['pager', 'skip-pager']) if cnf['pager']: special.set_pager(cnf['pager']) + if cnf['skip-pager']: + special.disable_pager() def refresh_completions(self, reset=False): if reset: diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 307a27906..74810328e 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -15,12 +15,22 @@ TIMING_ENABLED = False use_expanded_output = False ORIGINAL_PAGER = os.environ.get('PAGER', '') +PAGER_ENABLED = True @export def set_timing_enabled(val): global TIMING_ENABLED TIMING_ENABLED = val +@export +def set_pager_enabled(val): + global PAGER_ENABLED + PAGER_ENABLED = val + +@export +def is_pager_enabled(): + return PAGER_ENABLED + @export def get_original_pager(): return ORIGINAL_PAGER @@ -32,15 +42,29 @@ def set_pager(arg, **_): if not ORIGINAL_PAGER: os.environ.pop('PAGER', None) msg = 'Reset pager.' + set_pager_enabled(False) else: os.environ['PAGER'] = ORIGINAL_PAGER msg = 'Reset pager back to default. Default: %s' % ORIGINAL_PAGER + set_pager_enabled(True) else: os.environ['PAGER'] = arg msg = 'PAGER set to %s.' % arg + set_pager_enabled(True) return [(None, None, None, msg)] +@special_command('nopager', '\\n', 'Disable pager, print to stdout.', + arg_type=NO_QUERY, aliases=('\\n', ), case_sensitive=True) +def disable_pager(): + set_pager_enabled(False) + return [(None, None, None, 'Pager disabled.')] + +@export +def is_pager_enabled(): + return PAGER_ENABLED + + @special_command('\\timing', '\\t', 'Toggle timing of commands.', arg_type=NO_QUERY, aliases=('\\t', ), case_sensitive=True) def toggle_timing(): global TIMING_ENABLED From aac942e0768a0c98cc45bab3663371c1eb6cf0bc Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 9 Jan 2016 15:30:26 -0600 Subject: [PATCH 0140/1025] Removes duplicate function. --- mycli/packages/special/iocommands.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 74810328e..103b7e830 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -54,17 +54,13 @@ def set_pager(arg, **_): return [(None, None, None, msg)] +@export @special_command('nopager', '\\n', 'Disable pager, print to stdout.', - arg_type=NO_QUERY, aliases=('\\n', ), case_sensitive=True) + arg_type=NO_QUERY, aliases=('\\n', ), case_sensitive=True) def disable_pager(): set_pager_enabled(False) return [(None, None, None, 'Pager disabled.')] -@export -def is_pager_enabled(): - return PAGER_ENABLED - - @special_command('\\timing', '\\t', 'Toggle timing of commands.', arg_type=NO_QUERY, aliases=('\\t', ), case_sensitive=True) def toggle_timing(): global TIMING_ENABLED From c1e8d0b44bc4e3e04b2efe45cad0605bbaa8a7f9 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 31 Jan 2016 11:29:40 -0600 Subject: [PATCH 0141/1025] Adds support for MYSQL_TEST_LOGIN_FILE env variable. --- mycli/config.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 0d0dc2e9c..56c54df1a 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -75,14 +75,14 @@ def write_default_config(source, destination, overwrite=False): def get_mylogin_cnf_path(): """Return the path to the .mylogin.cnf file or None if doesn't exist.""" - app_data = os.getenv('APPDATA') - if app_data is None: - mylogin_cnf_dir = os.path.expanduser('~') - else: - mylogin_cnf_dir = os.path.join(app_data, 'MySQL') + mylogin_cnf_path = os.getenv('MYSQL_TEST_LOGIN_FILE') + + if mylogin_cnf_path is None: + app_data = os.getenv('APPDATA') + default_dir = os.path.join(app_data, 'MySQL') if app_data else '~' + mylogin_cnf_path = os.path.join(default_dir, '.mylogin.cnf') - mylogin_cnf_dir = os.path.abspath(mylogin_cnf_dir) - mylogin_cnf_path = os.path.join(mylogin_cnf_dir, '.mylogin.cnf') + mylogin_cnf_path = os.path.expanduser(mylogin_cnf_path) if exists(mylogin_cnf_path): logger.debug("Found login path file at '{0}'".format(mylogin_cnf_path)) From 51feb265763d797464b2740870d29735622af780 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 31 Jan 2016 13:52:17 -0600 Subject: [PATCH 0142/1025] Adds test for finding login path file. --- tests/test_login_path.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/test_login_path.py b/tests/test_login_path.py index 642751531..4a89d1ebf 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -3,10 +3,12 @@ import os import pip import struct +import sys +import tempfile import pytest -from mycli.config import open_mylogin_cnf, read_and_decrypt_mylogin_cnf, \ - CryptoError +from mycli.config import (CryptoError, get_mylogin_cnf_path, + open_mylogin_cnf, read_and_decrypt_mylogin_cnf) with_pycrypto = ['pycrypto' in set([package.project_name for package in pip.get_installed_distributions()])] @@ -90,3 +92,30 @@ def test_corrupted_pad(): for word in ('[test]', 'password', 'host', 'port'): assert word in contents assert 'user' not in contents + + +def test_get_mylogin_cnf_path(): + """Tests that the path for .mylogin.cnf is detected.""" + del os.environ['MYSQL_TEST_LOGIN_FILE'] + is_windows = sys.platform == 'win32' + + login_cnf_path = get_mylogin_cnf_path() + + if login_cnf_path is not None: + assert login_cnf_path.endswith('.mylogin.cnf') + + if is_windows is True: + assert 'MySQL' in login_cnf_path + else: + home_dir = os.path.expanduser('~') + assert login_cnf_path.startswith(home_dir) + + +def test_alternate_get_mylogin_cnf_path(): + """Tests that the alternate path for .mylogin.cnf is detected.""" + temp_fh, temp_path = tempfile.mkstemp() + os.environ['MYSQL_TEST_LOGIN_FILE'] = temp_path + + login_cnf_path = get_mylogin_cnf_path() + + assert temp_path == login_cnf_path From 6e907ab744db5efaebf9244445405978493d2245 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 31 Jan 2016 14:00:01 -0600 Subject: [PATCH 0143/1025] Updates docstring for login path function. --- mycli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/config.py b/mycli/config.py index 56c54df1a..980143780 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -74,7 +74,7 @@ def write_default_config(source, destination, overwrite=False): shutil.copyfile(source, destination) def get_mylogin_cnf_path(): - """Return the path to the .mylogin.cnf file or None if doesn't exist.""" + """Return the path to the login path file or None if it doesn't exist.""" mylogin_cnf_path = os.getenv('MYSQL_TEST_LOGIN_FILE') if mylogin_cnf_path is None: From 88b6cc38ac4abfe6735b7072d766ba720c290455 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 31 Jan 2016 15:25:34 -0600 Subject: [PATCH 0144/1025] Fixes failing test on Travis CI. --- tests/test_login_path.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_login_path.py b/tests/test_login_path.py index 4a89d1ebf..2c303ef34 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -96,7 +96,8 @@ def test_corrupted_pad(): def test_get_mylogin_cnf_path(): """Tests that the path for .mylogin.cnf is detected.""" - del os.environ['MYSQL_TEST_LOGIN_FILE'] + if 'MYSQL_TEST_LOGIN_FILE' in os.environ: + del os.environ['MYSQL_TEST_LOGIN_FILE'] is_windows = sys.platform == 'win32' login_cnf_path = get_mylogin_cnf_path() From 5e63bfda36ab4043f4bd7ee4f9a9d171cbf062ea Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 31 Jan 2016 18:46:17 -0600 Subject: [PATCH 0145/1025] Makes tests restore original env variable. --- tests/test_login_path.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_login_path.py b/tests/test_login_path.py index 2c303ef34..1473264ab 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -96,12 +96,16 @@ def test_corrupted_pad(): def test_get_mylogin_cnf_path(): """Tests that the path for .mylogin.cnf is detected.""" + original_env = None if 'MYSQL_TEST_LOGIN_FILE' in os.environ: - del os.environ['MYSQL_TEST_LOGIN_FILE'] + original_env = os.environ.pop('MYSQL_TEST_LOGIN_FILE') is_windows = sys.platform == 'win32' login_cnf_path = get_mylogin_cnf_path() + if original_env is not None: + os.environ['MYSQL_TEST_LOGIN_FILE'] = original_env + if login_cnf_path is not None: assert login_cnf_path.endswith('.mylogin.cnf') @@ -114,9 +118,16 @@ def test_get_mylogin_cnf_path(): def test_alternate_get_mylogin_cnf_path(): """Tests that the alternate path for .mylogin.cnf is detected.""" + original_env = None + if 'MYSQL_TEST_LOGIN_FILE' in os.environ: + original_env = os.environ.pop('MYSQL_TEST_LOGIN_FILE') + temp_fh, temp_path = tempfile.mkstemp() os.environ['MYSQL_TEST_LOGIN_FILE'] = temp_path login_cnf_path = get_mylogin_cnf_path() + if original_env is not None: + os.environ['MYSQL_TEST_LOGIN_FILE'] = original_env + assert temp_path == login_cnf_path From ec54a6b1961677f4eb1f071f027f29b7be8e8d17 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 Feb 2016 05:48:26 -0600 Subject: [PATCH 0146/1025] Cleans up test code. --- tests/test_login_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_login_path.py b/tests/test_login_path.py index 1473264ab..b8329291c 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -122,7 +122,7 @@ def test_alternate_get_mylogin_cnf_path(): if 'MYSQL_TEST_LOGIN_FILE' in os.environ: original_env = os.environ.pop('MYSQL_TEST_LOGIN_FILE') - temp_fh, temp_path = tempfile.mkstemp() + temp_path = tempfile.mkstemp()[1] os.environ['MYSQL_TEST_LOGIN_FILE'] = temp_path login_cnf_path = get_mylogin_cnf_path() From 8baeb89adbf10c2b7ff693aab5280cf0ca48b2f3 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 1 Feb 2016 10:14:13 -0200 Subject: [PATCH 0147/1025] Add small change to login_path test code --- tests/test_login_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_login_path.py b/tests/test_login_path.py index b8329291c..84e21ae1a 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -122,7 +122,7 @@ def test_alternate_get_mylogin_cnf_path(): if 'MYSQL_TEST_LOGIN_FILE' in os.environ: original_env = os.environ.pop('MYSQL_TEST_LOGIN_FILE') - temp_path = tempfile.mkstemp()[1] + _, temp_path = tempfile.mkstemp() os.environ['MYSQL_TEST_LOGIN_FILE'] = temp_path login_cnf_path = get_mylogin_cnf_path() From 4a51740fd7e8db5f6929590ef5344c94624725d0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 Feb 2016 21:37:12 -0600 Subject: [PATCH 0148/1025] Adds str_to_bool function and tests. --- mycli/config.py | 17 +++++++++++++++++ tests/test_login_path.py | 26 ++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 980143780..1399dac08 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -195,3 +195,20 @@ def read_and_decrypt_mylogin_cnf(f): plaintext.seek(0) return plaintext + +def str_to_bool(s): + """Convert a string value to it's corresponding boolean value.""" + if isinstance(s, bool): + return s + elif not isinstance(s, basestring): + raise TypeError('argument must be a string') + + true_values = ('true', 'on', '1') + false_values = ('false', 'off', '0') + + if s.lower() in true_values: + return True + elif s.lower() in false_values: + return False + else: + raise ValueError('not a recognized boolean value: %s'.format(s)) diff --git a/tests/test_login_path.py b/tests/test_login_path.py index 84e21ae1a..efd66d017 100644 --- a/tests/test_login_path.py +++ b/tests/test_login_path.py @@ -1,4 +1,4 @@ -"""Unit tests for mycli.config login path decryption.""" +"""Unit tests for the mycli.config module.""" from io import BytesIO, TextIOWrapper import os import pip @@ -8,7 +8,8 @@ import pytest from mycli.config import (CryptoError, get_mylogin_cnf_path, - open_mylogin_cnf, read_and_decrypt_mylogin_cnf) + open_mylogin_cnf, read_and_decrypt_mylogin_cnf, + str_to_bool) with_pycrypto = ['pycrypto' in set([package.project_name for package in pip.get_installed_distributions()])] @@ -131,3 +132,24 @@ def test_alternate_get_mylogin_cnf_path(): os.environ['MYSQL_TEST_LOGIN_FILE'] = original_env assert temp_path == login_cnf_path + + +def test_str_to_bool(): + """Tests that str_to_bool function converts values correctly.""" + + assert str_to_bool(False) is False + assert str_to_bool(True) is True + assert str_to_bool('False') is False + assert str_to_bool('True') is True + assert str_to_bool('TRUE') is True + assert str_to_bool('1') is True + assert str_to_bool('0') is False + assert str_to_bool('on') is True + assert str_to_bool('off') is False + assert str_to_bool('off') is False + + with pytest.raises(ValueError): + str_to_bool('foo') + + with pytest.raises(TypeError): + str_to_bool(None) From 8620b4bf8d5358778236e300acbd50fdb48d57c9 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 Feb 2016 21:38:05 -0600 Subject: [PATCH 0149/1025] Renames config test module. --- tests/{test_login_path.py => test_config.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_login_path.py => test_config.py} (100%) diff --git a/tests/test_login_path.py b/tests/test_config.py similarity index 100% rename from tests/test_login_path.py rename to tests/test_config.py From b87e94840e47bb32f7ea5933c2fbb789bac89e51 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 Feb 2016 21:38:23 -0600 Subject: [PATCH 0150/1025] Add support for local-infile argument and config option. --- mycli/completion_refresher.py | 2 +- mycli/main.py | 32 ++++++++++++++++++++++---------- mycli/sqlexecute.py | 14 ++++++++++---- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index 4e05ac121..9265ae9fe 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -48,7 +48,7 @@ def _bg_refresh(self, sqlexecute, callbacks): # Create a new pgexecute method to popoulate the completions. e = sqlexecute executor = SQLExecute(e.dbname, e.user, e.password, e.host, e.port, - e.socket, e.charset) + e.socket, e.charset, e.local_infile) # If callbacks is a single function then push it into a list. if callable(callbacks): diff --git a/mycli/main.py b/mycli/main.py index 1459a24c0..01b6885b3 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -38,7 +38,7 @@ from .completion_refresher import CompletionRefresher from .config import (write_default_config, get_mylogin_cnf_path, open_mylogin_cnf, CryptoError, read_config_file, - read_config_files) + read_config_files, str_to_bool) from .key_bindings import mycli_bindings from .encodingutils import utf8tounicode from .lexer import MyCliLexer @@ -243,11 +243,11 @@ def initialize_logging(self): root_logger.debug('Initializing mycli logging.') root_logger.debug('Log file %r.', log_file) - def connect_uri(self, uri): + def connect_uri(self, uri, local_infile=None): uri = urlparse(uri) database = uri.path[1:] # ignore the leading fwd slash self.connect(database, uri.username, uri.password, uri.hostname, - uri.port) + uri.port, local_infile=local_infile) def read_my_cnf_files(self, files, keys): """ @@ -275,7 +275,7 @@ def get(key): return dict([(x, get(x)) for x in keys]) def connect(self, database='', user='', passwd='', host='', port='', - socket='', charset=''): + socket='', charset='', local_infile=''): cnf = {'database': None, 'user': None, @@ -283,7 +283,8 @@ def connect(self, database='', user='', passwd='', host='', port='', 'host': None, 'port': None, 'socket': None, - 'default-character-set': None} + 'default-character-set': None, + 'local-infile': None} cnf = self.read_my_cnf_files(self.cnf_files, cnf.keys()) @@ -307,18 +308,26 @@ def connect(self, database='', user='', passwd='', host='', port='', passwd = passwd or cnf['password'] charset = charset or cnf['default-character-set'] or 'utf8' + # Favor whichever local_infile option is set. + for local_infile_option in (local_infile, cnf['local-infile'], False): + try: + local_infile = str_to_bool(local_infile_option) + break + except (TypeError, ValueError): + pass + # Connect to the database. try: try: sqlexecute = SQLExecute(database, user, passwd, host, port, - socket, charset) + socket, charset, local_infile) except OperationalError as e: if ('Access denied for user' in e.args[1]): passwd = click.prompt('Password', hide_input=True, show_default=False, type=str) sqlexecute = SQLExecute(database, user, passwd, host, port, - socket, charset) + socket, charset, local_infile) else: raise e except Exception as e: # Connecting to a database could fail. @@ -646,12 +655,14 @@ def get_prompt(self, string): help='Only read default options from the given file') @click.option('--auto-vertical-output', is_flag=True, help='Automatically switch to vertical output mode if the result is wider than the terminal width.') +@click.option('--local-infile', type=bool, + help='Enable/disable LOAD DATA LOCAL INFILE.') @click.option('--login-path', type=str, help='Read this path from the login file.') @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, - login_path, auto_vertical_output): + login_path, auto_vertical_output, local_infile): if version: print('Version:', __version__) sys.exit(0) @@ -665,9 +676,10 @@ def cli(database, user, host, port, socket, password, dbname, database = database or dbname if database and '://' in database: - mycli.connect_uri(database) + mycli.connect_uri(database, local_infile) else: - mycli.connect(database, user, password, host, port, socket) + mycli.connect(database, user, password, host, port, socket, + local_infile=local_infile) mycli.logger.debug('Launch Params: \n' '\tdatabase: %r' diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 2ae56d8e9..64d488d06 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -26,7 +26,8 @@ class SQLExecute(object): where table_schema = '%s' order by table_name,ordinal_position''' - def __init__(self, database, user, password, host, port, socket, charset): + def __init__(self, database, user, password, host, port, socket, charset, + local_infile): self.dbname = database self.user = user self.password = password @@ -34,11 +35,12 @@ def __init__(self, database, user, password, host, port, socket, charset): self.port = port self.socket = socket self.charset = charset + self.local_infile = local_infile self._server_type = None self.connect() def connect(self, database=None, user=None, password=None, host=None, - port=None, socket=None, charset=None): + port=None, socket=None, charset=None, local_infile=None): db = (database or self.dbname) user = (user or self.user) password = (password or self.password) @@ -46,18 +48,21 @@ def connect(self, database=None, user=None, password=None, host=None, port = (port or self.port) socket = (socket or self.socket) charset = (charset or self.charset) + local_infile = (local_infile or self.local_infile) _logger.debug('Connection DB Params: \n' '\tdatabase: %r' '\tuser: %r' '\thost: %r' '\tport: %r' '\tsocket: %r' - '\tcharset: %r', database, user, host, port, socket, charset) + '\tcharset: %r' + '\tlocal_infile: %r', + database, user, host, port, socket, charset, local_infile) conn = connection.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, use_unicode=True, charset=charset, autocommit=True, client_flag=pymysql.constants.CLIENT.INTERACTIVE, - cursorclass=connection.Cursor) + cursorclass=connection.Cursor, local_infile=local_infile) if hasattr(self, 'conn'): self.conn.close() self.conn = conn @@ -87,6 +92,7 @@ def run(self, statement): # want to save them all together. if statement.startswith('\\fs'): components = [statement] + else: components = sqlparse.split(statement) From c3008907f3d4c3368198ab64dbfe09971a155647 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 Feb 2016 21:39:17 -0600 Subject: [PATCH 0151/1025] Fixes typo in docstring. --- mycli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/config.py b/mycli/config.py index 1399dac08..284a8b400 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -197,7 +197,7 @@ def read_and_decrypt_mylogin_cnf(f): return plaintext def str_to_bool(s): - """Convert a string value to it's corresponding boolean value.""" + """Convert a string value to its corresponding boolean value.""" if isinstance(s, bool): return s elif not isinstance(s, basestring): From e3342da85b5efb2185fafdaff54cda7fb03c733f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 Feb 2016 21:58:38 -0600 Subject: [PATCH 0152/1025] Adds support for loose-local-infile option. --- mycli/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 01b6885b3..d8a2af6c9 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -284,7 +284,8 @@ def connect(self, database='', user='', passwd='', host='', port='', 'port': None, 'socket': None, 'default-character-set': None, - 'local-infile': None} + 'local-infile': None, + 'loose-local-infile': None} cnf = self.read_my_cnf_files(self.cnf_files, cnf.keys()) @@ -309,7 +310,8 @@ def connect(self, database='', user='', passwd='', host='', port='', charset = charset or cnf['default-character-set'] or 'utf8' # Favor whichever local_infile option is set. - for local_infile_option in (local_infile, cnf['local-infile'], False): + for local_infile_option in (local_infile, cnf['local-infile'], + cnf['loose-local-infile'], False): try: local_infile = str_to_bool(local_infile_option) break From cf16b9d5eafb497941579ea67f84d00491bfc276 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 Feb 2016 22:06:19 -0600 Subject: [PATCH 0153/1025] Updates test call to sqlexecute. --- tests/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 5d0ccbec6..dda72dddf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,7 +9,8 @@ def db_connection(dbname=None): conn = connection.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, - charset=CHARSET, cursorclass=connection.Cursor) + charset=CHARSET, cursorclass=connection.Cursor, + local_infile=False) conn.autocommit = True return conn From dc311dcc679f2d86141c9ca1b8c0ece6e8a4af9b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 Feb 2016 22:09:30 -0600 Subject: [PATCH 0154/1025] Updates executor call. --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6e36a450a..d24d26bc4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,4 +22,5 @@ def cursor(connection): def executor(connection): return mycli.sqlexecute.SQLExecute( database='_test_db', user=USER, - host=HOST, password=PASSWORD, port=PORT, socket=None, charset=CHARSET) + host=HOST, password=PASSWORD, port=PORT, socket=None, charset=CHARSET, + local_infile=False) From 7a0ea7093a3f55fe568c185506c2341bf7ef67d4 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 Feb 2016 23:51:23 -0600 Subject: [PATCH 0155/1025] Refactors pager code. --- mycli/main.py | 15 +++------------ mycli/packages/special/iocommands.py | 23 ++++++++--------------- 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index d8a2af6c9..1c8c420de 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -368,8 +368,7 @@ def handle_editor_command(self, cli, document): def run_cli(self): sqlexecute = self.sqlexecute logger = self.logger - original_less_opts = self.adjust_less_opts() - self.set_pager_from_config() + self.configure_pager() self.refresh_completions() @@ -560,10 +559,6 @@ def prompt_tokens(cli): except EOFError: self.output('Goodbye!') - finally: # Reset the less opts back to original. - logger.debug('Restoring env var LESS to %r.', original_less_opts) - os.environ['LESS'] = original_less_opts - os.environ['PAGER'] = special.get_original_pager() def output(self, text, **kwargs): if self.logfile: @@ -577,14 +572,10 @@ def output_via_pager(self, text): self.logfile.write('\n') click.echo_via_pager(text) - def adjust_less_opts(self): - less_opts = os.environ.get('LESS', '') - self.logger.debug('Original value for LESS env var: %r', less_opts) + def configure_pager(self): + # Provide sane defaults for less. os.environ['LESS'] = '-SRXF' - return less_opts - - def set_pager_from_config(self): cnf = self.read_my_cnf_files(self.cnf_files, ['pager', 'skip-pager']) if cnf['pager']: special.set_pager(cnf['pager']) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 103b7e830..1971a7c93 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -14,7 +14,6 @@ TIMING_ENABLED = False use_expanded_output = False -ORIGINAL_PAGER = os.environ.get('PAGER', '') PAGER_ENABLED = True @export @@ -31,26 +30,20 @@ def set_pager_enabled(val): def is_pager_enabled(): return PAGER_ENABLED -@export -def get_original_pager(): - return ORIGINAL_PAGER - @export @special_command('pager', '\\P [command]', 'Set PAGER. Print the query results via PAGER', arg_type=PARSED_QUERY, aliases=('\\P', ), case_sensitive=True) def set_pager(arg, **_): - if not arg: - if not ORIGINAL_PAGER: - os.environ.pop('PAGER', None) - msg = 'Reset pager.' - set_pager_enabled(False) - else: - os.environ['PAGER'] = ORIGINAL_PAGER - msg = 'Reset pager back to default. Default: %s' % ORIGINAL_PAGER - set_pager_enabled(True) - else: + if arg: os.environ['PAGER'] = arg msg = 'PAGER set to %s.' % arg set_pager_enabled(True) + else: + if 'PAGER' in os.environ: + msg = 'PAGER set to %s.' % os.environ['PAGER'] + else: + # This uses click's default per echo_via_pager. + msg = 'Pager enabled.' + set_pager_enabled(True) return [(None, None, None, msg)] From 59a35a5da5f0479176d332b037e1e7ab3202227c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 5 Feb 2016 21:49:43 -0600 Subject: [PATCH 0156/1025] Adds status command. --- mycli/packages/special/dbcommands.py | 128 +++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/mycli/packages/special/dbcommands.py b/mycli/packages/special/dbcommands.py index aa6a7694f..f47533b65 100644 --- a/mycli/packages/special/dbcommands.py +++ b/mycli/packages/special/dbcommands.py @@ -1,4 +1,9 @@ import logging +import os +import platform +from mycli import __version__ +from mycli.packages.tabulate import tabulate +from mycli.packages.special import iocommands from .main import special_command, RAW_QUERY, PARSED_QUERY log = logging.getLogger(__name__) @@ -28,3 +33,126 @@ def list_databases(cur, **_): else: return [(None, None, None, '')] +def format_uptime(uptime_in_seconds): + """Format number of seconds into human-readable string. + + :param uptime_in_seconds: The server uptime in seconds. + :returns: A human-readable string representing the uptime. + + >>> uptime = format_uptime('56892') + >>> print(uptime) + 15 hours 48 min 12 sec + """ + + m, s = divmod(int(uptime_in_seconds), 60) + h, m = divmod(m, 60) + d, h = divmod(h, 24) + + uptime_values = [] + + for value, unit in ((d, 'days'), (h, 'hours'), (m, 'min'), (s, 'sec')): + if value == 0 and not uptime_values: + # Don't include a value/unit if the unit isn't applicable to + # the uptime. E.g. don't do 0 days 0 hours 1 min 30 sec. + continue + elif value == 1 and unit.endswith('s'): + # Remove the "s" if the unit is singular. + unit = unit[:-1] + uptime_values.append('{} {}'.format(value, unit)) + + uptime = ' '.join(uptime_values) + return uptime + +@special_command('status', '\\s', 'Get status information from the server.', + arg_type=RAW_QUERY, case_sensitive=True) +def status(cur, **_): + query = 'SHOW GLOBAL STATUS;' + log.debug(query) + cur.execute(query) + status = dict(cur.fetchall()) + + query = 'SHOW GLOBAL VARIABLES;' + log.debug(query) + cur.execute(query) + variables = dict(cur.fetchall()) + + print('--------------') + + # Output the mycli client information. + implementation = platform.python_implementation() + version = platform.python_version() + header = [] + header.append('mycli {},'.format(__version__)) + header.append('running on {} {}'.format(implementation, version)) + print(' '.join(header) + '\n') + + # Build the output that will be displayed as a table. + output = [] + + output.append(('Connection id:', cur.connection.thread_id())) + + query = 'SELECT DATABASE(), USER();' + log.debug(query) + cur.execute(query) + db, user = cur.fetchone() + if db is None: + db = '' + + output.append(('Current database:', db)) + output.append(('Current user:', user)) + + if iocommands.is_pager_enabled(): + if 'PAGER' in os.environ: + pager = os.environ['PAGER'] + else: + pager = 'System default' + else: + pager = 'stdout' + output.append(('Current pager:', pager)) + + output.append(('Server version:', '{} {}'.format( + variables['version'], variables['version_comment']))) + output.append(('Protocol version:', variables['protocol_version'])) + + if 'unix' in cur.connection.host_info.lower(): + host_info = cur.connection.host_info + else: + host_info = '{} via TCP/IP'.format(cur.connection.host) + + output.append(('Connection:', host_info)) + + query = ('SELECT @@character_set_server, @@character_set_database, ' + '@@character_set_client, @@character_set_connection LIMIT 1;') + log.debug(query) + cur.execute(query) + charset = cur.fetchone() + output.append(('Server characterset:', charset[0])) + output.append(('Db characterset:', charset[1])) + output.append(('Client characterset:', charset[2])) + output.append(('Conn. characterset:', charset[3])) + + if 'TCP/IP' in host_info: + output.append(('TCP port:', cur.connection.port)) + else: + output.append(('UNIX socket:', variables['socket'])) + + output.append(('Uptime:', format_uptime(status['Uptime']))) + + # Print the buffered output in two columns. + print(tabulate(output, tablefmt='plain')[0]) + + # Print the current server statistics. + stats = [] + stats.append('Connections: {}'.format(status['Threads_connected'])) + stats.append('Questions: {}'.format(status['Queries'])) + stats.append('Slow queries: {}'.format(status['Slow_queries'])) + stats.append('Opens: {}'.format(status['Opened_tables'])) + stats.append('Flush tables: {}'.format(status['Flush_commands'])) + stats.append('Open tables: {}'.format(status['Open_tables'])) + queries_per_second = int(status['Queries']) / int(status['Uptime']) + stats.append('Queries per second avg: {:.3f}'.format(queries_per_second)) + stats = ' '.join(stats) + print('\n' + stats) + + print('--------------') + return [(None, None, None, '')] From c9eb0b243236237e579f5ec7880e056dedad4093 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 5 Feb 2016 21:54:07 -0600 Subject: [PATCH 0157/1025] Makes alias work for status command. --- mycli/packages/special/dbcommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/dbcommands.py b/mycli/packages/special/dbcommands.py index f47533b65..4d6318b72 100644 --- a/mycli/packages/special/dbcommands.py +++ b/mycli/packages/special/dbcommands.py @@ -64,7 +64,7 @@ def format_uptime(uptime_in_seconds): return uptime @special_command('status', '\\s', 'Get status information from the server.', - arg_type=RAW_QUERY, case_sensitive=True) + arg_type=RAW_QUERY, aliases=('\\s', ), case_sensitive=True) def status(cur, **_): query = 'SHOW GLOBAL STATUS;' log.debug(query) From bfd91a92b02123d96b73b4d2876a39578e474acf Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 5 Feb 2016 21:58:58 -0600 Subject: [PATCH 0158/1025] Moves format_uptime to utils module. --- mycli/packages/special/dbcommands.py | 31 +--------------------------- mycli/packages/special/utils.py | 30 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/mycli/packages/special/dbcommands.py b/mycli/packages/special/dbcommands.py index 4d6318b72..f1feedb86 100644 --- a/mycli/packages/special/dbcommands.py +++ b/mycli/packages/special/dbcommands.py @@ -4,6 +4,7 @@ from mycli import __version__ from mycli.packages.tabulate import tabulate from mycli.packages.special import iocommands +from mycli.packages.special.utils import format_uptime from .main import special_command, RAW_QUERY, PARSED_QUERY log = logging.getLogger(__name__) @@ -33,36 +34,6 @@ def list_databases(cur, **_): else: return [(None, None, None, '')] -def format_uptime(uptime_in_seconds): - """Format number of seconds into human-readable string. - - :param uptime_in_seconds: The server uptime in seconds. - :returns: A human-readable string representing the uptime. - - >>> uptime = format_uptime('56892') - >>> print(uptime) - 15 hours 48 min 12 sec - """ - - m, s = divmod(int(uptime_in_seconds), 60) - h, m = divmod(m, 60) - d, h = divmod(h, 24) - - uptime_values = [] - - for value, unit in ((d, 'days'), (h, 'hours'), (m, 'min'), (s, 'sec')): - if value == 0 and not uptime_values: - # Don't include a value/unit if the unit isn't applicable to - # the uptime. E.g. don't do 0 days 0 hours 1 min 30 sec. - continue - elif value == 1 and unit.endswith('s'): - # Remove the "s" if the unit is singular. - unit = unit[:-1] - uptime_values.append('{} {}'.format(value, unit)) - - uptime = ' '.join(uptime_values) - return uptime - @special_command('status', '\\s', 'Get status information from the server.', arg_type=RAW_QUERY, aliases=('\\s', ), case_sensitive=True) def status(cur, **_): diff --git a/mycli/packages/special/utils.py b/mycli/packages/special/utils.py index e1a160acb..a14af5bd4 100644 --- a/mycli/packages/special/utils.py +++ b/mycli/packages/special/utils.py @@ -14,3 +14,33 @@ def handle_cd_command(arg): return True, None except OSError as e: return False, e.strerror + +def format_uptime(uptime_in_seconds): + """Format number of seconds into human-readable string. + + :param uptime_in_seconds: The server uptime in seconds. + :returns: A human-readable string representing the uptime. + + >>> uptime = format_uptime('56892') + >>> print(uptime) + 15 hours 48 min 12 sec + """ + + m, s = divmod(int(uptime_in_seconds), 60) + h, m = divmod(m, 60) + d, h = divmod(h, 24) + + uptime_values = [] + + for value, unit in ((d, 'days'), (h, 'hours'), (m, 'min'), (s, 'sec')): + if value == 0 and not uptime_values: + # Don't include a value/unit if the unit isn't applicable to + # the uptime. E.g. don't do 0 days 0 hours 1 min 30 sec. + continue + elif value == 1 and unit.endswith('s'): + # Remove the "s" if the unit is singular. + unit = unit[:-1] + uptime_values.append('{} {}'.format(value, unit)) + + uptime = ' '.join(uptime_values) + return uptime From 28b01a08b1b2193d27c7d54a68da082dd4cdb5df Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 5 Feb 2016 22:08:32 -0600 Subject: [PATCH 0159/1025] Adds tests for format_uptime. --- tests/test_dbspecial.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_dbspecial.py b/tests/test_dbspecial.py index f1ee49e51..17309b29c 100644 --- a/tests/test_dbspecial.py +++ b/tests/test_dbspecial.py @@ -1,5 +1,6 @@ from mycli.packages.completion_engine import suggest_type from test_completion_engine import sorted_dicts +from mycli.packages.special.utils import format_uptime def test_u_suggests_databases(): suggestions = suggest_type('\\u ', '\\u ') @@ -13,3 +14,20 @@ def test_describe_table(): {'type': 'table', 'schema': []}, {'type': 'view', 'schema': []}, {'type': 'schema'}]) + + +def test_format_uptime(): + seconds = 59 + assert '59 sec' == format_uptime(seconds) + + seconds = 120 + assert '2 min 0 sec' == format_uptime(seconds) + + seconds = 54890 + assert '15 hours 14 min 50 sec' == format_uptime(seconds) + + seconds = 598244 + assert '6 days 22 hours 10 min 44 sec' == format_uptime(seconds) + + seconds = 522600 + assert '6 days 1 hour 10 min 0 sec' == format_uptime(seconds) From 40c19b272c3b544ab518ae0805844c7071a3ce24 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 5 Feb 2016 22:14:00 -0600 Subject: [PATCH 0160/1025] Fixes Python 2.6 compatibility. --- mycli/packages/special/dbcommands.py | 20 ++++++++++---------- mycli/packages/special/utils.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mycli/packages/special/dbcommands.py b/mycli/packages/special/dbcommands.py index f1feedb86..f31477d58 100644 --- a/mycli/packages/special/dbcommands.py +++ b/mycli/packages/special/dbcommands.py @@ -53,8 +53,8 @@ def status(cur, **_): implementation = platform.python_implementation() version = platform.python_version() header = [] - header.append('mycli {},'.format(__version__)) - header.append('running on {} {}'.format(implementation, version)) + header.append('mycli {0},'.format(__version__)) + header.append('running on {0} {1}'.format(implementation, version)) print(' '.join(header) + '\n') # Build the output that will be displayed as a table. @@ -81,14 +81,14 @@ def status(cur, **_): pager = 'stdout' output.append(('Current pager:', pager)) - output.append(('Server version:', '{} {}'.format( + output.append(('Server version:', '{0} {1}'.format( variables['version'], variables['version_comment']))) output.append(('Protocol version:', variables['protocol_version'])) if 'unix' in cur.connection.host_info.lower(): host_info = cur.connection.host_info else: - host_info = '{} via TCP/IP'.format(cur.connection.host) + host_info = '{0} via TCP/IP'.format(cur.connection.host) output.append(('Connection:', host_info)) @@ -114,12 +114,12 @@ def status(cur, **_): # Print the current server statistics. stats = [] - stats.append('Connections: {}'.format(status['Threads_connected'])) - stats.append('Questions: {}'.format(status['Queries'])) - stats.append('Slow queries: {}'.format(status['Slow_queries'])) - stats.append('Opens: {}'.format(status['Opened_tables'])) - stats.append('Flush tables: {}'.format(status['Flush_commands'])) - stats.append('Open tables: {}'.format(status['Open_tables'])) + stats.append('Connections: {0}'.format(status['Threads_connected'])) + stats.append('Questions: {0}'.format(status['Queries'])) + stats.append('Slow queries: {0}'.format(status['Slow_queries'])) + stats.append('Opens: {0}'.format(status['Opened_tables'])) + stats.append('Flush tables: {0}'.format(status['Flush_commands'])) + stats.append('Open tables: {0}'.format(status['Open_tables'])) queries_per_second = int(status['Queries']) / int(status['Uptime']) stats.append('Queries per second avg: {:.3f}'.format(queries_per_second)) stats = ' '.join(stats) diff --git a/mycli/packages/special/utils.py b/mycli/packages/special/utils.py index a14af5bd4..ef96093a9 100644 --- a/mycli/packages/special/utils.py +++ b/mycli/packages/special/utils.py @@ -40,7 +40,7 @@ def format_uptime(uptime_in_seconds): elif value == 1 and unit.endswith('s'): # Remove the "s" if the unit is singular. unit = unit[:-1] - uptime_values.append('{} {}'.format(value, unit)) + uptime_values.append('{0} {1}'.format(value, unit)) uptime = ' '.join(uptime_values) return uptime From e3a67c6120416d430575e58a208d02ed7565e995 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 5 Feb 2016 22:51:39 -0600 Subject: [PATCH 0161/1025] Renames Questions to Queries. --- mycli/packages/special/dbcommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/dbcommands.py b/mycli/packages/special/dbcommands.py index f31477d58..6e95f1e37 100644 --- a/mycli/packages/special/dbcommands.py +++ b/mycli/packages/special/dbcommands.py @@ -115,7 +115,7 @@ def status(cur, **_): # Print the current server statistics. stats = [] stats.append('Connections: {0}'.format(status['Threads_connected'])) - stats.append('Questions: {0}'.format(status['Queries'])) + stats.append('Queries: {0}'.format(status['Queries'])) stats.append('Slow queries: {0}'.format(status['Slow_queries'])) stats.append('Opens: {0}'.format(status['Opened_tables'])) stats.append('Flush tables: {0}'.format(status['Flush_commands'])) From 37704a5f7a7d45f56fd1f0be00259a9a7103af64 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 Feb 2016 13:33:41 -0600 Subject: [PATCH 0162/1025] Makes system command work with Python 3. --- mycli/packages/special/iocommands.py | 5 +++++ tests/test_iospecial.py | 13 +++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/test_iospecial.py diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 1971a7c93..70f943547 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -1,5 +1,6 @@ import os import re +import locale import logging import subprocess from io import open @@ -231,6 +232,10 @@ def execute_system_command(arg, **_): output, error = process.communicate() response = output if not error else error + if isinstance(response, bytes): + encoding = locale.getpreferredencoding(False) + response = response.decode(encoding) + return [(None, None, None, response)] except OSError as e: return [(None, None, None, 'OSError: %s' % e.strerror)] diff --git a/tests/test_iospecial.py b/tests/test_iospecial.py new file mode 100644 index 000000000..3579727ae --- /dev/null +++ b/tests/test_iospecial.py @@ -0,0 +1,13 @@ +from mycli.packages.special.iocommands import execute_system_command + +try: + basestring +except NameError: + basestring = str + + +def test_system_command(): + cmd = 'ls' + response = execute_system_command(cmd) + + assert isinstance(response[0][3], basestring) From 16df6fe849239112558680bdc5ebdefb50824c88 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 Feb 2016 13:35:20 -0600 Subject: [PATCH 0163/1025] Adds system command comments. --- mycli/packages/special/iocommands.py | 1 + tests/test_iospecial.py | 1 + 2 files changed, 2 insertions(+) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 70f943547..415e33d3e 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -232,6 +232,7 @@ def execute_system_command(arg, **_): output, error = process.communicate() response = output if not error else error + # Python 3 returns bytes. This needs to be decoded to a string. if isinstance(response, bytes): encoding = locale.getpreferredencoding(False) response = response.decode(encoding) diff --git a/tests/test_iospecial.py b/tests/test_iospecial.py index 3579727ae..8a357ca2b 100644 --- a/tests/test_iospecial.py +++ b/tests/test_iospecial.py @@ -7,6 +7,7 @@ def test_system_command(): + """Tests that the system command always returns a string.""" cmd = 'ls' response = execute_system_command(cmd) From 0325c94060b0e152f4c74cbaf87004e060679a29 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 Feb 2016 13:47:11 -0600 Subject: [PATCH 0164/1025] Fixes system-command tests. --- tests/test_iospecial.py | 14 -------------- tests/test_sqlexecute.py | 3 +-- 2 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 tests/test_iospecial.py diff --git a/tests/test_iospecial.py b/tests/test_iospecial.py deleted file mode 100644 index 8a357ca2b..000000000 --- a/tests/test_iospecial.py +++ /dev/null @@ -1,14 +0,0 @@ -from mycli.packages.special.iocommands import execute_system_command - -try: - basestring -except NameError: - basestring = str - - -def test_system_command(): - """Tests that the system command always returns a string.""" - cmd = 'ls' - response = execute_system_command(cmd) - - assert isinstance(response[0][3], basestring) diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index 286e172db..bb4c53d34 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -232,8 +232,7 @@ def test_system_command_output(executor): results = run(executor, 'system cat {0}'.format(test_file_path)) assert len(results) == 1 expected_line = u'mycli rocks!\n' - result_str = results[0].decode('utf-8') # python3 returns a bytes-string - assert expected_line == result_str + assert expected_line == results[0] @dbtest def test_cd_command_current_dir(executor): From a5a78774aa22f624ee83ff05b1fa6ab38fadd197 Mon Sep 17 00:00:00 2001 From: mrdeathless Date: Mon, 15 Feb 2016 20:59:36 +0100 Subject: [PATCH 0165/1025] Adds support for SSL connections --- mycli/completion_refresher.py | 2 +- mycli/main.py | 78 +++++++++++++++++++++++++++++++---- mycli/sqlexecute.py | 16 ++++--- 3 files changed, 81 insertions(+), 15 deletions(-) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index 9265ae9fe..b5b4142ed 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -48,7 +48,7 @@ def _bg_refresh(self, sqlexecute, callbacks): # Create a new pgexecute method to popoulate the completions. e = sqlexecute executor = SQLExecute(e.dbname, e.user, e.password, e.host, e.port, - e.socket, e.charset, e.local_infile) + e.socket, e.charset, e.local_infile, e.ssl) # If callbacks is a single function then push it into a list. if callable(callbacks): diff --git a/mycli/main.py b/mycli/main.py index 1c8c420de..b18ca169e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -243,11 +243,11 @@ def initialize_logging(self): root_logger.debug('Initializing mycli logging.') root_logger.debug('Log file %r.', log_file) - def connect_uri(self, uri, local_infile=None): + def connect_uri(self, uri, local_infile=None, ssl=None): uri = urlparse(uri) database = uri.path[1:] # ignore the leading fwd slash self.connect(database, uri.username, uri.password, uri.hostname, - uri.port, local_infile=local_infile) + uri.port, local_infile=local_infile, ssl=ssl) def read_my_cnf_files(self, files, keys): """ @@ -274,8 +274,31 @@ def get(key): return dict([(x, get(x)) for x in keys]) + def merge_ssl_with_cnf(self, ssl, cnf): + """Merge SSL configuration dict with cnf dict""" + + merged = {} + merged.update(ssl) + prefix = 'ssl-' + for k, v in cnf.items(): + # skip unrelated options + if not k.startswith(prefix): + continue + if v is None: + continue + # special case because PyMySQL argument is significantly different + # from commandline + if k == 'ssl-verify-server-cert': + merged['check_hostname'] = v + else: + # use argument name just strip "ssl-" prefix + arg = k[len(prefix):] + merged[arg] = v + + return merged + def connect(self, database='', user='', passwd='', host='', port='', - socket='', charset='', local_infile=''): + socket='', charset='', local_infile='', ssl=''): cnf = {'database': None, 'user': None, @@ -285,7 +308,13 @@ def connect(self, database='', user='', passwd='', host='', port='', 'socket': None, 'default-character-set': None, 'local-infile': None, - 'loose-local-infile': None} + 'loose-local-infile': None, + 'ssl-ca': None, + 'ssl-cert': None, + 'ssl-key': None, + 'ssl-cipher': None, + 'ssl-verify-serer-cert': None, + } cnf = self.read_my_cnf_files(self.cnf_files, cnf.keys()) @@ -299,6 +328,8 @@ def connect(self, database='', user='', passwd='', host='', port='', user = user or cnf['user'] or os.getenv('USER') host = host or cnf['host'] or 'localhost' port = port or cnf['port'] or 3306 + ssl = ssl or {} + try: port = int(port) except ValueError as e: @@ -318,18 +349,23 @@ def connect(self, database='', user='', passwd='', host='', port='', except (TypeError, ValueError): pass + ssl = self.merge_ssl_with_cnf(ssl, cnf) + # prune lone check_hostname=False + if not any(v for v in ssl.values()): + ssl = None + # Connect to the database. try: try: sqlexecute = SQLExecute(database, user, passwd, host, port, - socket, charset, local_infile) + socket, charset, local_infile, ssl) except OperationalError as e: if ('Access denied for user' in e.args[1]): passwd = click.prompt('Password', hide_input=True, show_default=False, type=str) sqlexecute = SQLExecute(database, user, passwd, host, port, - socket, charset, local_infile) + socket, charset, local_infile, ssl) else: raise e except Exception as e: # Connecting to a database could fail. @@ -635,6 +671,20 @@ def get_prompt(self, string): help='Password to connect to the database') @click.option('--pass', 'password', envvar='MYSQL_PWD', type=str, help='Password to connect to the database') +@click.option('--ssl-ca', help='CA file in PEM format', + type=click.Path(exists=True)) +@click.option('--ssl-capath', help='CA directory') +@click.option('--ssl-cert', help='X509 cert in PEM format', + type=click.Path(exists=True)) +@click.option('--ssl-key', help='X509 key in PEM format', + type=click.Path(exists=True)) +@click.option('--ssl-cipher', help='SSL cipher to use') +@click.option('--ssl-verify-server-cert', is_flag=True, + help=('Verify server\'s "Common Name" in its cert against ' + 'hostname used when connecting. This option is disabled ' + 'by default')) +# as of 2016-02-15 revocation list is not supported by underling PyMySQL +# library (--ssl-crl and --ssl-crlpath options in vanilla mysql client) @click.option('-v', '--version', is_flag=True, help='Version of mycli.') @click.option('-D', '--database', 'dbname', help='Database to use.') @click.option('-R', '--prompt', 'prompt', @@ -655,7 +705,8 @@ def get_prompt(self, string): @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, - login_path, auto_vertical_output, local_infile): + login_path, auto_vertical_output, local_infile, ssl_ca, ssl_capath, + ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert): if version: print('Version:', __version__) sys.exit(0) @@ -668,11 +719,20 @@ def cli(database, user, host, port, socket, password, dbname, # Choose which ever one has a valid value. database = database or dbname + ssl = { + 'ca': ssl_ca, + 'capath': ssl_capath, + 'cert': ssl_cert, + 'key': ssl_key, + 'cipher': ssl_cipher, + 'check_hostname': ssl_verify_server_cert, + } + ssl = dict((k, v) for (k, v) in ssl.items() if v is not None) if database and '://' in database: - mycli.connect_uri(database, local_infile) + mycli.connect_uri(database, local_infile, ssl) else: mycli.connect(database, user, password, host, port, socket, - local_infile=local_infile) + local_infile=local_infile, ssl=ssl) mycli.logger.debug('Launch Params: \n' '\tdatabase: %r' diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 64d488d06..57a7f70f0 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -27,7 +27,7 @@ class SQLExecute(object): order by table_name,ordinal_position''' def __init__(self, database, user, password, host, port, socket, charset, - local_infile): + local_infile, ssl): self.dbname = database self.user = user self.password = password @@ -36,11 +36,12 @@ def __init__(self, database, user, password, host, port, socket, charset, self.socket = socket self.charset = charset self.local_infile = local_infile + self.ssl = ssl self._server_type = None self.connect() def connect(self, database=None, user=None, password=None, host=None, - port=None, socket=None, charset=None, local_infile=None): + port=None, socket=None, charset=None, local_infile=None, ssl=None): db = (database or self.dbname) user = (user or self.user) password = (password or self.password) @@ -49,6 +50,7 @@ def connect(self, database=None, user=None, password=None, host=None, socket = (socket or self.socket) charset = (charset or self.charset) local_infile = (local_infile or self.local_infile) + ssl = (ssl or self.ssl) _logger.debug('Connection DB Params: \n' '\tdatabase: %r' '\tuser: %r' @@ -56,13 +58,16 @@ def connect(self, database=None, user=None, password=None, host=None, '\tport: %r' '\tsocket: %r' '\tcharset: %r' - '\tlocal_infile: %r', - database, user, host, port, socket, charset, local_infile) + '\tlocal_infile: %r' + '\tssl: %r', + database, user, host, port, socket, charset, local_infile, ssl) + conn = connection.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, use_unicode=True, charset=charset, autocommit=True, client_flag=pymysql.constants.CLIENT.INTERACTIVE, - cursorclass=connection.Cursor, local_infile=local_infile) + cursorclass=connection.Cursor, local_infile=local_infile, + ssl=ssl) if hasattr(self, 'conn'): self.conn.close() self.conn = conn @@ -75,6 +80,7 @@ def connect(self, database=None, user=None, password=None, host=None, self.port = port self.socket = socket self.charset = charset + self.ssl = ssl def run(self, statement): """Execute the sql in the database and return the results. The results From 9c6e7e18d9e6adc8f17d87c407934beb42af774f Mon Sep 17 00:00:00 2001 From: Artem Bezsmertnyi Date: Sun, 21 Feb 2016 06:38:58 +0100 Subject: [PATCH 0166/1025] Fixes sqlexecute tests Adding default value for ssl argument was chosen because it's a seldom used option. --- mycli/sqlexecute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 57a7f70f0..418e6bd87 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -27,7 +27,7 @@ class SQLExecute(object): order by table_name,ordinal_position''' def __init__(self, database, user, password, host, port, socket, charset, - local_infile, ssl): + local_infile, ssl=False): self.dbname = database self.user = user self.password = password From fdcd2ab012ee73d14675531801cd0c491172b876 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Tue, 23 Feb 2016 17:26:50 -0300 Subject: [PATCH 0167/1025] Add auto-completion and highlight support for OFFSET keyword --- mycli/lexer.py | 10 ++--- mycli/sqlcompleter.py | 94 +++++++++++++++++++++---------------------- 2 files changed, 49 insertions(+), 55 deletions(-) diff --git a/mycli/lexer.py b/mycli/lexer.py index 712386727..4b14d72de 100644 --- a/mycli/lexer.py +++ b/mycli/lexer.py @@ -4,13 +4,9 @@ class MyCliLexer(MySqlLexer): - """ - Extends MySQL lexer to add keywords. - """ + """Extends MySQL lexer to add keywords.""" tokens = { - 'root': [ - (r'\brepair\b', Keyword), - inherit, - ], + 'root': [(r'\brepair\b', Keyword), + (r'\boffset\b', Keyword), inherit], } diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 20a2b3c89..ccc1dabc8 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -16,46 +16,49 @@ _logger = logging.getLogger(__name__) + class SQLCompleter(Completer): keywords = ['ACCESS', 'ADD', 'ALL', 'ALTER TABLE', 'AND', 'ANY', 'AS', - 'ASC', 'AUDIT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', - 'CASE', 'CHANGE MASTER TO', 'CHAR', 'CHECK', 'CLUSTER', 'COLUMN', - 'COMMENT', 'COMPRESS', 'COMMIT', 'CONNECT', 'COPY', 'CREATE', - 'CURRENT', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', 'DELETE FROM', - 'DELIMITER', 'DESC', 'DESCRIBE', 'DISTINCT', 'DROP', 'ELSE', - 'ENCODING', 'END', 'ESCAPE', 'EXCLUSIVE', 'EXISTS', 'EXTENSION', - 'FILE', 'FLOAT', 'FOR', 'FORMAT', 'FORCE_QUOTE', 'FORCE_NOT_NULL', - 'FREEZE', 'FROM', 'FULL', 'FUNCTION', 'GRANT', 'GROUP BY', - 'HAVING', 'HEADER', 'HOST', 'IDENTIFIED', 'IMMEDIATE', 'IN', - 'INCREMENT', 'INDEX', 'INITIAL', 'INSERT INTO', 'INTEGER', - 'INTERSECT', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'LEFT', 'LEVEL', - 'LIKE', 'LIMIT', 'LOCK', 'LOG', 'LOGS', 'LONG', 'MASTER', 'MINUS', - 'MODE', 'MODIFY', 'NOAUDIT', 'NOCOMPRESS', 'NOT', 'NOWAIT', 'NULL', - 'NUMBER', 'OIDS', 'OF', 'OFFLINE', 'ON', 'ONLINE', 'OPTION', 'OR', - 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', 'PCTFREE', 'PORT', - 'PRIMARY', 'PRIOR', 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'QUOTE', - 'RAW', 'RENAME', 'REPAIR', 'RESOURCE', 'RESET', 'REVOKE', 'RIGHT', - 'ROLLBACK', 'ROW', 'ROWID', 'ROWNUM', 'ROWS', 'SELECT', 'SESSION', - 'SET', 'SHARE', 'SHOW', 'SIZE', 'SLAVE', 'SLAVES', 'SMALLINT', - 'START', 'STOP', 'SUCCESSFUL', 'SYNONYM', 'SYSDATE', 'TABLE', - 'TEMPLATE', 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', - 'UID', 'UNION', 'UNIQUE', 'UPDATE', 'USE', 'USER', 'USING', - 'VALIDATE', 'VALUES', 'VARCHAR', 'VARCHAR2', 'VIEW', 'WHEN', - 'WHENEVER', 'WHERE', 'WITH'] + 'ASC', 'AUDIT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', + 'CASE', 'CHANGE MASTER TO', 'CHAR', 'CHECK', 'CLUSTER', + 'COLUMN', 'COMMENT', 'COMPRESS', 'COMMIT', 'CONNECT', 'COPY', + 'CREATE', 'CURRENT', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', + 'DELETE FROM', 'DELIMITER', 'DESC', 'DESCRIBE', 'DISTINCT', + 'DROP', 'ELSE', 'ENCODING', 'END', 'ESCAPE', 'EXCLUSIVE', + 'EXISTS', 'EXTENSION', 'FILE', 'FLOAT', 'FOR', 'FORMAT', + 'FORCE_QUOTE', 'FORCE_NOT_NULL', 'FREEZE', 'FROM', 'FULL', + 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HEADER', 'HOST', + 'IDENTIFIED', 'IMMEDIATE', 'IN', 'INCREMENT', 'INDEX', + 'INITIAL', 'INSERT INTO', 'INTEGER', 'INTERSECT', 'INTO', + 'INTERVAL', 'IS', 'JOIN', 'LEFT', 'LEVEL', 'LIKE', 'LIMIT', + 'LOCK', 'LOG', 'LOGS', 'LONG', 'MASTER', 'MINUS', 'MODE', + 'MODIFY', 'NOAUDIT', 'NOCOMPRESS', 'NOT', 'NOWAIT', 'NULL', + 'NUMBER', 'OIDS', 'OF', 'OFFLINE', 'OFFSET', 'ON', 'ONLINE', + 'OPTION', 'OR', 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', + 'PCTFREE', 'PORT', 'PRIMARY', 'PRIOR', 'PRIVILEGES', + 'PROCESSLIST', 'PURGE', 'QUOTE', 'RAW', 'RENAME', 'REPAIR', + 'RESOURCE', 'RESET', 'REVOKE', 'RIGHT', 'ROLLBACK', 'ROW', + 'ROWID', 'ROWNUM', 'ROWS', 'SELECT', 'SESSION', 'SET', 'SHARE', + 'SHOW', 'SIZE', 'SLAVE', 'SLAVES', 'SMALLINT', 'START', 'STOP', + 'SUCCESSFUL', 'SYNONYM', 'SYSDATE', 'TABLE', 'TEMPLATE', + 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UID', + 'UNION', 'UNIQUE', 'UPDATE', 'USE', 'USER', 'USING', + 'VALIDATE', 'VALUES', 'VARCHAR', 'VARCHAR2', 'VIEW', 'WHEN', + 'WHENEVER', 'WHERE', 'WITH'] functions = ['AVG', 'COUNT', 'DISTINCT', 'FIRST', 'FORMAT', 'LAST', - 'LCASE', 'LEN', 'MAX', 'MIN', 'MID', 'NOW', 'ROUND', 'SUM', 'TOP', - 'UCASE'] + 'LCASE', 'LEN', 'MAX', 'MIN', 'MID', 'NOW', 'ROUND', 'SUM', + 'TOP', 'UCASE'] show_items = [] change_items = ['MASTER_BIND', 'MASTER_HOST', 'MASTER_USER', - 'MASTER_PASSWORD', 'MASTER_PORT', 'MASTER_CONNECT_RETRY', - 'MASTER_HEARTBEAT_PERIOD', 'MASTER_LOG_FILE', 'MASTER_LOG_POS', - 'RELAY_LOG_FILE', 'RELAY_LOG_POS', 'MASTER_SSL', 'MASTER_SSL_CA', - 'MASTER_SSL_CAPATH', 'MASTER_SSL_CERT', 'MASTER_SSL_KEY', - 'MASTER_SSL_CIPHER', 'MASTER_SSL_VERIFY_SERVER_CERT', - 'IGNORE_SERVER_IDS'] + 'MASTER_PASSWORD', 'MASTER_PORT', 'MASTER_CONNECT_RETRY', + 'MASTER_HEARTBEAT_PERIOD', 'MASTER_LOG_FILE', + 'MASTER_LOG_POS', 'RELAY_LOG_FILE', 'RELAY_LOG_POS', + 'MASTER_SSL', 'MASTER_SSL_CA', 'MASTER_SSL_CAPATH', + 'MASTER_SSL_CERT', 'MASTER_SSL_KEY', 'MASTER_SSL_CIPHER', + 'MASTER_SSL_VERIFY_SERVER_CERT', 'IGNORE_SERVER_IDS'] users = [] @@ -75,12 +78,12 @@ def escape_name(self, name): if name and ((not self.name_pattern.match(name)) or (name.upper() in self.reserved_words) or (name.upper() in self.functions)): - name = '`%s`' % name + name = '`%s`' % name return name def unescape_name(self, name): - """ Unquote a string.""" + """Unquote a string.""" if name and name[0] == '"' and name[-1] == '"': name = name[1:-1] @@ -128,13 +131,12 @@ def extend_schemata(self, schema): self.all_completions.update(schema) def extend_relations(self, data, kind): - """ extend metadata for tables or views + """Extend metadata for tables or views :param data: list of (rel_name, ) tuples :param kind: either 'tables' or 'views' :return: """ - # 'data' is a generator object. It can throw an exception while being # consumed. This could happen if the user has launched the app without # specifying a database name. This exception must be handled to prevent @@ -156,7 +158,7 @@ def extend_relations(self, data, kind): self.all_completions.add(relname[0]) def extend_columns(self, column_data, kind): - """ extend column metadata + """Extend column metadata :param column_data: list of (rel_name, column_name) tuples :param kind: either 'tables' or 'views' @@ -203,7 +205,6 @@ def reset_completions(self): self.dbmetadata = {'tables': {}, 'views': {}, 'functions': {}} self.all_completions = set(self.keywords + self.functions) - @staticmethod def find_matches(text, collection, start_only=False, fuzzy=True): """Find completion matches for the given text. @@ -218,9 +219,7 @@ def find_matches(text, collection, start_only=False, fuzzy=True): yields prompt_toolkit Completion instances for any matches found in the collection of available completions. - """ - text = last_word(text, include='most_punctuations').lower() completions = [] @@ -267,9 +266,10 @@ def get_completions(self, document, complete_event, smart_completion=None): # drop_unique is used for 'tb11 JOIN tbl2 USING (...' # which should suggest only columns that appear in more than # one table - scoped_cols = [col for (col, count) - in Counter(scoped_cols).items() - if count > 1 and col != '*'] + scoped_cols = [ + col for (col, count) in Counter(scoped_cols).items() + if count > 1 and col != '*' + ] cols = self.find_matches(word_before_cursor, scoped_cols) completions.extend(cols) @@ -334,8 +334,8 @@ def get_completions(self, document, complete_event, smart_completion=None): completions.extend(change_items) elif suggestion['type'] == 'user': users = self.find_matches(word_before_cursor, self.users, - start_only=False, - fuzzy=True) + start_only=False, + fuzzy=True) completions.extend(users) elif suggestion['type'] == 'special': @@ -358,11 +358,10 @@ def get_completions(self, document, complete_event, smart_completion=None): return completions def populate_scoped_cols(self, scoped_tbls): - """ Find all columns in a set of scoped_tables + """Find all columns in a set of scoped_tables :param scoped_tbls: list of (schema, table, alias) tuples :return: list of column names """ - columns = [] meta = self.dbmetadata @@ -398,7 +397,6 @@ def populate_scoped_cols(self, scoped_tbls): def populate_schema_objects(self, schema, obj_type): """Returns list of tables or functions for a (optional) schema""" - metadata = self.dbmetadata[obj_type] schema = schema or self.dbname From 38fa3698076080b09771856352496a6ca696aba5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 29 Feb 2016 22:15:18 -0600 Subject: [PATCH 0168/1025] Makes status command return output, not print output. --- mycli/packages/special/dbcommands.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/mycli/packages/special/dbcommands.py b/mycli/packages/special/dbcommands.py index 6e95f1e37..997d3d0a3 100644 --- a/mycli/packages/special/dbcommands.py +++ b/mycli/packages/special/dbcommands.py @@ -2,7 +2,6 @@ import os import platform from mycli import __version__ -from mycli.packages.tabulate import tabulate from mycli.packages.special import iocommands from mycli.packages.special.utils import format_uptime from .main import special_command, RAW_QUERY, PARSED_QUERY @@ -47,19 +46,22 @@ def status(cur, **_): cur.execute(query) variables = dict(cur.fetchall()) - print('--------------') + # Create output buffers. + title = [] + output = [] + footer = [] + + title.append('--------------') # Output the mycli client information. implementation = platform.python_implementation() version = platform.python_version() - header = [] - header.append('mycli {0},'.format(__version__)) - header.append('running on {0} {1}'.format(implementation, version)) - print(' '.join(header) + '\n') + client_info = [] + client_info.append('mycli {0},'.format(__version__)) + client_info.append('running on {0} {1}'.format(implementation, version)) + title.append(' '.join(client_info) + '\n') # Build the output that will be displayed as a table. - output = [] - output.append(('Connection id:', cur.connection.thread_id())) query = 'SELECT DATABASE(), USER();' @@ -109,9 +111,6 @@ def status(cur, **_): output.append(('Uptime:', format_uptime(status['Uptime']))) - # Print the buffered output in two columns. - print(tabulate(output, tablefmt='plain')[0]) - # Print the current server statistics. stats = [] stats.append('Connections: {0}'.format(status['Threads_connected'])) @@ -123,7 +122,7 @@ def status(cur, **_): queries_per_second = int(status['Queries']) / int(status['Uptime']) stats.append('Queries per second avg: {:.3f}'.format(queries_per_second)) stats = ' '.join(stats) - print('\n' + stats) + footer.append('\n' + stats) - print('--------------') - return [(None, None, None, '')] + footer.append('--------------') + return [('\n'.join(title), output, '', '\n'.join(footer))] From bf1f7ed2286e18732b9dd8906f40b9f9b8871dd8 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 29 Feb 2016 22:29:15 -0600 Subject: [PATCH 0169/1025] removes -S less option. --- mycli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 1c8c420de..d45508cb6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -310,7 +310,7 @@ def connect(self, database='', user='', passwd='', host='', port='', charset = charset or cnf['default-character-set'] or 'utf8' # Favor whichever local_infile option is set. - for local_infile_option in (local_infile, cnf['local-infile'], + for local_infile_option in (local_infile, cnf['local-infile'], cnf['loose-local-infile'], False): try: local_infile = str_to_bool(local_infile_option) @@ -574,7 +574,7 @@ def output_via_pager(self, text): def configure_pager(self): # Provide sane defaults for less. - os.environ['LESS'] = '-SRXF' + os.environ['LESS'] = '-RXF' cnf = self.read_my_cnf_files(self.cnf_files, ['pager', 'skip-pager']) if cnf['pager']: From e32f419672601fea71d428d9455c550b38acc9c2 Mon Sep 17 00:00:00 2001 From: Artem Bezsmertnyi Date: Thu, 3 Mar 2016 18:10:07 +0100 Subject: [PATCH 0170/1025] Adds support for user folder in paths to ssl files --- mycli/main.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index b18ca169e..509fbb965 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -3,6 +3,7 @@ from __future__ import print_function import os +import os.path import sys import traceback import logging @@ -719,14 +720,23 @@ def cli(database, user, host, port, socket, password, dbname, # Choose which ever one has a valid value. database = database or dbname - ssl = { + ssl_paths = { 'ca': ssl_ca, - 'capath': ssl_capath, 'cert': ssl_cert, 'key': ssl_key, + } + ssl_params = { + 'capath': ssl_capath, 'cipher': ssl_cipher, 'check_hostname': ssl_verify_server_cert, } + for p in ssl_paths: + if p: + ssl_paths[p] = os.path.expanduser(p) + ssl = {} + ssl.update(ssl_paths) + ssl.update(ssl_params) + # remove empty ssl options ssl = dict((k, v) for (k, v) in ssl.items() if v is not None) if database and '://' in database: mycli.connect_uri(database, local_infile, ssl) From bc65e9864e19a9c178dd6ee9bd2fd253f2e37c7e Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 3 Mar 2016 09:38:53 -0800 Subject: [PATCH 0171/1025] Use expanduser to expand the tilde in the path names. --- mycli/main.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 4335cc4d2..8bc3e171e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -720,22 +720,15 @@ def cli(database, user, host, port, socket, password, dbname, # Choose which ever one has a valid value. database = database or dbname - ssl_paths = { - 'ca': ssl_ca, - 'cert': ssl_cert, - 'key': ssl_key, - } - ssl_params = { - 'capath': ssl_capath, - 'cipher': ssl_cipher, - 'check_hostname': ssl_verify_server_cert, - } - for p in ssl_paths: - if p: - ssl_paths[p] = os.path.expanduser(p) - ssl = {} - ssl.update(ssl_paths) - ssl.update(ssl_params) + ssl = { + 'ca': os.path.expanduser(ssl_ca), + 'cert': os.path.expanduser(ssl_cert), + 'key': os.path.expanduser(ssl_key), + 'capath': ssl_capath, + 'cipher': ssl_cipher, + 'check_hostname': ssl_verify_server_cert, + } + # remove empty ssl options ssl = dict((k, v) for (k, v) in ssl.items() if v is not None) if database and '://' in database: From 2418a05c9ffbdccbc5190ff3183c0e00fc6b10ca Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 3 Mar 2016 12:53:36 -0800 Subject: [PATCH 0172/1025] Make sure the paths are not empty. --- mycli/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 8bc3e171e..33812f599 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -721,9 +721,9 @@ def cli(database, user, host, port, socket, password, dbname, database = database or dbname ssl = { - 'ca': os.path.expanduser(ssl_ca), - 'cert': os.path.expanduser(ssl_cert), - 'key': os.path.expanduser(ssl_key), + 'ca': ssl_ca and os.path.expanduser(ssl_ca), + 'cert': ssl_cert and os.path.expanduser(ssl_cert), + 'key': ssl_key and os.path.expanduser(ssl_key), 'capath': ssl_capath, 'cipher': ssl_cipher, 'check_hostname': ssl_verify_server_cert, From bac9e9650065fe2eab93199c60b3c857219dcfe4 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 4 Mar 2016 21:47:56 -0600 Subject: [PATCH 0173/1025] Simplify status message generation. --- mycli/sqlexecute.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 418e6bd87..2c3cc60ff 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -122,28 +122,19 @@ def execute_normal_sql(self, split_sql): _logger.debug('Regular sql statement. sql: %r', split_sql) cur = self.conn.cursor() num_rows = cur.execute(split_sql) - title = None - if num_rows == 1: - status = '%d row in set' % num_rows - else: - status = '%d rows in set' % num_rows - with self.conn.cursor() as temp_cursor: - temp_cursor.execute('SELECT row_count()') - n = temp_cursor.fetchone()[0] - if n < 0: - pass - elif n == 1: - status = 'Query OK, %d row affected' % n - else: - status = 'Query OK, %d rows affected' % n - # cur.description will be None for operations that do not return - # rows. - if cur.description: + title = headers = None + + # cur.description is not None for queries that return result sets, e.g. + # SELECT or SHOW. + if cur.description is not None: headers = [x[0] for x in cur.description] - return (title, cur, headers, status) # cur.statusmessage) + status = '{} row{} in set' else: _logger.debug('No rows in result.') - return (title, None, None, status) # cur.statusmessage) + status = 'Query OK, {} row{} affected' + status = status.format(num_rows, '' if num_rows == 1 else 's') + + return (title, cur if cur.description else None, headers, status) def tables(self): """Yields table names""" From 8f81c19ae41bad9baf9953e7fb546ee1d5d3c8a6 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 4 Mar 2016 22:00:13 -0600 Subject: [PATCH 0174/1025] Fix Python 2.6 support. --- mycli/sqlexecute.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 2c3cc60ff..de977fe6f 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -128,10 +128,10 @@ def execute_normal_sql(self, split_sql): # SELECT or SHOW. if cur.description is not None: headers = [x[0] for x in cur.description] - status = '{} row{} in set' + status = '{0} row{1} in set' else: _logger.debug('No rows in result.') - status = 'Query OK, {} row{} affected' + status = 'Query OK, {0} row{1} affected' status = status.format(num_rows, '' if num_rows == 1 else 's') return (title, cur if cur.description else None, headers, status) From 8db5307f83441cc05f37c5f6a01447635325284e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 12 Mar 2016 05:59:30 -0600 Subject: [PATCH 0175/1025] Compare PyMySQL version using int, not str. --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 33812f599..77d2f0307 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -533,7 +533,7 @@ def prompt_tokens(cli): mutating = mutating or is_mutating(status) except UnicodeDecodeError as e: import pymysql - if pymysql.VERSION < ('0', '6', '7'): + if pymysql.VERSION < (0, 6, 7): message = ('You are running an older version of pymysql.\n' 'Please upgrade to 0.6.7 or above to view binary data.\n' 'Try \'pip install -U pymysql\'.') From 8b2a775a44706fb6c29510d144c4db45b65bc8f1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 12 Mar 2016 05:59:51 -0600 Subject: [PATCH 0176/1025] Skip binary tests if PyMySQL version does not support it. --- tests/test_sqlexecute.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index bb4c53d34..ac5152480 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -6,6 +6,9 @@ from textwrap import dedent from utils import run, dbtest, set_expanded_output + +pymysql_support_binary = pymysql.VERSION >= (0, 6, 7) + @dbtest def test_conn(executor): run(executor, '''create table test(a text)''') @@ -33,6 +36,7 @@ def test_bools(executor): 1 row in set""") @dbtest +@pytest.mark.skipif(not pymysql_support_binary, reason='pymysql < 0.6.7') def test_binary(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') run(executor, '''INSERT INTO bt VALUES (GeomFromText('LINESTRING(116.37604 39.73979,116.375 39.73965)'));''') @@ -46,6 +50,7 @@ def test_binary(executor): 1 row in set""") @dbtest +@pytest.mark.skipif(not pymysql_support_binary, reason='pymysql < 0.6.7') def test_binary_expanded(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') run(executor, '''INSERT INTO bt VALUES (GeomFromText('LINESTRING(116.37604 39.73979,116.375 39.73965)'));''') From 173b0d476ac260883481ae4272a028113bc69cb1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 12 Mar 2016 07:08:16 -0600 Subject: [PATCH 0177/1025] Remove license meta-data. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 92b11a219..054b9e987 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ author='Amjith Ramanujam', author_email='amjith[dot]r[at]gmail.com', version=version, - license='LICENSE.txt', url='http://mycli.net', packages=find_packages(), package_data={'mycli': ['myclirc', '../AUTHORS', '../SPONSORS']}, From 66463ba4cf2b545d4120495cba589198fac59eb9 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 12 Mar 2016 12:47:02 -0600 Subject: [PATCH 0178/1025] Add Python 3.5 to test environments. --- .travis.yml | 1 + setup.py | 1 + tox.ini | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5a8610ec0..5450b982b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "2.7" - "3.3" - "3.4" + - "3.5" env: - PYMYSQL_VERSION=0.6.7 diff --git a/setup.py b/setup.py index 92b11a219..f1385c942 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Programming Language :: SQL', 'Topic :: Database', 'Topic :: Database :: Front-Ends', diff --git a/tox.ini b/tox.ini index 96d168014..a2d769118 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34 +envlist = py26, py27, py33, py34, py35 [testenv] deps = pytest mock From 37146fb696e02ffbc8248f222f8a7889aa7fc479 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 20 Mar 2016 08:47:10 -0700 Subject: [PATCH 0179/1025] Upgrade prompt_toolkit to 0.60. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1d9de9e3c..a5eec5e93 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ install_requirements = [ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? - 'prompt_toolkit==0.57', + 'prompt_toolkit==0.60', 'PyMySQL >= 0.6.2', 'sqlparse >= 0.1.16', 'configobj >= 5.0.6', From 14beb3d87d4c28b4ca6c4722bed297209ddf5092 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 20 Mar 2016 08:47:31 -0700 Subject: [PATCH 0180/1025] Add continutation prompt for multi-line mode. --- mycli/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 77d2f0307..e423aa76f 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -429,12 +429,16 @@ def set_key_bindings(value): def prompt_tokens(cli): return [(Token.Prompt, self.get_prompt(self.prompt_format))] + def get_continuation_tokens(cli, width): + return [(Token.Continuation, ' ' * (width - 3) + '-> ')] + get_toolbar_tokens = create_toolbar_tokens_func(lambda: self.key_bindings, self.completion_refresher.is_refreshing) layout = create_prompt_layout(lexer=MyCliLexer, multiline=True, get_prompt_tokens=prompt_tokens, + get_continuation_tokens=get_continuation_tokens, get_bottom_toolbar_tokens=get_toolbar_tokens, display_completions_in_columns=self.wider_completion_menu, extra_input_processors=[ From 8363d05e66df41bcb3917706cbeed8e73c081426 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 20 Mar 2016 18:29:29 -0700 Subject: [PATCH 0181/1025] Update AUTHORS file. --- AUTHORS | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0c566da43..a6552890c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,21 +13,29 @@ Contributors: * Steve Robbins * Daniel West + * Shoma Suzuki * Daniel Black * Jonathan Bruno + * Artem Bezsmertnyi * Heath Naylor + * Mikhail Borisov + * Phil Cohen + * Yasuhiro Matsumoto * bjarnagin * jbruno + * mrdeathless * Abirami P * spacewander * Adam Chainz + * Casper Langemeijer * Johannes Hoff * Jonathan Slenders * Kacper Kwapisz + * Lennart Weller * Martijn Engler - * Shoma Suzuki + * Terseus * Tyler Kuipers - * Yasuhiro Matsumoto + * William GARCIA Creator: -------- From 954c8620e4838a73a384cb0b679858567345a61b Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 20 Mar 2016 19:00:12 -0700 Subject: [PATCH 0182/1025] Update changelog for release 1.6.0. --- changelog.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/changelog.md b/changelog.md index 7e3446b75..22827a49f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,35 @@ +1.6.0: +====== + +Features: +--------- + +* Change continutation prompt for multi-line mode to match default mysql. +* Add `status` command to match mysql's `status` command. (Thanks: [Thomas Roten]). +* Add SSL support for `mycli`. (Thanks: [Artem Bezsmertnyi]). +* Add auto-completion and highlight support for OFFSET keyword. (Thanks: [Matheus Rosa]). +* Add support for `MYSQL_TEST_LOGIN_FILE` env variable to specify alternate login file. (Thanks: [Thomas Roten]). +* Add support for `--auto-vertical-output` to automatically switch to vertical output if the output doesn't fit in the table format. +* Add support for system-wide config. Now /etc/myclirc will be honored. (Thanks: [Thomas Roten]). + +Bug Fixes: +---------- + +* Remove -S from `less` option which was clobbering the scroll back in history. (Thanks: [Thomas Roten]). +* Make system command work with Python 3. (Thanks: [Thomas Roten]). +* Support \G terminator for \f queries. (Thanks: [Terseus]). + +Internal Changes: +----------------- + +* Upgrade prompt_toolkit to 0.60. +* Add Python 3.5 to test environments. (Thanks: [Thomas Roten]). +* Remove license meta-data. (Thanks: [Thomas Roten]). +* Skip binary tests if PyMySQL version does not support it. (Thanks: [Thomas Roten]). +* Refactor pager handling. (Thanks: [Thomas Roten]) +* Capture warnings to log file. (Thanks: [Mikhail Borisov]). +* Make `syntax_style` a tiny bit more intuitive. (Thanks: [Phil Cohen]). + 1.5.2: ====== @@ -252,3 +284,10 @@ Bug Fixes: [Shoma Suzuki]: https://github.com/shoma [spacewander]: https://github.com/spacewander [Thomas Roten]: https://github.com/tsroten +[Artem Bezsmertnyi]: https://github.com/mrdeathless +[Mikhail Borisov]: https://github.com/borman +[Casper Langemeijer]: Casper Langemeijer +[Lennart Weller]: https://github.com/lhw +[Phil Cohen]: https://github.com/phlipper +[Terseus]: https://github.com/Terseus +[William GARCIA]: https://github.com/willgarcia From 300a8fbcb4c290cd5d98e983f16537b2574d6a14 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 22 Mar 2016 07:50:11 -0500 Subject: [PATCH 0183/1025] Add stdin batch mode. --- mycli/main.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index e423aa76f..3d875df99 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -703,6 +703,8 @@ def get_prompt(self, string): help='Only read default options from the given file') @click.option('--auto-vertical-output', is_flag=True, help='Automatically switch to vertical output mode if the result is wider than the terminal width.') +@click.option('-t', '--table', is_flag=True, + help='Display batch output in table format.') @click.option('--local-infile', type=bool, help='Enable/disable LOAD DATA LOCAL INFILE.') @click.option('--login-path', type=str, @@ -711,7 +713,8 @@ def get_prompt(self, string): def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, login_path, auto_vertical_output, local_infile, ssl_ca, ssl_capath, - ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert): + ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table): + if version: print('Version:', __version__) sys.exit(0) @@ -747,6 +750,17 @@ def cli(database, user, host, port, socket, password, dbname, '\thost: %r' '\tport: %r', database, user, host, port) + stdin = click.get_text_stream('stdin') + if not stdin.isatty(): + results = mycli.sqlexecute.run(stdin.read()) + for result in results: + title, cur, headers, status = result + table_format = mycli.table_format if table else None + output = format_output(title, cur, headers, None, table_format) + for line in output: + click.echo(line) + exit(0) + mycli.run_cli() def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): @@ -757,7 +771,7 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max headers = [utf8tounicode(x) for x in headers] if expanded: output.append(expanded_table(cur, headers)) - else: + elif table_format is not None: rows = list(cur) tabulated, frows = tabulate(rows, headers, tablefmt=table_format, missingval='') @@ -767,6 +781,10 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max output.append(expanded_table(rows, headers)) else: output.append(tabulated) + else: + output.append('\t'.join(headers)) + for row in cur: + output.append('\t'.join([str(r) for r in row])) if status: # Only print the status if it's not None. output.append(status) return output From c9a0a3a8a26c56f47f3993f1e4832064c15bafa8 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 22 Mar 2016 07:50:28 -0500 Subject: [PATCH 0184/1025] Add test for stdin batch mode --- tests/test_main.py | 52 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index de7737644..898c5dde9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,10 @@ -import pytest -from mycli.main import format_output +from click.testing import CliRunner + +from mycli.main import cli, format_output +from utils import USER, HOST, PORT, PASSWORD, dbtest, run + +CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, + '--password', PASSWORD, '_test_db'] def test_format_output(): results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], @@ -19,3 +24,46 @@ def test_format_output_auto_expand(): max_width=1) expanded = ['Title', u'***************************[ 1. row ]***************************\nhead1 | abc\nhead2 | def\n', 'test status'] assert expanded_results == expanded + +def test_format_output_no_table(): + results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], + 'test status', None) + expected = ['Title', 'head1\thead2', 'abc\tdef', 'test status'] + assert results == expected + +@dbtest +def test_batch_mode(executor): + run(executor, '''create table test(a text)''') + run(executor, '''insert into test values('abc'), ('def'), ('ghi')''') + + sql = ( + 'select count(*) from test;\n' + 'select * from test limit 1;' + ) + + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS, input=sql) + + assert result.exit_code == 0 + assert 'count(*)\n3\na\nabc' in result.output + +@dbtest +def test_batch_mode_table(executor): + run(executor, '''create table test(a text)''') + run(executor, '''insert into test values('abc'), ('def'), ('ghi')''') + + sql = ( + 'select count(*) from test;\n' + 'select * from test limit 1;' + ) + + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['-t'], input=sql) + + expected = ( + '| count(*) |\n|------------|\n| 3 |\n+------------+\n' + '+-----+\n| a |\n|-----|\n| abc |\n+-----+' + ) + + assert result.exit_code == 0 + assert expected in result.output From 2a77c376c89ac9a3accafae7c1e94a5c56e7f2f1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 22 Mar 2016 07:59:48 -0500 Subject: [PATCH 0185/1025] Fix test_special test. --- tests/test_sqlexecute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index ac5152480..4a6f99101 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -213,7 +213,7 @@ def test_favorite_query_expanded_output(executor): @dbtest def test_special_command(executor): results = run(executor, '\\?') - expected_line = u'| help | \\? | Show this help. |\n' + expected_line = u'| help | \\? | Show this help. |\n' assert len(results) == 1 assert expected_line in results[0] From 2f957b7a37c3e31be7d010ad7282b1fbaa42e7c9 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 22 Mar 2016 18:53:15 -0700 Subject: [PATCH 0186/1025] Add missing features changelog and fix typo. --- changelog.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 22827a49f..759c3b4d8 100644 --- a/changelog.md +++ b/changelog.md @@ -4,13 +4,15 @@ Features: --------- -* Change continutation prompt for multi-line mode to match default mysql. +* Change continuation prompt for multi-line mode to match default mysql. * Add `status` command to match mysql's `status` command. (Thanks: [Thomas Roten]). * Add SSL support for `mycli`. (Thanks: [Artem Bezsmertnyi]). * Add auto-completion and highlight support for OFFSET keyword. (Thanks: [Matheus Rosa]). * Add support for `MYSQL_TEST_LOGIN_FILE` env variable to specify alternate login file. (Thanks: [Thomas Roten]). * Add support for `--auto-vertical-output` to automatically switch to vertical output if the output doesn't fit in the table format. * Add support for system-wide config. Now /etc/myclirc will be honored. (Thanks: [Thomas Roten]). +* Add support for `nopager` and `\n` to turn off the pager. (Thanks: [Thomas Roten]). +* Add support for `--local-infile` command-line option. (Thanks: [Thomas Roten]). Bug Fixes: ---------- From 27f714f9c89d24fdc98467ec66a2cc6c21f1a31d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 23 Mar 2016 08:52:06 -0500 Subject: [PATCH 0187/1025] Only capture warnings on Python 2.7 and later. --- mycli/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 3d875df99..2f4a53554 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -239,7 +239,11 @@ def initialize_logging(self): root_logger.addHandler(handler) root_logger.setLevel(level_map[log_level.upper()]) - logging.captureWarnings(True) + # Only capture warnings on Python 2.7 and later. + try: + logging.captureWarnings(True) + except AttributeError: + pass root_logger.debug('Initializing mycli logging.') root_logger.debug('Log file %r.', log_file) From b48e1b4b31cdf70e23d87e5a1a9d772121e62ad1 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 24 Mar 2016 07:53:03 -0700 Subject: [PATCH 0188/1025] Releasing version 1.6.0 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index c3b384154..bcd8d54ef 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.5.2' +__version__ = '1.6.0' From c599eb4f10f0c8a1a7650b4a02da6a715eab535d Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Thu, 24 Mar 2016 14:32:38 -0300 Subject: [PATCH 0189/1025] Remove extra \n in features list in README.md --- README.md | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index af6c67498..d40447330 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A command line client for MySQL that can do auto-completion and syntax highlight HomePage: [http://mycli.net](http://mycli.net) -Debian Packages via [PackageCloud.io](https://packagecloud.io/amjith/mycli). +Debian Packages via [PackageCloud.io](https://packagecloud.io/amjith/mycli). ![Completion](screenshots/tables.png) ![CompletionGif](screenshots/main.gif) @@ -76,20 +76,13 @@ Features columns in the database. * Syntax highlighting using Pygments. * Smart-completion (enabled by default) will suggest context-sensitive completion. - - - `SELECT * FROM ` will only show table names. - - `SELECT * FROM users WHERE ` will only show column names. - + - `SELECT * FROM ` will only show table names. + - `SELECT * FROM users WHERE ` will only show column names. * Support for multiline queries. - * Favorite queries. Save a query using `\fs alias query` and execute it with `\f alias` whenever you need. - * Timing of sql statments and table rendering. - * Config file is automatically created at ``~/.myclirc`` at first launch. - * Log every query and its results to a file (disabled by default). - * Pretty prints tabular data. Contributions: @@ -101,9 +94,9 @@ get this running in a development setup. https://github.com/dbcli/mycli/blob/master/DEVELOP.rst -Please feel free to reach out to me if you need help. +Please feel free to reach out to me if you need help. -My email: amjith.r@gmail.com +My email: amjith.r@gmail.com Twitter: [@amjithr](http://twitter.com/amjithr) @@ -118,7 +111,7 @@ Add the gpg key for packagecloud for package verification. ``` curl https://packagecloud.io/gpg.key | apt-key add - ``` - + Install a package called apt-transport-https to make it possible for apt to fetch packages over https. ``` @@ -156,10 +149,10 @@ $ sudo pip install mycli ### Thanks: -This project was funded through kickstarter. My thanks to the [backers](http://mycli.net/sponsors) who supported the project. +This project was funded through kickstarter. My thanks to the [backers](http://mycli.net/sponsors) who supported the project. A special thanks to [Jonathan Slenders](https://twitter.com/jonathan_s) for -creating [Python Prompt Toolkit](http://github.com/jonathanslenders/python-prompt-toolkit), +creating [Python Prompt Toolkit](http://github.com/jonathanslenders/python-prompt-toolkit), which is quite literally the backbone library, that made this app possible. Jonathan has also provided valuable feedback and support during the development of this app. @@ -167,9 +160,9 @@ of this app. [Click](http://click.pocoo.org/3/) is used for command line option parsing and printing error messages. -Thanks to [PyMysql](http://www.pymysql.org/) for a pure python adapter to MySQL database. +Thanks to [PyMysql](http://www.pymysql.org/) for a pure python adapter to MySQL database. -[Tabulate](https://pypi.python.org/pypi/tabulate) library is used for pretty printing the output of tables. +[Tabulate](https://pypi.python.org/pypi/tabulate) library is used for pretty printing the output of tables. ### Compatibility From 4b5e02de24ebc2c627e9687e3a216db44b8b19c5 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Thu, 24 Mar 2016 14:34:02 -0300 Subject: [PATCH 0190/1025] Update features list in README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d40447330..94dd5b69c 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Features * Config file is automatically created at ``~/.myclirc`` at first launch. * Log every query and its results to a file (disabled by default). * Pretty prints tabular data. +* Support for SSL connections Contributions: -------------- From 33cecbd7f874e96c6a8ba1a545e70179dd82139f Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 27 Mar 2016 20:22:23 -0700 Subject: [PATCH 0191/1025] Upgrade sqlparse dependency to 0.1.19 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a5eec5e93..b82957543 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit==0.60', 'PyMySQL >= 0.6.2', - 'sqlparse >= 0.1.16', + 'sqlparse >= 0.1.19', 'configobj >= 5.0.6', ] From 01760e46224493f5f2857a8e00373e495efeab6e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 10 Apr 2016 15:06:04 -0700 Subject: [PATCH 0192/1025] Refactor confirm_destructive_query. --- mycli/main.py | 34 ++++++++++++++++++++++------------ tests/test_main.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 2f4a53554..f70ce73ea 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -841,24 +841,34 @@ def is_select(status): return False return status.split(None, 1)[0].lower() == 'select' +def query_starts_with(query, prefixes): + """Check if the query starts with any item from *prefixes*.""" + prefixes = [prefix.lower() for prefix in prefixes] + formatted_sql = sqlparse.format(query.lower(), strip_comments=True) + return formatted_sql.split()[0] in prefixes + +def queries_start_with(queries, prefixes): + """Check if any queries start with any item from *prefixes*.""" + for query in sqlparse.split(queries): + if query and query_starts_with(query, prefixes) is True: + return True + return False + +def is_destructive(queries): + keywords = ('drop', 'shutdown', 'delete', 'truncate') + return queries_start_with(queries, keywords) + def confirm_destructive_query(queries): - """Checks if the query is destructive and prompts the user to confirm. + """Check if the query is destructive and prompts the user to confirm. Returns: None if the query is non-destructive. True if the query is destructive and the user wants to proceed. False if the query is destructive and the user doesn't want to proceed. """ - destructive = set(['drop', 'shutdown', 'delete', 'truncate']) - queries = queries.strip() - for query in sqlparse.split(queries): - try: - first_token = query.split()[0] - if first_token.lower() in destructive: - destroy = click.prompt("You're about to run a destructive command.\nDo you want to proceed? (y/n)", - type=bool) - return destroy - except Exception: - return False + prompt_text = ("You're about to run a destructive command.\n" + "Do you want to proceed? (y/n)") + if is_destructive(queries): + return click.prompt(prompt_text, type=bool) def quit_command(sql): return (sql.strip().lower() == 'exit' diff --git a/tests/test_main.py b/tests/test_main.py index 898c5dde9..b1c71b844 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,7 @@ from click.testing import CliRunner -from mycli.main import cli, format_output +from mycli.main import (cli, format_output, is_destructive, query_starts_with, + queries_start_with) from utils import USER, HOST, PORT, PASSWORD, dbtest, run CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, @@ -67,3 +68,32 @@ def test_batch_mode_table(executor): assert result.exit_code == 0 assert expected in result.output + +def test_query_starts_with(executor): + query = 'USE test;' + assert query_starts_with(query, ('use', )) is True + + query = 'DROP DATABASE test;' + assert query_starts_with(query, ('use', )) is False + +def test_query_starts_with_comment(executor): + query = '# comment\nUSE test;' + assert query_starts_with(query, ('use', )) is True + +def test_queries_start_with(executor): + sql = ( + '# comment\n' + 'show databases;' + 'use foo;' + ) + assert queries_start_with(sql, ('show', 'select')) is True + assert queries_start_with(sql, ('use', 'drop')) is True + assert queries_start_with(sql, ('delete', 'update')) is False + +def test_is_destructive(executor): + sql = ( + 'use test;\n' + 'show databases;\n' + 'drop database foo;' + ) + assert is_destructive(sql) is True From 4f8f1fd35da2564fb489c29ae4c1697541ddd25f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 10 Apr 2016 15:12:55 -0700 Subject: [PATCH 0193/1025] Make confirm_destructive_query detect TTY. --- mycli/main.py | 4 ++-- tests/test_main.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index f70ce73ea..5c45aa891 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -861,13 +861,13 @@ def is_destructive(queries): def confirm_destructive_query(queries): """Check if the query is destructive and prompts the user to confirm. Returns: - None if the query is non-destructive. + None if the query is non-destructive or we can't prompt the user. True if the query is destructive and the user wants to proceed. False if the query is destructive and the user doesn't want to proceed. """ prompt_text = ("You're about to run a destructive command.\n" "Do you want to proceed? (y/n)") - if is_destructive(queries): + if is_destructive(queries) and sys.stdin.isatty(): return click.prompt(prompt_text, type=bool) def quit_command(sql): diff --git a/tests/test_main.py b/tests/test_main.py index b1c71b844..22ef48d0c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,7 +1,8 @@ +import click from click.testing import CliRunner -from mycli.main import (cli, format_output, is_destructive, query_starts_with, - queries_start_with) +from mycli.main import (cli, confirm_destructive_query, format_output, + is_destructive, query_starts_with, queries_start_with) from utils import USER, HOST, PORT, PASSWORD, dbtest, run CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, @@ -97,3 +98,10 @@ def test_is_destructive(executor): 'drop database foo;' ) assert is_destructive(sql) is True + +def test_confirm_destructive_query_notty(executor): + stdin = click.get_text_stream('stdin') + assert stdin.isatty() is False + + sql = 'drop database foo;' + assert confirm_destructive_query(sql) is None From 29cedba12889c7e68672feeb107ef573aeaabc1d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 10 Apr 2016 15:19:54 -0700 Subject: [PATCH 0194/1025] Prompt user in batch mode for destructive queries. --- mycli/main.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 5c45aa891..677cf830c 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -754,18 +754,27 @@ def cli(database, user, host, port, socket, password, dbname, '\thost: %r' '\tport: %r', database, user, host, port) + if sys.stdin.isatty(): + mycli.run_cli() + stdin = click.get_text_stream('stdin') - if not stdin.isatty(): - results = mycli.sqlexecute.run(stdin.read()) - for result in results: - title, cur, headers, status = result - table_format = mycli.table_format if table else None - output = format_output(title, cur, headers, None, table_format) - for line in output: - click.echo(line) - exit(0) + stdin_text = stdin.read() - mycli.run_cli() + try: + sys.stdin = open('/dev/tty') + except FileNotFoundError: + mycli.logger.warning('Unable to open TTY as stdin.') + + if confirm_destructive_query(stdin_text) is False: + exit(0) + results = mycli.sqlexecute.run(stdin_text) + for result in results: + title, cur, headers, status = result + table_format = mycli.table_format if table else None + output = format_output(title, cur, headers, None, table_format) + for line in output: + click.echo(line) + exit(0) def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): output = [] From 46711d3210c33449b259babd70ce0fbf21eb0ab5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 10 Apr 2016 15:21:52 -0700 Subject: [PATCH 0195/1025] Fix Python 2 test. --- mycli/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/main.py b/mycli/main.py index 677cf830c..357872f23 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -49,6 +49,7 @@ try: from urlparse import urlparse + FileNotFoundError = OSError except ImportError: from urllib.parse import urlparse from pymysql import OperationalError From 08178887b27cbc0c4e12efd0f6821a021181f165 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 11 Apr 2016 07:26:47 -0500 Subject: [PATCH 0196/1025] Fix hang on interactive mode exit. --- mycli/main.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 357872f23..485dfaaaf 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -202,7 +202,6 @@ def execute_from_file(self, arg, **_): query = f.read() except IOError as e: return [(None, None, None, str(e))] - return self.sqlexecute.run(query) def change_prompt_format(self, arg, **_): @@ -757,25 +756,24 @@ def cli(database, user, host, port, socket, password, dbname, if sys.stdin.isatty(): mycli.run_cli() + else: + stdin = click.get_text_stream('stdin') + stdin_text = stdin.read() - stdin = click.get_text_stream('stdin') - stdin_text = stdin.read() - - try: - sys.stdin = open('/dev/tty') - except FileNotFoundError: - mycli.logger.warning('Unable to open TTY as stdin.') - - if confirm_destructive_query(stdin_text) is False: - exit(0) - results = mycli.sqlexecute.run(stdin_text) - for result in results: - title, cur, headers, status = result - table_format = mycli.table_format if table else None - output = format_output(title, cur, headers, None, table_format) - for line in output: - click.echo(line) - exit(0) + try: + sys.stdin = open('/dev/tty') + except FileNotFoundError: + mycli.logger.warning('Unable to open TTY as stdin.') + + if confirm_destructive_query(stdin_text) is False: + exit(0) + results = mycli.sqlexecute.run(stdin_text) + for result in results: + title, cur, headers, status = result + table_format = mycli.table_format if table else None + output = format_output(title, cur, headers, None, table_format) + for line in output: + click.echo(line) def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): output = [] From 2f51dc6c5027973eff77fb16bbcf7340a6292c78 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 11 Apr 2016 07:29:56 -0500 Subject: [PATCH 0197/1025] Prompt user on destructive queries for source command. --- mycli/main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 485dfaaaf..85ed4d2b1 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -202,6 +202,12 @@ def execute_from_file(self, arg, **_): query = f.read() except IOError as e: return [(None, None, None, str(e))] + + if (self.destructive_warning and + confirm_destructive_query(query) is False): + message = 'Wise choice. Command execution stopped.' + return [(None, None, None, message)] + return self.sqlexecute.run(query) def change_prompt_format(self, arg, **_): From 017e512332d5af4b7b079609969e1b0286b5d48a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 11 Apr 2016 07:35:53 -0500 Subject: [PATCH 0198/1025] Make destructive warning in batch mode respect config. --- mycli/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 85ed4d2b1..253492c5e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -204,7 +204,7 @@ def execute_from_file(self, arg, **_): return [(None, None, None, str(e))] if (self.destructive_warning and - confirm_destructive_query(query) is False): + confirm_destructive_query(query) is False): message = 'Wise choice. Command execution stopped.' return [(None, None, None, message)] @@ -771,7 +771,8 @@ def cli(database, user, host, port, socket, password, dbname, except FileNotFoundError: mycli.logger.warning('Unable to open TTY as stdin.') - if confirm_destructive_query(stdin_text) is False: + if (mycli.destructive_warning and + confirm_destructive_query(stdin_text) is False): exit(0) results = mycli.sqlexecute.run(stdin_text) for result in results: From 085f22800532efb38af8acafc31b0409e7946ca4 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 11 Apr 2016 07:55:29 -0500 Subject: [PATCH 0199/1025] Add warn/no-warn command-line options. --- mycli/main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 253492c5e..4912d6f1d 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -84,7 +84,7 @@ class MyCli(object): def __init__(self, sqlexecute=None, prompt=None, logfile=None, defaults_suffix=None, defaults_file=None, - login_path=None, auto_vertical_output=False): + login_path=None, auto_vertical_output=False, warn=None): self.sqlexecute = sqlexecute self.logfile = logfile self.defaults_suffix = defaults_suffix @@ -103,13 +103,14 @@ def __init__(self, sqlexecute=None, prompt=None, [self.user_config_file]) c = self.config = read_config_files(config_files) self.multi_line = c['main'].as_bool('multi_line') - self.destructive_warning = c['main'].as_bool('destructive_warning') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) self.table_format = c['main']['table_format'] self.syntax_style = c['main']['syntax_style'] self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') + c_dest_warning = c['main'].as_bool('destructive_warning') + self.destructive_warning = c_dest_warning if warn is None else warn # Write user config if system config wasn't the last config loaded. if c.filename not in self.system_config_files: @@ -715,6 +716,8 @@ def get_prompt(self, string): help='Automatically switch to vertical output mode if the result is wider than the terminal width.') @click.option('-t', '--table', is_flag=True, help='Display batch output in table format.') +@click.option('--warn/--no-warn', default=None, + help='Warn before running a destructive query.') @click.option('--local-infile', type=bool, help='Enable/disable LOAD DATA LOCAL INFILE.') @click.option('--login-path', type=str, @@ -723,7 +726,7 @@ def get_prompt(self, string): def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, login_path, auto_vertical_output, local_infile, ssl_ca, ssl_capath, - ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table): + ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, warn): if version: print('Version:', __version__) @@ -732,7 +735,7 @@ def cli(database, user, host, port, socket, password, dbname, mycli = MyCli(prompt=prompt, logfile=logfile, defaults_suffix=defaults_group_suffix, defaults_file=defaults_file, login_path=login_path, - auto_vertical_output=auto_vertical_output) + auto_vertical_output=auto_vertical_output, warn=warn) # Choose which ever one has a valid value. database = database or dbname From 42ed9c4567c0d47e03db97453e50b4c385c8cd60 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 11 Apr 2016 22:07:41 -0500 Subject: [PATCH 0200/1025] Print errors in batch mode. --- mycli/main.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 4912d6f1d..1ea642f67 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -777,13 +777,17 @@ def cli(database, user, host, port, socket, password, dbname, if (mycli.destructive_warning and confirm_destructive_query(stdin_text) is False): exit(0) - results = mycli.sqlexecute.run(stdin_text) - for result in results: - title, cur, headers, status = result - table_format = mycli.table_format if table else None - output = format_output(title, cur, headers, None, table_format) - for line in output: - click.echo(line) + try: + results = mycli.sqlexecute.run(stdin_text) + for result in results: + title, cur, headers, status = result + table_format = mycli.table_format if table else None + output = format_output(title, cur, headers, None, table_format) + for line in output: + click.echo(line) + except Exception as e: + click.secho(str(e), err=True, fg='red') + exit(1) def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): output = [] From 71711446c768bf33e75be970955239f899c7c861 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 11 Apr 2016 22:28:43 -0500 Subject: [PATCH 0201/1025] Update options shown for mycli help. --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index af6c67498..8bbf5d848 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,14 @@ Check the [detailed install instructions](#detailed-install-instructions) for de -S, --socket TEXT The socket file to use for connection. -p, --password Force password prompt. --pass TEXT Password to connect to the database + --ssl-ca PATH CA file in PEM format + --ssl-capath TEXT CA directory + --ssl-cert PATH X509 cert in PEM format + --ssl-key PATH X509 key in PEM format + --ssl-cipher TEXT SSL cipher to use + --ssl-verify-server-cert Verify server's "Common Name" in its cert + against hostname used when connecting. This + option is disabled by default -v, --version Version of mycli. -D, --database TEXT Database to use. -R, --prompt TEXT Prompt format (Default: "\t \u@\h:\d> ") @@ -56,6 +64,9 @@ Check the [detailed install instructions](#detailed-install-instructions) for de --auto-vertical-output Automatically switch to vertical output mode if the result is wider than the terminal width. + -t, --table Display batch output in table format. + --warn / --no-warn Warn before running a destructive query. + --local-infile BOOLEAN Enable/disable LOAD DATA LOCAL INFILE. --login-path TEXT Read this path from the login file. --help Show this message and exit. From 279680a28377f277b70a15442a6652d9ff29b7b1 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 23 Apr 2016 14:25:23 -0700 Subject: [PATCH 0202/1025] Enable history search via . --- mycli/key_bindings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index 94de541a9..3dc13b853 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -17,6 +17,7 @@ def mycli_bindings(get_key_bindings, set_key_bindings): key_binding_manager = KeyBindingManager( enable_open_in_editor=True, enable_system_bindings=True, + enable_search=True, enable_abort_and_exit_bindings=True, enable_vi_mode=Condition(lambda cli: get_key_bindings() == 'vi')) From c4b2d16ea96aef60709e0fe6c04ef6355d2e0d78 Mon Sep 17 00:00:00 2001 From: Jonathan Slenders Date: Fri, 6 May 2016 11:26:15 +0200 Subject: [PATCH 0203/1025] Upgrade to prompt-toolkit 1.0.0 --- mycli/clistyle.py | 17 +++++++---------- mycli/clitoolbar.py | 8 +++----- mycli/key_bindings.py | 15 ++++++--------- mycli/main.py | 22 +++++++++++----------- mycli/myclirc | 1 + setup.py | 2 +- 6 files changed, 29 insertions(+), 36 deletions(-) diff --git a/mycli/clistyle.py b/mycli/clistyle.py index a74df4506..a2a7b2a70 100644 --- a/mycli/clistyle.py +++ b/mycli/clistyle.py @@ -1,7 +1,6 @@ from pygments.token import string_to_tokentype -from pygments.style import Style from pygments.util import ClassNotFound -from prompt_toolkit.styles import default_style_extensions, PygmentsStyle +from prompt_toolkit.styles import default_style_extensions, style_from_dict import pygments.styles @@ -11,12 +10,10 @@ def style_factory(name, cli_style): except ClassNotFound: style = pygments.styles.get_style_by_name('native') - class CLIStyle(Style): - styles = {} + styles = {} + styles.update(style.styles) + styles.update(default_style_extensions) + custom_styles = dict([(string_to_tokentype(x), y) for x, y in cli_style.items()]) + styles.update(custom_styles) - styles.update(style.styles) - styles.update(default_style_extensions) - custom_styles = dict([(string_to_tokentype(x), y) for x, y in cli_style.items()]) - styles.update(custom_styles) - - return PygmentsStyle(CLIStyle) + return style_from_dict(styles) diff --git a/mycli/clitoolbar.py b/mycli/clitoolbar.py index 8abe948ab..b62d8edbe 100644 --- a/mycli/clitoolbar.py +++ b/mycli/clitoolbar.py @@ -1,12 +1,10 @@ from pygments.token import Token -from prompt_toolkit.enums import DEFAULT_BUFFER +from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode -def create_toolbar_tokens_func(get_key_bindings, get_is_refreshing): +def create_toolbar_tokens_func(get_is_refreshing): """ Return a function that generates the toolbar tokens. """ - assert callable(get_key_bindings) - token = Token.Toolbar def get_toolbar_tokens(cli): @@ -27,7 +25,7 @@ def get_toolbar_tokens(cli): result.append((token, ' (Semi-colon [;] will end the line)')) - if get_key_bindings() == 'vi': + if cli.editing_mode == EditingMode.VI: result.append((token.On, '[F4] Vi-mode')) else: result.append((token.On, '[F4] Emacs-mode')) diff --git a/mycli/key_bindings.py b/mycli/key_bindings.py index 3dc13b853..1651347e4 100644 --- a/mycli/key_bindings.py +++ b/mycli/key_bindings.py @@ -1,4 +1,5 @@ import logging +from prompt_toolkit.enums import EditingMode from prompt_toolkit.keys import Keys from prompt_toolkit.key_binding.manager import KeyBindingManager from prompt_toolkit.filters import Condition @@ -7,19 +8,15 @@ _logger = logging.getLogger(__name__) -def mycli_bindings(get_key_bindings, set_key_bindings): +def mycli_bindings(): """ Custom key bindings for mycli. """ - assert callable(get_key_bindings) - assert callable(set_key_bindings) - key_binding_manager = KeyBindingManager( enable_open_in_editor=True, enable_system_bindings=True, enable_search=True, - enable_abort_and_exit_bindings=True, - enable_vi_mode=Condition(lambda cli: get_key_bindings() == 'vi')) + enable_abort_and_exit_bindings=True) @key_binding_manager.registry.add_binding(Keys.F2) def _(event): @@ -45,10 +42,10 @@ def _(event): Toggle between Vi and Emacs mode. """ _logger.debug('Detected F4 key.') - if get_key_bindings() == 'vi': - set_key_bindings('emacs') + if event.cli.editing_mode == EditingMode.VI: + event.cli.editing_mode = EditingMode.EMACS else: - set_key_bindings('vi') + event.cli.editing_mode = EditingMode.VI @key_binding_manager.registry.add_binding(Keys.Tab) def _(event): diff --git a/mycli/main.py b/mycli/main.py index 1ea642f67..cc42d6e82 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -17,7 +17,7 @@ import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction from prompt_toolkit.interface import AcceptAction -from prompt_toolkit.enums import DEFAULT_BUFFER +from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode from prompt_toolkit.shortcuts import create_prompt_layout, create_eventloop from prompt_toolkit.document import Document from prompt_toolkit.filters import Always, HasFocus, IsDone @@ -420,17 +420,12 @@ def run_cli(self): self.refresh_completions() - def set_key_bindings(value): - if value not in ('emacs', 'vi'): - value = 'emacs' - self.key_bindings = value - project_root = os.path.dirname(PACKAGE_ROOT) author_file = os.path.join(project_root, 'AUTHORS') sponsor_file = os.path.join(project_root, 'SPONSORS') - key_binding_manager = mycli_bindings(get_key_bindings=lambda: self.key_bindings, - set_key_bindings=set_key_bindings) + key_binding_manager = mycli_bindings() + print('Version:', __version__) print('Chat: https://gitter.im/dbcli/mycli') print('Mail: https://groups.google.com/forum/#!forum/mycli-users') @@ -443,8 +438,7 @@ def prompt_tokens(cli): def get_continuation_tokens(cli, width): return [(Token.Continuation, ' ' * (width - 3) + '-> ')] - get_toolbar_tokens = create_toolbar_tokens_func(lambda: self.key_bindings, - self.completion_refresher.is_refreshing) + get_toolbar_tokens = create_toolbar_tokens_func(self.completion_refresher.is_refreshing) layout = create_prompt_layout(lexer=MyCliLexer, multiline=True, @@ -462,18 +456,24 @@ def get_continuation_tokens(cli, width): history=FileHistory(os.path.expanduser('~/.mycli-history')), complete_while_typing=Always(), accept_action=AcceptAction.RETURN_DOCUMENT) + if self.key_bindings == 'vi': + editing_mode = EditingMode.VI + else: + editing_mode = EditingMode.EMACS + application = Application(style=style_factory(self.syntax_style, self.cli_style), layout=layout, buffer=buf, key_bindings_registry=key_binding_manager.registry, on_exit=AbortAction.RAISE_EXCEPTION, on_abort=AbortAction.RETRY, + editing_mode=editing_mode, ignore_case=True) self.cli = CommandLineInterface(application=application, eventloop=create_eventloop()) try: while True: - document = self.cli.run() + document = self.cli.run(reset_current_buffer=True) special.set_expanded_output(False) diff --git a/mycli/myclirc b/mycli/myclirc index c59e621c7..0c162e9d3 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -75,6 +75,7 @@ Token.SearchMatch = '#ffffff bg:#4444aa' Token.SearchMatch.Current = '#ffffff bg:#44aa44' # The bottom toolbar. +Token.Toolbar = 'bg:#222222 #aaaaaa' Token.Toolbar.Off = 'bg:#222222 #888888' Token.Toolbar.On = 'bg:#222222 #ffffff' diff --git a/setup.py b/setup.py index b82957543..1876c6a62 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ install_requirements = [ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? - 'prompt_toolkit==0.60', + 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', 'sqlparse >= 0.1.19', 'configobj >= 5.0.6', From ca9ef2df459f5020f41ceaa6496af201a040a300 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 11 May 2016 23:07:27 -0700 Subject: [PATCH 0204/1025] Update changelog for release 1.7.0. --- changelog.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/changelog.md b/changelog.md index 759c3b4d8..48a148b49 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,20 @@ +1.7.0: +====== + +Features: +--------- + +* Add stdin batch mode. (Thanks: [Thomas Roten]). +* Add warn/no-warn command-line options. (Thanks: [Thomas Roten]). +* Upgrade sqlparse dependency to 0.1.19. (Thanks: [Amjith Ramanujam]). +* Update features list in README.md. (Thanks: [Matheus Rosa]). +* Remove extra \n in features list in README.md. (Thanks: [Matheus Rosa]). + +Bug Fixes: +---------- + +* Enable history search via . (Thanks: [Amjith Ramanujam]). + 1.6.0: ====== From 01bc2c820b088bd1a0f89ea8f6bc2751b5b2b58a Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 12 May 2016 09:32:19 -0700 Subject: [PATCH 0205/1025] Update changelog for release 1.7.0. --- changelog.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 48a148b49..0964af18a 100644 --- a/changelog.md +++ b/changelog.md @@ -15,6 +15,11 @@ Bug Fixes: * Enable history search via . (Thanks: [Amjith Ramanujam]). +Internal Changes: +----------------- + +* Upgrade prompt_toolkit to 1.0.0. (Thanks: [Jonathan Slenders]) + 1.6.0: ====== @@ -140,7 +145,7 @@ Features: * Add custom styles to color the menus and toolbars. -* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders](https://github.com/jonathanslenders)) +* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders]) Multi-line queries are automatically indented. @@ -310,3 +315,4 @@ Bug Fixes: [Phil Cohen]: https://github.com/phlipper [Terseus]: https://github.com/Terseus [William GARCIA]: https://github.com/willgarcia +[Jonathan Slenders]: https://github.com/jonathanslenders From 3f1e1bc27e8c91c6f9c78db0352624d978753276 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 12 May 2016 21:15:23 -0700 Subject: [PATCH 0206/1025] Releasing version 1.7.0 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index bcd8d54ef..0e1a38d3c 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.6.0' +__version__ = '1.7.0' From 08d3939a9eb5676c8ee546cf3dcd62c09a78b7ce Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 24 May 2016 11:11:07 -0700 Subject: [PATCH 0207/1025] Releasing version 1.7.1 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index 0e1a38d3c..48c2f6b0b 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.7.0' +__version__ = '1.7.1' From 6396a3f522c2ce3ac0752174dd3aa8e4e2b4a936 Mon Sep 17 00:00:00 2001 From: Casper Langemeijer Date: Fri, 27 May 2016 12:29:39 +0200 Subject: [PATCH 0208/1025] New debian release (1.7.0) --- debian/changelog | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/debian/changelog b/debian/changelog index 18da86c59..5135bf68a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,39 @@ +mycli (1.7.0) unstable; urgency=medium + + * Add stdin batch mode. (Thanks: Thomas Roten). + * Add warn/no-warn command-line options. (Thanks: Thomas Roten). + * Upgrade sqlparse dependency to 0.1.19. (Thanks: [Amjith Ramanujam]). + * Update features list in README.md. (Thanks: Matheus Rosa). + * Remove extra \n in features list in README.md. (Thanks: Matheus Rosa). + * Enable history search via . (Thanks: [Amjith Ramanujam]). + * Upgrade prompt_toolkit to 1.0.0. (Thanks: Jonathan Slenders) + + -- Casper Langemeijer Fri, 27 May 2016 12:03:31 +0200 + +mycli (1.6.0) unstable; urgency=medium + + * Change continuation prompt for multi-line mode to match default mysql. + * Add status command to match mysql's status command. (Thanks: Thomas Roten). + * Add SSL support for mycli. (Thanks: Artem Bezsmertnyi). + * Add auto-completion and highlight support for OFFSET keyword. (Thanks: Matheus Rosa). + * Add support for MYSQL_TEST_LOGIN_FILE env variable to specify alternate login file. (Thanks: Thomas Roten). + * Add support for --auto-vertical-output to automatically switch to vertical output if the output doesn't fit in the table format. + * Add support for system-wide config. Now /etc/myclirc will be honored. (Thanks: Thomas Roten). + * Add support for nopager and \n to turn off the pager. (Thanks: Thomas Roten). + * Add support for --local-infile command-line option. (Thanks: Thomas Roten). + * Remove -S from less option which was clobbering the scroll back in history. (Thanks: Thomas Roten). + * Make system command work with Python 3. (Thanks: Thomas Roten). + * Support \G terminator for \f queries. (Thanks: Terseus). + * Upgrade prompt_toolkit to 0.60. + * Add Python 3.5 to test environments. (Thanks: Thomas Roten). + * Remove license meta-data. (Thanks: Thomas Roten). + * Skip binary tests if PyMySQL version does not support it. (Thanks: Thomas Roten). + * Refactor pager handling. (Thanks: Thomas Roten) + * Capture warnings to log file. (Thanks: Mikhail Borisov). + * Make syntax_style a tiny bit more intuitive. (Thanks: Phil Cohen). + + -- Casper Langemeijer Fri, 27 May 2016 12:03:31 +0200 + mycli (1.5.2) unstable; urgency=low * Protect against port number being None when no port is specified in command line. From 1f27566497c33d54b8900c483867ee72ce0a6932 Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Sun, 29 May 2016 21:24:12 +0200 Subject: [PATCH 0209/1025] add skip_intro config --- mycli/main.py | 12 +++++++----- mycli/myclirc | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index cc42d6e82..8ef4ad691 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -107,6 +107,7 @@ def __init__(self, sqlexecute=None, prompt=None, special.set_timing_enabled(c['main'].as_bool('timing')) self.table_format = c['main']['table_format'] self.syntax_style = c['main']['syntax_style'] + self.skip_intro = c['main'].as_bool('skip_intro') self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') c_dest_warning = c['main'].as_bool('destructive_warning') @@ -426,11 +427,12 @@ def run_cli(self): key_binding_manager = mycli_bindings() - print('Version:', __version__) - print('Chat: https://gitter.im/dbcli/mycli') - print('Mail: https://groups.google.com/forum/#!forum/mycli-users') - print('Home: http://mycli.net') - print('Thanks to the contributor -', thanks_picker([author_file, sponsor_file])) + if not self.skip_intro: + print('Version:', __version__) + print('Chat: https://gitter.im/dbcli/mycli') + print('Mail: https://groups.google.com/forum/#!forum/mycli-users') + print('Home: http://mycli.net') + print('Thanks to the contributor -', thanks_picker([author_file, sponsor_file])) def prompt_tokens(cli): return [(Token.Prompt, self.get_prompt(self.prompt_format))] diff --git a/mycli/myclirc b/mycli/myclirc index 0c162e9d3..f0b29376a 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -58,6 +58,9 @@ wider_completion_menu = False # \n - Newline prompt = '\t \u@\h:\d> ' +# Skip intro info on startup. +skip_intro = False + # Custom colors for the completion menu, toolbar, etc. [colors] # Completion menus. From 1f4f4249b9b681b272e7464e4b48c24e64256526 Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Sun, 29 May 2016 21:39:28 +0200 Subject: [PATCH 0210/1025] update skip_intro to include outro, and rename to less_chatty --- mycli/main.py | 7 ++++--- mycli/myclirc | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 8ef4ad691..1b220fb80 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -107,7 +107,7 @@ def __init__(self, sqlexecute=None, prompt=None, special.set_timing_enabled(c['main'].as_bool('timing')) self.table_format = c['main']['table_format'] self.syntax_style = c['main']['syntax_style'] - self.skip_intro = c['main'].as_bool('skip_intro') + self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') c_dest_warning = c['main'].as_bool('destructive_warning') @@ -427,7 +427,7 @@ def run_cli(self): key_binding_manager = mycli_bindings() - if not self.skip_intro: + if not self.less_chatty: print('Version:', __version__) print('Chat: https://gitter.im/dbcli/mycli') print('Mail: https://groups.google.com/forum/#!forum/mycli-users') @@ -612,7 +612,8 @@ def get_continuation_tokens(cli, width): self.query_history.append(query) except EOFError: - self.output('Goodbye!') + if not self.less_chatty: + self.output('Goodbye!') def output(self, text, **kwargs): if self.logfile: diff --git a/mycli/myclirc b/mycli/myclirc index f0b29376a..f5f0dd979 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -58,8 +58,8 @@ wider_completion_menu = False # \n - Newline prompt = '\t \u@\h:\d> ' -# Skip intro info on startup. -skip_intro = False +# Skip intro info on startup and outro info on exit +less_chatty = False # Custom colors for the completion menu, toolbar, etc. [colors] From a415fbdd7b4391bd8fa53613a16a787aad164158 Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Sun, 29 May 2016 21:46:23 +0200 Subject: [PATCH 0211/1025] add log level NONE to use a no-op logging handler --- mycli/main.py | 13 ++++++++++++- mycli/myclirc | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 1b220fb80..e5e1458dc 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -61,6 +61,11 @@ PACKAGE_ROOT = os.path.dirname(__file__) +# no-op logging handler +class NullHandler(logging.Handler): + def emit(self, record): + pass + class MyCli(object): default_prompt = '\\t \\u@\\h:\\d> ' @@ -235,7 +240,13 @@ def initialize_logging(self): 'DEBUG': logging.DEBUG } - handler = logging.FileHandler(os.path.expanduser(log_file)) + # Disable logging if value is NONE by switching to a no-op handler + # Set log level to a high value so it doesn't even waste cycles getting called. + if log_level.upper() == "NONE": + handler = NullHandler() + log_level = "CRITICAL" + else: + handler = logging.FileHandler(os.path.expanduser(log_file)) formatter = logging.Formatter( '%(asctime)s (%(process)d/%(threadName)s) ' diff --git a/mycli/myclirc b/mycli/myclirc index f5f0dd979..7000b573e 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -20,7 +20,7 @@ destructive_warning = True log_file = ~/.mycli.log # Default log level. Possible values: "CRITICAL", "ERROR", "WARNING", "INFO" -# and "DEBUG". +# and "DEBUG". "NONE" disables logging. log_level = INFO # Log every query and its results to a file. Enable this by uncommenting the From 73a5732f96d99307328b6458e44b812e997aea7b Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Sun, 29 May 2016 22:31:26 +0200 Subject: [PATCH 0212/1025] mimic mysql and support MYCLI_HISTFILE environment variable --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index e5e1458dc..75a344b90 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -466,7 +466,7 @@ def get_continuation_tokens(cli, width): ]) with self._completer_lock: buf = CLIBuffer(always_multiline=self.multi_line, completer=self.completer, - history=FileHistory(os.path.expanduser('~/.mycli-history')), + history=FileHistory(os.path.expanduser(os.environ.get('MYCLI_HISTFILE', '~/.mycli-history'))), complete_while_typing=Always(), accept_action=AcceptAction.RETURN_DOCUMENT) if self.key_bindings == 'vi': From 012d173867b31dbf86d94ecd73120eb553e2a963 Mon Sep 17 00:00:00 2001 From: Scrappy Soft Date: Thu, 9 Jun 2016 15:05:43 +0200 Subject: [PATCH 0213/1025] add prompt_continuation support --- mycli/main.py | 4 +++- mycli/myclirc | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 75a344b90..26ca86795 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -138,6 +138,7 @@ def __init__(self, sqlexecute=None, prompt=None, prompt_cnf = self.read_my_cnf_files(self.cnf_files, ['prompt'])['prompt'] self.prompt_format = prompt or prompt_cnf or c['main']['prompt'] or \ self.default_prompt + self.prompt_continuation_format = c['main']['prompt_continuation'] self.query_history = [] @@ -449,7 +450,8 @@ def prompt_tokens(cli): return [(Token.Prompt, self.get_prompt(self.prompt_format))] def get_continuation_tokens(cli, width): - return [(Token.Continuation, ' ' * (width - 3) + '-> ')] + continuation_prompt = self.get_prompt(self.prompt_continuation_format) + return [(Token.Continuation, ' ' * (width - len(continuation_prompt)) + continuation_prompt)] get_toolbar_tokens = create_toolbar_tokens_func(self.completion_refresher.is_refreshing) diff --git a/mycli/myclirc b/mycli/myclirc index 7000b573e..ee3490770 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -57,6 +57,7 @@ wider_completion_menu = False # \d - Database name # \n - Newline prompt = '\t \u@\h:\d> ' +prompt_continuation = '-> ' # Skip intro info on startup and outro info on exit less_chatty = False From 3623eab6ca31df567870fa1374ae67356af37517 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Mon, 13 Jun 2016 16:01:42 -0700 Subject: [PATCH 0214/1025] Display login-path instead of host in prompt. Connect #271. --- mycli/main.py | 6 ++++-- mycli/myclirc | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 26ca86795..5a8541f6c 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -80,7 +80,7 @@ class MyCli(object): ] system_config_files = [ - '/etc/myclirc', + '/etc/myclirc', ] default_config_file = os.path.join(PACKAGE_ROOT, 'myclirc') @@ -117,6 +117,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') c_dest_warning = c['main'].as_bool('destructive_warning') self.destructive_warning = c_dest_warning if warn is None else warn + self.login_path_as_host = c['main'].as_bool('login_path_as_host') # Write user config if system config wasn't the last config loaded. if c.filename not in self.system_config_files: @@ -686,8 +687,9 @@ def get_completions(self, text, cursor_positition): def get_prompt(self, string): sqlexecute = self.sqlexecute + host = self.login_path if self.login_path and self.login_path_as_host else sqlexecute.host string = string.replace('\\u', sqlexecute.user or '(none)') - string = string.replace('\\h', sqlexecute.host or '(none)') + string = string.replace('\\h', host or '(none)') string = string.replace('\\d', sqlexecute.dbname or '(none)') string = string.replace('\\t', sqlexecute.server_type()[0] or 'mycli') string = string.replace('\\n', "\n") diff --git a/mycli/myclirc b/mycli/myclirc index ee3490770..3a6911c72 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -62,6 +62,9 @@ prompt_continuation = '-> ' # Skip intro info on startup and outro info on exit less_chatty = False +# Use alias from --login-path instead of host name in prompt +login_path_as_host = False + # Custom colors for the completion menu, toolbar, etc. [colors] # Completion menus. From 0fee40c7b8c8d425a0ab29d6e341049d4d4241e2 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Mon, 13 Jun 2016 16:58:25 -0700 Subject: [PATCH 0215/1025] Completion inside function can handle operands. --- mycli/packages/completion_engine.py | 4 +++- tests/test_completion_engine.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 9f6a2c321..91a22c90a 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -133,6 +133,8 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier else: token_v = token.value.lower() + is_operand = lambda x: x and any([x.endswith(op) for op in ['+', '-', '*', '/']]) + if not token: return [{'type': 'keyword'}, {'type': 'special'}] elif token_v.endswith('('): @@ -270,7 +272,7 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier return [{'type': 'database'}] elif token_v == 'tableformat': return [{'type': 'table_format'}] - elif token_v.endswith(',') or token_v in ['=', 'and', 'or']: + elif token_v.endswith(',') or is_operand(token_v) or token_v in ['=', 'and', 'or']: prev_keyword, text_before_cursor = find_prev_keyword(text_before_cursor) if prev_keyword: return suggest_based_on_last_token( diff --git a/tests/test_completion_engine.py b/tests/test_completion_engine.py index ee1c46e2d..294fdbd56 100644 --- a/tests/test_completion_engine.py +++ b/tests/test_completion_engine.py @@ -67,6 +67,16 @@ def test_lparen_suggests_cols(): assert suggestion == [ {'type': 'column', 'tables': [(None, 'tbl', None)]}] +def test_operand_inside_function_suggests_cols1(): + suggestion = suggest_type('SELECT MAX(col1 + FROM tbl', 'SELECT MAX(col1 + ') + assert suggestion == [ + {'type': 'column', 'tables': [(None, 'tbl', None)]}] + +def test_operand_inside_function_suggests_cols2(): + suggestion = suggest_type('SELECT MAX(col1 + col2 + FROM tbl', 'SELECT MAX(col1 + col2 + ') + assert suggestion == [ + {'type': 'column', 'tables': [(None, 'tbl', None)]}] + def test_select_suggests_cols_and_funcs(): suggestions = suggest_type('SELECT ', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ From 08a65279eb5eeb4b29c8542201dea533f18c5aaf Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Wed, 15 Jun 2016 01:07:46 -0300 Subject: [PATCH 0216/1025] Remove unsupported keywords This commit remove keywords that are not supported by MySQL 5.7, 5.5 and 5.6. --- mycli/sqlcompleter.py | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index ccc1dabc8..599dc9e52 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -19,32 +19,23 @@ class SQLCompleter(Completer): keywords = ['ACCESS', 'ADD', 'ALL', 'ALTER TABLE', 'AND', 'ANY', 'AS', - 'ASC', 'AUDIT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', - 'CASE', 'CHANGE MASTER TO', 'CHAR', 'CHECK', 'CLUSTER', - 'COLUMN', 'COMMENT', 'COMPRESS', 'COMMIT', 'CONNECT', 'COPY', - 'CREATE', 'CURRENT', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', - 'DELETE FROM', 'DELIMITER', 'DESC', 'DESCRIBE', 'DISTINCT', - 'DROP', 'ELSE', 'ENCODING', 'END', 'ESCAPE', 'EXCLUSIVE', - 'EXISTS', 'EXTENSION', 'FILE', 'FLOAT', 'FOR', 'FORMAT', - 'FORCE_QUOTE', 'FORCE_NOT_NULL', 'FREEZE', 'FROM', 'FULL', - 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HEADER', 'HOST', - 'IDENTIFIED', 'IMMEDIATE', 'IN', 'INCREMENT', 'INDEX', - 'INITIAL', 'INSERT INTO', 'INTEGER', 'INTERSECT', 'INTO', - 'INTERVAL', 'IS', 'JOIN', 'LEFT', 'LEVEL', 'LIKE', 'LIMIT', - 'LOCK', 'LOG', 'LOGS', 'LONG', 'MASTER', 'MINUS', 'MODE', - 'MODIFY', 'NOAUDIT', 'NOCOMPRESS', 'NOT', 'NOWAIT', 'NULL', - 'NUMBER', 'OIDS', 'OF', 'OFFLINE', 'OFFSET', 'ON', 'ONLINE', - 'OPTION', 'OR', 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', - 'PCTFREE', 'PORT', 'PRIMARY', 'PRIOR', 'PRIVILEGES', - 'PROCESSLIST', 'PURGE', 'QUOTE', 'RAW', 'RENAME', 'REPAIR', - 'RESOURCE', 'RESET', 'REVOKE', 'RIGHT', 'ROLLBACK', 'ROW', - 'ROWID', 'ROWNUM', 'ROWS', 'SELECT', 'SESSION', 'SET', 'SHARE', - 'SHOW', 'SIZE', 'SLAVE', 'SLAVES', 'SMALLINT', 'START', 'STOP', - 'SUCCESSFUL', 'SYNONYM', 'SYSDATE', 'TABLE', 'TEMPLATE', - 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UID', - 'UNION', 'UNIQUE', 'UPDATE', 'USE', 'USER', 'USING', - 'VALIDATE', 'VALUES', 'VARCHAR', 'VARCHAR2', 'VIEW', 'WHEN', - 'WHENEVER', 'WHERE', 'WITH'] + 'ASC', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', + 'CASE', 'CHAR', 'CHECK', 'COLUMN', 'COMMENT', 'COMMIT', + 'CHANGE MASTER TO', 'CREATE', 'CURRENT', 'DATABASE', 'DATE', + 'DECIMAL', 'DEFAULT', 'DELETE FROM', 'DELIMITER', 'DESC', + 'DESCRIBE', 'DISTINCT', 'DROP', 'ELSE', 'END', 'ESCAPE', 'EXISTS', + 'FILE', 'FLOAT', 'FOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', + 'GROUP BY', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', 'INCREMENT', 'INDEX', + 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'LEFT', + 'LEVEL', 'LIKE', 'LIMIT', 'LOCK', 'LOGS', 'LONG', 'MASTER', 'MODE', + 'MODIFY', 'NOT', 'NULL', 'NUMBER', 'OFFSET', 'ON', 'OPTION', 'OR', + 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', 'PORT', 'PRIMARY', + 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'RENAME', 'REPAIR', 'RESET', + 'REVOKE', 'RIGHT', 'ROLLBACK','ROW', 'ROWS', 'SELECT', 'SESSION', 'SET', + 'SHARE', 'SHOW', 'SLAVE', 'SMALLINT', 'START', 'STOP', 'TABLE', 'THEN', + 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UNION', 'UNIQUE', 'UPDATE', + 'USE', 'USER', 'USING', 'VALUES', 'VARCHAR', 'VIEW', 'WHEN', 'WHERE', + 'WITH'] functions = ['AVG', 'COUNT', 'DISTINCT', 'FIRST', 'FORMAT', 'LAST', 'LCASE', 'LEN', 'MAX', 'MIN', 'MID', 'NOW', 'ROUND', 'SUM', From e17c44f347dd1eec0f4c9eddea7da2320c4a0e2d Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 18 Jul 2016 00:29:10 -0300 Subject: [PATCH 0217/1025] Add support for --execute param --- mycli/main.py | 34 +++++++++++++++++++++++++--------- tests/test_main.py | 12 ++++++++++++ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 5a8541f6c..4330a195b 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -695,6 +695,16 @@ def get_prompt(self, string): string = string.replace('\\n', "\n") return string + def run_query(self, query, table_format=None): + """Runs query""" + results = self.sqlexecute.run(query) + for result in results: + title, cur, headers, status = result + table_format = self.table_format if table_format else None + output = format_output(title, cur, headers, None, table_format) + for line in output: + click.echo(line) + @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') @click.option('-P', '--port', envvar='MYSQL_TCP_PORT', type=int, help='Port number to use for connection. Honors ' @@ -740,11 +750,13 @@ def get_prompt(self, string): help='Enable/disable LOAD DATA LOCAL INFILE.') @click.option('--login-path', type=str, help='Read this path from the login file.') +@click.option('--execute', type=str, help='Execute query to the database.') @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, login_path, auto_vertical_output, local_infile, ssl_ca, ssl_capath, - ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, warn): + ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, warn, + execute): if version: print('Version:', __version__) @@ -781,6 +793,15 @@ def cli(database, user, host, port, socket, password, dbname, '\thost: %r' '\tport: %r', database, user, host, port) + # --execute argument + if execute: + try: + mycli.run_query(execute, table_format=table) + exit(0) + except Exception as e: + click.secho(str(e), err=True, fg='red') + exit(1) + if sys.stdin.isatty(): mycli.run_cli() else: @@ -796,17 +817,12 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - results = mycli.sqlexecute.run(stdin_text) - for result in results: - title, cur, headers, status = result - table_format = mycli.table_format if table else None - output = format_output(title, cur, headers, None, table_format) - for line in output: - click.echo(line) + mycli.run_query(stdin_text, table_format=table) except Exception as e: click.secho(str(e), err=True, fg='red') exit(1) + def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): output = [] if title: # Only print the title if it's not None. @@ -818,7 +834,7 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max elif table_format is not None: rows = list(cur) tabulated, frows = tabulate(rows, headers, tablefmt=table_format, - missingval='') + missingval='') if (max_width and rows and content_exceeds_width(frows[0], max_width) and headers): diff --git a/tests/test_main.py b/tests/test_main.py index 22ef48d0c..d2c351417 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -33,6 +33,18 @@ def test_format_output_no_table(): expected = ['Title', 'head1\thead2', 'abc\tdef', 'test status'] assert results == expected +@dbtest +def test_execute_arg(executor): + run(executor, 'create table test (a text)') + run(executor, 'insert into test values("abc")') + + sql = 'select * from test;' + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['--execute', sql]) + + assert result.exit_code == 0 + assert 'abc' in result.output + @dbtest def test_batch_mode(executor): run(executor, '''create table test(a text)''') From afbeec030ce5d73a42dfd01de59ba0da520607fc Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 18 Jul 2016 00:29:40 -0300 Subject: [PATCH 0218/1025] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 0cab00de1..12588dc82 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ .vagrant *.pyc *.deb +*.swp +.cache/ From 0e13dfcfb59e92adc132dabadb4740e8a18c97bc Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Tue, 19 Jul 2016 19:35:20 -0300 Subject: [PATCH 0219/1025] Add '-e' as an option for the '--execute' argument --- mycli/main.py | 3 ++- tests/test_main.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 4330a195b..82d817f66 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -750,7 +750,8 @@ def run_query(self, query, table_format=None): help='Enable/disable LOAD DATA LOCAL INFILE.') @click.option('--login-path', type=str, help='Read this path from the login file.') -@click.option('--execute', type=str, help='Execute query to the database.') +@click.option('-e', '--execute', type=str, + help='Execute query to the database.') @click.argument('database', default='', nargs=1) def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, diff --git a/tests/test_main.py b/tests/test_main.py index d2c351417..f174a45e1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -40,11 +40,17 @@ def test_execute_arg(executor): sql = 'select * from test;' runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['-e', sql]) + + assert result.exit_code == 0 + assert 'abc' in result.output + result = runner.invoke(cli, args=CLI_ARGS + ['--execute', sql]) assert result.exit_code == 0 assert 'abc' in result.output + @dbtest def test_batch_mode(executor): run(executor, '''create table test(a text)''') From 9688f652d8a4d730426e40e66527c94b7f16f763 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 31 Jul 2016 22:28:47 -0700 Subject: [PATCH 0220/1025] Pin sqlparse to 0.1.19 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1876c6a62..f21bdb4c9 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse >= 0.1.19', + 'sqlparse == 0.1.19', 'configobj >= 5.0.6', ] From 91055a2b1a90099e16459c0e258f7647a6a52dd7 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 1 Aug 2016 07:23:58 -0700 Subject: [PATCH 0221/1025] Changelog and authors update for 1.8.0 --- AUTHORS | 23 ++++++++++++----------- changelog.md | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index a6552890c..85a53e67b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -3,39 +3,40 @@ Many thanks to the following contributors. Core Developers: ---------------- - * Iryna Cherniavska * Thomas Roten - * Darik Gamble + * Iryna Cherniavska * Matheus Rosa + * Darik Gamble Contributors: ------------- * Steve Robbins - * Daniel West * Shoma Suzuki + * Daniel West + * Scrappy Soft * Daniel Black * Jonathan Bruno + * Casper Langemeijer + * Jonathan Slenders * Artem Bezsmertnyi - * Heath Naylor * Mikhail Borisov + * Heath Naylor * Phil Cohen - * Yasuhiro Matsumoto - * bjarnagin - * jbruno - * mrdeathless - * Abirami P * spacewander * Adam Chainz - * Casper Langemeijer * Johannes Hoff - * Jonathan Slenders * Kacper Kwapisz * Lennart Weller * Martijn Engler * Terseus * Tyler Kuipers * William GARCIA + * Yasuhiro Matsumoto + * bjarnagin + * jbruno + * mrdeathless + * Abirami P Creator: -------- diff --git a/changelog.md b/changelog.md index 0964af18a..91cf208d7 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,22 @@ +1.8.0: +====== + +Features: +--------- + +* Add support for --execute/-e commandline arg. (Thanks: [Matheus Rosa]). +* Add `less_chatty` config option to skip the intro messages. (Thanks: [Scrappy Soft]). +* Support MYCLI_HISTFILE environment variable to specify where to write the history file. (Thanks: [Scrappy Soft]). +* Add `prompt_continuation` config option to allow configuring the continuation prompt for multi-line queries. (Thanks: [Scrappy Soft]). +* Display login-path instead of host in prompt. (Thanks: [Irina Truong]). + +Bug Fixes: +---------- + +* Pin sqlparse to version 0.1.19 since the new version is breaking completion. (Thanks: [Amjith Ramanujam]). +* Remove unsupported keywords. (Thanks: [Matheus Rosa]). +* Fix completion suggestion inside functions with operands. (Thanks: [Irina Truong]). + 1.7.0: ====== @@ -316,3 +335,5 @@ Bug Fixes: [Terseus]: https://github.com/Terseus [William GARCIA]: https://github.com/willgarcia [Jonathan Slenders]: https://github.com/jonathanslenders +[Casper Langemeijer]: https://github.com/langemeijer +[Scrappy Soft]: https://github.com/scrappysoft From 9104a19d5b8e8a39697902bb5a5e3532aef0b1ac Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 3 Aug 2016 11:13:11 -0700 Subject: [PATCH 0222/1025] Releasing version 1.8.0 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index 48c2f6b0b..b28097579 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.7.1' +__version__ = '1.8.0' From 25316468b821ca14f175f6746c439d873f565129 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 9 Aug 2016 06:34:48 -0700 Subject: [PATCH 0223/1025] Reset the show items when completion is refreshed. --- mycli/sqlcompleter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 599dc9e52..e92332738 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -192,6 +192,7 @@ def set_dbname(self, dbname): def reset_completions(self): self.databases = [] + self.show_items = [] self.dbname = '' self.dbmetadata = {'tables': {}, 'views': {}, 'functions': {}} self.all_completions = set(self.keywords + self.functions) From eb6b8aa2430d11e08c6234c2c0841d3965f22904 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 19 Jul 2016 22:11:57 -0700 Subject: [PATCH 0224/1025] Custom converters for time/date types. Having invalid time/date values in database causes mycli to display their values as . Having custom converters will handle that case. --- mycli/sqlexecute.py | 11 ++++++++- tests/test_sqlexecute.py | 52 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index de977fe6f..38116496d 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -2,6 +2,9 @@ import pymysql import sqlparse from .packages import connection, special +from pymysql.constants import FIELD_TYPE +from pymysql.converters import (convert_mysql_timestamp, convert_datetime, + convert_timedelta, convert_date) _logger = logging.getLogger(__name__) @@ -61,13 +64,19 @@ def connect(self, database=None, user=None, password=None, host=None, '\tlocal_infile: %r' '\tssl: %r', database, user, host, port, socket, charset, local_infile, ssl) + conv = { + FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or '0000-00-00 00:00:00'), + FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or '0000-00-00 00:00:00'), + FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or '00:00:00'), + FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or '0000-00-00'), + } conn = connection.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, use_unicode=True, charset=charset, autocommit=True, client_flag=pymysql.constants.CLIENT.INTERACTIVE, cursorclass=connection.Cursor, local_infile=local_infile, - ssl=ssl) + conv=conv, ssl=ssl) if hasattr(self, 'conn'): self.conn.close() self.conn = conn diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index 4a6f99101..69b9a6d26 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -277,3 +277,55 @@ def test_favorite_query_multiline_statement(executor): results = run(executor, "\\fd test-ad") assert results == ['test-ad: Deleted'] + +@dbtest +def test_timestamp_null(executor): + run(executor, '''create table ts_null(a timestamp)''') + run(executor, '''insert into ts_null values(0)''') + results = run(executor, '''select * from ts_null''', join=True) + assert results == dedent("""\ + +---------------------+ + | a | + |---------------------| + | 0000-00-00 00:00:00 | + +---------------------+ + 1 row in set""") + +@dbtest +def test_datetime_null(executor): + run(executor, '''create table dt_null(a datetime)''') + run(executor, '''insert into dt_null values(0)''') + results = run(executor, '''select * from dt_null''', join=True) + assert results == dedent("""\ + +---------------------+ + | a | + |---------------------| + | 0000-00-00 00:00:00 | + +---------------------+ + 1 row in set""") + +@dbtest +def test_date_null(executor): + run(executor, '''create table date_null(a date)''') + run(executor, '''insert into date_null values(0)''') + results = run(executor, '''select * from date_null''', join=True) + assert results == dedent("""\ + +------------+ + | a | + |------------| + | 0000-00-00 | + +------------+ + 1 row in set""") + +@dbtest +def test_time_null(executor): + run(executor, '''create table time_null(a time)''') + run(executor, '''insert into time_null values(0)''') + results = run(executor, '''select * from time_null''', join=True) + assert results == dedent("""\ + +----------+ + | a | + |----------| + | 00:00:00 | + +----------+ + 1 row in set""") From e3c41835a84a6371bd25f574cf7a6e82fd6cbe1f Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 9 Aug 2016 06:38:17 -0700 Subject: [PATCH 0225/1025] Fallback to the raw object for invalid time values. --- mycli/sqlexecute.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 38116496d..d44a11fec 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -65,10 +65,10 @@ def connect(self, database=None, user=None, password=None, host=None, '\tssl: %r', database, user, host, port, socket, charset, local_infile, ssl) conv = { - FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or '0000-00-00 00:00:00'), - FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or '0000-00-00 00:00:00'), - FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or '00:00:00'), - FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or '0000-00-00'), + FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), + FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), + FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), + FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), } conn = connection.connect(database=db, user=user, password=password, From 6dc8fff09513a36e6d893974f5764959e339362b Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 26 Aug 2016 14:04:30 -0700 Subject: [PATCH 0226/1025] Update readme to change the install instructions. --- README.md | 39 +++++---------------------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index d7de272d6..04e315396 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,6 @@ A command line client for MySQL that can do auto-completion and syntax highlight HomePage: [http://mycli.net](http://mycli.net) -Debian Packages via [PackageCloud.io](https://packagecloud.io/amjith/mycli). - ![Completion](screenshots/tables.png) ![CompletionGif](screenshots/main.gif) @@ -32,7 +30,11 @@ or $ brew update && brew install mycli # Only on OS X ``` -Check the [detailed install instructions](#detailed-install-instructions) for debian packages or getting started with pip. +or + +``` +$ sudo apt-get install mycli # Only on debian or ubuntu +``` ### Usage @@ -114,37 +116,6 @@ Twitter: [@amjithr](http://twitter.com/amjithr) ## Detailed Install Instructions: -### Debian/Ubuntu Package: - -The debian package for `mycli` is hosted on [packagecloud.io](https://packagecloud.io/amjith/mycli). - -Add the gpg key for packagecloud for package verification. - -``` -curl https://packagecloud.io/gpg.key | apt-key add - -``` - -Install a package called apt-transport-https to make it possible for apt to fetch packages over https. - -``` -apt-get install -y apt-transport-https -``` - -Add the mycli package repo to the apt source. - -``` -echo "deb https://packagecloud.io/amjith/mycli/ubuntu/ trusty main" | sudo tee -a /etc/apt/sources.list -``` - -Update the apt sources and install mycli. - -``` -$ sudo apt-get update -$ sudo apt-get install mycli -``` - -Now `mycli` can be upgraded easily by using ``sudo apt-get upgrade mycli``. - ### RHEL, Centos, Fedora: I haven't built an RPM package for mycli yet. So please use `pip` to install `mycli`. You can install pip on your system using: From 895e109ca169e240e0f26e087b93c3b47df2232a Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 17 Sep 2016 10:10:26 +0200 Subject: [PATCH 0227/1025] Support python-sqlparse 0.2 --- mycli/packages/completion_engine.py | 9 +++++---- setup.py | 2 +- tests/test_sqlexecute.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 91a22c90a..774bb1ef1 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -2,6 +2,7 @@ import sys import sqlparse from sqlparse.sql import Comparison, Identifier, Where +from sqlparse.compat import text_type from .parseutils import last_word, extract_tables, find_prev_keyword from .special import parse_special_command @@ -56,7 +57,7 @@ def suggest_type(full_text, text_before_cursor): stmt_start, stmt_end = 0, 0 for statement in parsed: - stmt_len = len(statement.to_unicode()) + stmt_len = len(text_type(statement)) stmt_start, stmt_end = stmt_end, stmt_end + stmt_len if stmt_end >= current_pos: @@ -79,7 +80,7 @@ def suggest_type(full_text, text_before_cursor): if tok1 and tok1.value == '\\': return suggest_special(text_before_cursor) - last_token = statement and statement.token_prev(len(statement.tokens)) or '' + last_token = statement and statement.token_prev(len(statement.tokens))[1] or '' return suggest_based_on_last_token(last_token, text_before_cursor, full_text, identifier) @@ -157,7 +158,7 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier # Check for a subquery expression (cases 3 & 4) where = p.tokens[-1] - prev_tok = where.token_prev(len(where.tokens) - 1) + idx, prev_tok = where.token_prev(len(where.tokens) - 1) if isinstance(prev_tok, Comparison): # e.g. "SELECT foo FROM bar WHERE foo = ANY(" @@ -170,7 +171,7 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier return column_suggestions # Get the token before the parens - prev_tok = p.token_prev(len(p.tokens) - 1) + idx, prev_tok = p.token_prev(len(p.tokens) - 1) if prev_tok and prev_tok.value and prev_tok.value.lower() == 'using': # tbl1 INNER JOIN tbl2 USING (col1, col2) tables = extract_tables(full_text) diff --git a/setup.py b/setup.py index f21bdb4c9..31f2fde4c 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse == 0.1.19', + 'sqlparse >= 0.2.0', 'configobj >= 5.0.6', ] diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index 69b9a6d26..d9ed62532 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -90,10 +90,10 @@ def test_invalid_column_name(executor): @dbtest def test_unicode_support_in_output(executor): run(executor, "create table unicodechars(t text)") - run(executor, "insert into unicodechars (t) values ('é')") + run(executor, u"insert into unicodechars (t) values ('é')") # See issue #24, this raises an exception without proper handling - assert u'é' in run(executor, "select * from unicodechars", join=True) + assert u'é' in run(executor, u"select * from unicodechars", join=True) @dbtest def test_expanded_output(executor): @@ -247,7 +247,7 @@ def test_cd_command_current_dir(executor): @dbtest def test_unicode_support(executor): - assert u'日本語' in run(executor, "SELECT '日本語' AS japanese;", join=True) + assert u'日本語' in run(executor, u"SELECT '日本語' AS japanese;", join=True) @dbtest def test_favorite_query_multiline_statement(executor): From 9b30aba3318014456eab2400109d41b7ac348ff5 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 17 Sep 2016 21:21:24 -0700 Subject: [PATCH 0228/1025] Add an try/except for AS keyword crash. --- mycli/packages/completion_engine.py | 44 ++++++++++++++++------------- tests/test_completion_engine.py | 8 ++++++ 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 774bb1ef1..31ef8744c 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -28,26 +28,32 @@ def suggest_type(full_text, text_before_cursor): identifier = None - # If we've partially typed a word then word_before_cursor won't be an empty - # string. In that case we want to remove the partially typed string before - # sending it to the sqlparser. Otherwise the last token will always be the - # partially typed string which renders the smart completion useless because - # it will always return the list of keywords as completion. - if word_before_cursor: - if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': - parsed = sqlparse.parse(text_before_cursor) + # This is a temporary hack; the exception handling + # here should be removed once sqlparse has been fixed + try: + # If we've partially typed a word then word_before_cursor won't be an empty + # string. In that case we want to remove the partially typed string before + # sending it to the sqlparser. Otherwise the last token will always be the + # partially typed string which renders the smart completion useless because + # it will always return the list of keywords as completion. + if word_before_cursor: + if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': + parsed = sqlparse.parse(text_before_cursor) + else: + parsed = sqlparse.parse( + text_before_cursor[:-len(word_before_cursor)]) + + # word_before_cursor may include a schema qualification, like + # "schema_name.partial_name" or "schema_name.", so parse it + # separately + p = sqlparse.parse(word_before_cursor)[0] + + if p.tokens and isinstance(p.tokens[0], Identifier): + identifier = p.tokens[0] else: - parsed = sqlparse.parse( - text_before_cursor[:-len(word_before_cursor)]) - - # word_before_cursor may include a schema qualification, like - # "schema_name.partial_name" or "schema_name.", so parse it - # separately - p = sqlparse.parse(word_before_cursor)[0] - if p.tokens and isinstance(p.tokens[0], Identifier): - identifier = p.tokens[0] - else: - parsed = sqlparse.parse(text_before_cursor) + parsed = sqlparse.parse(text_before_cursor) + except (TypeError, AttributeError): + return [] if len(parsed) > 1: # Multiple statements being edited -- isolate the current one by diff --git a/tests/test_completion_engine.py b/tests/test_completion_engine.py index 294fdbd56..9b8771083 100644 --- a/tests/test_completion_engine.py +++ b/tests/test_completion_engine.py @@ -442,3 +442,11 @@ def test_cross_join(): {'type': 'table', 'schema': []}, {'type': 'view', 'schema': []}, {'type': 'schema'}]) + +@pytest.mark.parametrize('expression', [ + 'SELECT 1 AS ', + 'SELECT 1 FROM tabl AS ', +]) +def test_after_as(expression): + suggestions = suggest_type(expression, expression) + assert set(suggestions) == set() From d1e8c849d374a130b86296716375dcb69afc757b Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 17 Sep 2016 21:21:49 -0700 Subject: [PATCH 0229/1025] Pin the version of sqlparse. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 31f2fde4c..82ff1dd6c 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse >= 0.2.0', + 'sqlparse == 0.2.0', 'configobj >= 5.0.6', ] From 6836753244a9bc34af4cad3e3da222fe454c6e0f Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 18 Sep 2016 07:35:17 -0700 Subject: [PATCH 0230/1025] Make the dependency of sqlparse slightly more liberal. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 82ff1dd6c..3d80c96ca 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse == 0.2.0', + 'sqlparse>=0.2.0,<0.2.2', 'configobj >= 5.0.6', ] From 9c3ce514e5305251e049b93a4ce8ae93629ad54e Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sun, 18 Sep 2016 22:29:38 -0700 Subject: [PATCH 0231/1025] Remove duplicate listing of DISTINCT keyword. --- mycli/sqlcompleter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index e92332738..c12ada771 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -23,7 +23,7 @@ class SQLCompleter(Completer): 'CASE', 'CHAR', 'CHECK', 'COLUMN', 'COMMENT', 'COMMIT', 'CHANGE MASTER TO', 'CREATE', 'CURRENT', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', 'DELETE FROM', 'DELIMITER', 'DESC', - 'DESCRIBE', 'DISTINCT', 'DROP', 'ELSE', 'END', 'ESCAPE', 'EXISTS', + 'DESCRIBE', 'DROP', 'ELSE', 'END', 'ESCAPE', 'EXISTS', 'FILE', 'FLOAT', 'FOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', 'INCREMENT', 'INDEX', 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'LEFT', From 220617bff1da5103e0f9e15528aabc3787848e56 Mon Sep 17 00:00:00 2001 From: Adam Chainz Date: Fri, 14 Oct 2016 11:50:07 +0100 Subject: [PATCH 0232/1025] Release as a universal wheel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By releasing as a [Python wheel](http://pythonwheels.com/) as well as a source distribution, you can speed up end user’s installs. After merging this command, to release you just need to run `python setup.py clean sdist bdist_wheel upload`. --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 From cf2b606856fd5aefe5d1607e2733187802ca8ae1 Mon Sep 17 00:00:00 2001 From: Joseph Caillet Date: Thu, 20 Oct 2016 10:41:50 +0200 Subject: [PATCH 0233/1025] Update README.md Documented workaround for the issues described here : https://github.com/dbcli/mycli/issues/281 --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 04e315396..016c5396f 100644 --- a/README.md +++ b/README.md @@ -153,3 +153,22 @@ Thanks to [PyMysql](http://www.pymysql.org/) for a pure python adapter to MySQL Tests have been run on OS X and Linux. THIS HAS NOT BEEN TESTED IN WINDOWS, but the libraries used in this app are Windows compatible. This means it should work without any modifications. If you're unable to run it on Windows, please file a bug. I will try my best to fix it. + +### Use with pager (mysql workaround) +As described [here](https://github.com/dbcli/mycli/issues/281), " we only read the [client] section of my.cnf not the [mysql] section". + +So, if you want to use a pager, your .my.cnf file should looks like this: + +``` +[mysql] +pager = mypager +[client] +pager = mypager +``` + +instead of just this : + +``` +[mysql] +pager = mypager +``` From df411698623bd0936cccb0db034313235f5cdfa7 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Fri, 21 Oct 2016 20:51:21 -0700 Subject: [PATCH 0234/1025] Update changelog for release 1.8.1. --- AUTHORS | 1 + changelog.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/AUTHORS b/AUTHORS index 85a53e67b..316fa8979 100644 --- a/AUTHORS +++ b/AUTHORS @@ -15,6 +15,7 @@ Contributors: * Shoma Suzuki * Daniel West * Scrappy Soft + * Dick Marinus * Daniel Black * Jonathan Bruno * Casper Langemeijer diff --git a/changelog.md b/changelog.md index 91cf208d7..2bd345e87 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,18 @@ +1.8.1: +====== + +Bug Fixes: +---------- +* Remove duplicate listing of DISTINCT keyword. (Thanks: [Amjith Ramanujam]). +* Add an try/except for AS keyword crash. (Thanks: [Amjith Ramanujam]). +* Support python-sqlparse 0.2. (Thanks: [Dick Marinus]). +* Fallback to the raw object for invalid time values. (Thanks: [Amjith Ramanujam]). +* Reset the show items when completion is refreshed. (Thanks: [Amjith Ramanujam]). + +Internal Changes: +----------------- +* Make the dependency of sqlparse slightly more liberal. (Thanks: [Amjith Ramanujam]). + 1.8.0: ====== @@ -337,3 +352,4 @@ Bug Fixes: [Jonathan Slenders]: https://github.com/jonathanslenders [Casper Langemeijer]: https://github.com/langemeijer [Scrappy Soft]: https://github.com/scrappysoft +[Dick Marinus]: https://github.com/meeuw From eec63f0ab3f216f10dbdc1c7137e2e929702297d Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 24 Oct 2016 06:45:22 -0700 Subject: [PATCH 0235/1025] Releasing version 1.8.1 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index b28097579..e8b6b090b 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.8.0' +__version__ = '1.8.1' From 95de1846ddccaa98ce5887566073f09330decb27 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 24 Oct 2016 06:53:00 -0700 Subject: [PATCH 0236/1025] Update the release script. --- release.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/release.py b/release.py index 7ced9bd9f..622056274 100644 --- a/release.py +++ b/release.py @@ -70,6 +70,12 @@ def register_with_pypi(): def create_source_tarball(): run_step('python', 'setup.py', 'sdist') +def create_python_wheel(): + run_step('python', 'setup.py', 'sdist', 'bdist_wheel') + +def upload_source_tarball(): + run_step('python', 'setup.py', 'sdist', 'upload') + def push_to_github(): run_step('git', 'push', 'origin', 'master') @@ -121,5 +127,7 @@ def checklist(questions): create_git_tag('v%s' % ver) register_with_pypi() create_source_tarball() + create_python_wheel() push_to_github() push_tags_to_github() + upload_source_tarball() From b975116b8fa099c68065d89e3d60c16bb8ce5437 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 29 Oct 2016 18:23:48 +0100 Subject: [PATCH 0237/1025] Add config option for auto-vertical-output in myclirc --- mycli/myclirc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mycli/myclirc b/mycli/myclirc index 3a6911c72..66e1ee16f 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -65,6 +65,9 @@ less_chatty = False # Use alias from --login-path instead of host name in prompt login_path_as_host = False +# Enable auto-verical-output +auto_vertical_output = False + # Custom colors for the completion menu, toolbar, etc. [colors] # Completion menus. From db5412f2d9c0570676722aaa3849f2ffda5f746e Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 29 Oct 2016 18:24:32 +0100 Subject: [PATCH 0238/1025] Update main.py to read the auto-vertical-output config --- mycli/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 82d817f66..8ebdc09c8 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -94,7 +94,6 @@ def __init__(self, sqlexecute=None, prompt=None, self.logfile = logfile self.defaults_suffix = defaults_suffix self.login_path = login_path - self.auto_vertical_output = auto_vertical_output # self.cnf_files is a class variable that stores the list of mysql # config files to read in at launch. @@ -119,6 +118,10 @@ def __init__(self, sqlexecute=None, prompt=None, self.destructive_warning = c_dest_warning if warn is None else warn self.login_path_as_host = c['main'].as_bool('login_path_as_host') + # read from cli argument or user config file + self.auto_vertical_output = auto_vertical_output or \ + c['main'].as_bool('auto_vertical_output') + # Write user config if system config wasn't the last config loaded. if c.filename not in self.system_config_files: write_default_config(self.default_config_file, self.user_config_file) From 6c6dfdd4eeeb7192abb607b85963612f92178c87 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 01:12:39 +0100 Subject: [PATCH 0239/1025] Add CSV support for batch output --- mycli/main.py | 53 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 82d817f66..418668ccd 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -5,6 +5,7 @@ import os import os.path import sys +import csv import traceback import logging import threading @@ -13,6 +14,12 @@ from random import choice from io import open +# support StringIO for Python 2 and 3 +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + import click import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction @@ -744,6 +751,8 @@ def run_query(self, query, table_format=None): help='Automatically switch to vertical output mode if the result is wider than the terminal width.') @click.option('-t', '--table', is_flag=True, help='Display batch output in table format.') +@click.option('--csv', is_flag=True, + help='Display batch output in CSV format.') @click.option('--warn/--no-warn', default=None, help='Warn before running a destructive query.') @click.option('--local-infile', type=bool, @@ -756,8 +765,8 @@ def run_query(self, query, table_format=None): def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, login_path, auto_vertical_output, local_infile, ssl_ca, ssl_capath, - ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, warn, - execute): + ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, csv, + warn, execute): if version: print('Version:', __version__) @@ -818,7 +827,21 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - mycli.run_query(stdin_text, table_format=table) + results = mycli.sqlexecute.run(stdin_text) + table_format = None + + if csv: + table_format = 'csv' + new_line = False + elif table: + new_line = True + + for result in results: + title, cur, headers, status = result + output = format_output(title, cur, headers, status, table_format) + for line in output: + click.echo(line, nl=new_line) + except Exception as e: click.secho(str(e), err=True, fg='red') exit(1) @@ -832,7 +855,24 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max headers = [utf8tounicode(x) for x in headers] if expanded: output.append(expanded_table(cur, headers)) - elif table_format is not None: + if table_format is None: + output.append('\t'.join(headers)) + for row in cur: + output.append('\t'.join([str(r) for r in row])) + elif table_format == 'csv': + content = StringIO() + writer = csv.writer(content) + writer.writerow(headers) + output.append(content.getvalue()) + content.truncate(0) + for row in cur: + row = ['null' if val is None else val.encode('utf-8') for val in row] + writer.writerow(row) + output.append(content.getvalue()) + content.truncate(0) + output.append('\n') + content.close() + else: rows = list(cur) tabulated, frows = tabulate(rows, headers, tablefmt=table_format, missingval='') @@ -842,12 +882,9 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max output.append(expanded_table(rows, headers)) else: output.append(tabulated) - else: - output.append('\t'.join(headers)) - for row in cur: - output.append('\t'.join([str(r) for r in row])) if status: # Only print the status if it's not None. output.append(status) + return output def content_exceeds_width(row, width): From ae9d3d894046083942a002152341a20675adeb76 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 02:44:54 +0000 Subject: [PATCH 0240/1025] Fix code for batch output --- mycli/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 418668ccd..7b582b374 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -829,16 +829,17 @@ def cli(database, user, host, port, socket, password, dbname, try: results = mycli.sqlexecute.run(stdin_text) table_format = None + new_line = True if csv: table_format = 'csv' new_line = False elif table: - new_line = True + table_format = mycli.table_format for result in results: title, cur, headers, status = result - output = format_output(title, cur, headers, status, table_format) + output = format_output(title, cur, headers, None, table_format) for line in output: click.echo(line, nl=new_line) @@ -855,7 +856,7 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max headers = [utf8tounicode(x) for x in headers] if expanded: output.append(expanded_table(cur, headers)) - if table_format is None: + elif table_format is None: output.append('\t'.join(headers)) for row in cur: output.append('\t'.join([str(r) for r in row])) From 44cc4ac55db0fed8b06bbc0c092e476eb99d00e6 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 02:45:10 +0000 Subject: [PATCH 0241/1025] Add test for CSV batch output --- tests/test_main.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index f174a45e1..4da21dc98 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -88,6 +88,21 @@ def test_batch_mode_table(executor): assert result.exit_code == 0 assert expected in result.output +@dbtest +def test_batch_mode_csv(executor): + run(executor, '''create table test(a text, b text)''') + run(executor, '''insert into test (a, b) values('abc', 'def'), ('ghi', 'jkl')''') + + sql = 'select * from test;' + + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['--csv'], input=sql) + + expected = 'a,b\nabc,def\nghi,jkl\n\n' + + assert result.exit_code == 0 + assert expected in result.output + def test_query_starts_with(executor): query = 'USE test;' assert query_starts_with(query, ('use', )) is True From b8bf879de64f51f68dc2d1f52de3759d7c46aa5a Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 03:19:02 +0000 Subject: [PATCH 0242/1025] Fix tests for Python 3.5.2 --- mycli/main.py | 3 +-- tests/test_main.py | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 7b582b374..f45facc20 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -867,11 +867,10 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max output.append(content.getvalue()) content.truncate(0) for row in cur: - row = ['null' if val is None else val.encode('utf-8') for val in row] + row = ['null' if val is None else str(val) for val in row] writer.writerow(row) output.append(content.getvalue()) content.truncate(0) - output.append('\n') content.close() else: rows = list(cur) diff --git a/tests/test_main.py b/tests/test_main.py index 4da21dc98..22deed83b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -98,10 +98,11 @@ def test_batch_mode_csv(executor): runner = CliRunner() result = runner.invoke(cli, args=CLI_ARGS + ['--csv'], input=sql) - expected = 'a,b\nabc,def\nghi,jkl\n\n' + expected = 'a,b\nabc,def\nghi,jkl\n' + result_output = result.output.replace('\x00', '') # python 3 assert result.exit_code == 0 - assert expected in result.output + assert expected in result_output def test_query_starts_with(executor): query = 'USE test;' From 4af283f0a0fba083aeed4313c0fb245a11a393d6 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 03:37:40 +0000 Subject: [PATCH 0243/1025] Update auto-vertical-output config description in myclirc --- mycli/myclirc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/myclirc b/mycli/myclirc index 66e1ee16f..baf505106 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -65,7 +65,8 @@ less_chatty = False # Use alias from --login-path instead of host name in prompt login_path_as_host = False -# Enable auto-verical-output +# Cause result sets to be displayed vertically if they are too wide for the current window, +# and using normal tabular format otherwise. (This applies to statements terminated by ; or \G.) auto_vertical_output = False # Custom colors for the completion menu, toolbar, etc. From a17da7e44a941190d77851fa29cf2afb3414ca0b Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sun, 30 Oct 2016 04:01:45 +0000 Subject: [PATCH 0244/1025] Add prompt support for \D, \p and \_ options --- mycli/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mycli/main.py b/mycli/main.py index 82d817f66..b99325bd3 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -6,6 +6,7 @@ import os.path import sys import traceback +import socket import logging import threading from time import time @@ -693,6 +694,9 @@ def get_prompt(self, string): string = string.replace('\\d', sqlexecute.dbname or '(none)') string = string.replace('\\t', sqlexecute.server_type()[0] or 'mycli') string = string.replace('\\n', "\n") + string = string.replace('\\D', datetime.now().strftime('%a %b %d %H:%M:%S %Y')) + string = string.replace('\\p', socket.gethostbyname(socket.gethostname())) + string = string.replace('\\_', ' ') return string def run_query(self, query, table_format=None): From d4cb1b3746e22b4c7f1b5b42de3935ccbf400243 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 12:38:08 +0000 Subject: [PATCH 0245/1025] Fix "\p" prompt representation --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index b99325bd3..0a9a5ad0f 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -695,7 +695,7 @@ def get_prompt(self, string): string = string.replace('\\t', sqlexecute.server_type()[0] or 'mycli') string = string.replace('\\n', "\n") string = string.replace('\\D', datetime.now().strftime('%a %b %d %H:%M:%S %Y')) - string = string.replace('\\p', socket.gethostbyname(socket.gethostname())) + string = string.replace('\\p', str(sqlexecute.port)) string = string.replace('\\_', ' ') return string From 24b9e35d5eb669b851b62c5f6b30b826b4c4653e Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 13:22:01 +0000 Subject: [PATCH 0246/1025] Update import StringIO to cStringIO for performance reasons --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index f45facc20..80cc8c5b3 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -16,7 +16,7 @@ # support StringIO for Python 2 and 3 try: - from StringIO import StringIO + from cStringIO import StringIO except ImportError: from io import StringIO From 365a94819c25ecad1a27a8bf56560f5e98897388 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 13:23:02 +0000 Subject: [PATCH 0247/1025] Remove unecessary `truncate()` calls when writing CSV --- mycli/main.py | 7 +++---- tests/test_main.py | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 80cc8c5b3..a6fdb1319 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -864,13 +864,12 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max content = StringIO() writer = csv.writer(content) writer.writerow(headers) - output.append(content.getvalue()) - content.truncate(0) + for row in cur: row = ['null' if val is None else str(val) for val in row] writer.writerow(row) - output.append(content.getvalue()) - content.truncate(0) + + output.append(content.getvalue()) content.close() else: rows = list(cur) diff --git a/tests/test_main.py b/tests/test_main.py index 22deed83b..c36e980ab 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -99,10 +99,9 @@ def test_batch_mode_csv(executor): result = runner.invoke(cli, args=CLI_ARGS + ['--csv'], input=sql) expected = 'a,b\nabc,def\nghi,jkl\n' - result_output = result.output.replace('\x00', '') # python 3 assert result.exit_code == 0 - assert expected in result_output + assert expected in result.output def test_query_starts_with(executor): query = 'USE test;' From 10bd6e1e1303c731ec75ba88662389b328822db4 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 19:05:28 +0000 Subject: [PATCH 0248/1025] Add new param "new_line" to MyCli.run_query() method --- mycli/main.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index a6fdb1319..561a003c7 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -702,15 +702,14 @@ def get_prompt(self, string): string = string.replace('\\n', "\n") return string - def run_query(self, query, table_format=None): + def run_query(self, query, table_format=None, new_line=True): """Runs query""" results = self.sqlexecute.run(query) for result in results: title, cur, headers, status = result - table_format = self.table_format if table_format else None output = format_output(title, cur, headers, None, table_format) for line in output: - click.echo(line) + click.echo(line, nl=new_line) @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') @@ -827,7 +826,6 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - results = mycli.sqlexecute.run(stdin_text) table_format = None new_line = True @@ -837,12 +835,8 @@ def cli(database, user, host, port, socket, password, dbname, elif table: table_format = mycli.table_format - for result in results: - title, cur, headers, status = result - output = format_output(title, cur, headers, None, table_format) - for line in output: - click.echo(line, nl=new_line) - + mycli.run_query(stdin_text, table_format=table_format, new_line=new_line) + exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') exit(1) From 124ca067713a4896a398e25e2367989fbc192cc7 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Mon, 31 Oct 2016 19:26:12 +0000 Subject: [PATCH 0249/1025] Add "tsv" as the default table_format --- mycli/main.py | 6 ++---- tests/test_main.py | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 561a003c7..1cd00a94b 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -848,12 +848,10 @@ def format_output(title, cur, headers, status, table_format, expanded=False, max output.append(title) if cur: headers = [utf8tounicode(x) for x in headers] + table_format = 'tsv' if table_format is None else table_format + if expanded: output.append(expanded_table(cur, headers)) - elif table_format is None: - output.append('\t'.join(headers)) - for row in cur: - output.append('\t'.join([str(r) for r in row])) elif table_format == 'csv': content = StringIO() writer = csv.writer(content) diff --git a/tests/test_main.py b/tests/test_main.py index c36e980ab..c7f939696 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -30,7 +30,8 @@ def test_format_output_auto_expand(): def test_format_output_no_table(): results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], 'test status', None) - expected = ['Title', 'head1\thead2', 'abc\tdef', 'test status'] + + expected = ['Title', u'head1 \thead2\nabc \tdef', 'test status'] assert results == expected @dbtest @@ -65,7 +66,7 @@ def test_batch_mode(executor): result = runner.invoke(cli, args=CLI_ARGS, input=sql) assert result.exit_code == 0 - assert 'count(*)\n3\na\nabc' in result.output + assert ' count(*)\n 3\na\nabc\n' in result.output @dbtest def test_batch_mode_table(executor): From 71f503afb699b2a8365f9e8fddcdddb1960a1e79 Mon Sep 17 00:00:00 2001 From: Darik Gamble Date: Fri, 4 Nov 2016 19:54:34 -0400 Subject: [PATCH 0250/1025] Bump sqlparse version --- mycli/packages/parseutils.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/packages/parseutils.py b/mycli/packages/parseutils.py index 3cbf4a05d..7f848ad85 100644 --- a/mycli/packages/parseutils.py +++ b/mycli/packages/parseutils.py @@ -64,7 +64,7 @@ def last_word(text, include='alphanum_underscore'): # This code is borrowed from sqlparse example script. # def is_subselect(parsed): - if not parsed.is_group(): + if not parsed.is_group: return False for item in parsed.tokens: if item.ttype is DML and item.value.upper() in ('SELECT', 'INSERT', diff --git a/setup.py b/setup.py index 3d80c96ca..c3693572e 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? 'prompt_toolkit>=1.0.0,<1.1.0', 'PyMySQL >= 0.6.2', - 'sqlparse>=0.2.0,<0.2.2', + 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.6', ] From 54a8206c5187347a6a6f6ae4db994e0d7864cbcc Mon Sep 17 00:00:00 2001 From: Darik Gamble Date: Sat, 5 Nov 2016 08:06:15 -0400 Subject: [PATCH 0251/1025] Dangling `as` doesn't get grouped into identifiers in this version of sqlparse, so we need to handle them explicitly --- mycli/packages/completion_engine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 31ef8744c..6e2165dca 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -197,6 +197,9 @@ def suggest_based_on_last_token(token, text_before_cursor, full_text, identifier return [{'type': 'column', 'tables': extract_tables(full_text)}] elif token_v in ('set', 'by', 'distinct'): return [{'type': 'column', 'tables': extract_tables(full_text)}] + elif token_v == 'as': + # Don't suggest anything for an alias + return [] elif token_v in ('show'): return [{'type': 'show'}] elif token_v in ('to',): From dbe05e840de5e2627a6007e7736f8b831d7dc131 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 5 Nov 2016 14:21:56 +0000 Subject: [PATCH 0252/1025] Update --execute arg code and add tests --- mycli/main.py | 7 ++++++- tests/test_main.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 1cd00a94b..65c1b1ff9 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -805,7 +805,12 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - mycli.run_query(execute, table_format=table) + table_format = None + if table: + table_format = mycli.table_format + elif csv: + table_format = 'csv' + mycli.run_query(execute, table_format=table_format) exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') diff --git a/tests/test_main.py b/tests/test_main.py index c7f939696..bed88945e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -51,6 +51,38 @@ def test_execute_arg(executor): assert result.exit_code == 0 assert 'abc' in result.output + expected = 'a\nabc\n' + + assert result.output == expected + + +@dbtest +def test_execute_arg_with_table(executor): + run(executor, 'create table test (a text)') + run(executor, 'insert into test values("abc")') + + sql = 'select * from test;' + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['-e', sql] + ['--table']) + expected = '+-----+\n| a |\n|-----|\n| abc |\n+-----+\n' + + assert result.exit_code == 0 + assert result.output == expected + + +@dbtest +def test_execute_arg_with_csv(executor): + run(executor, 'create table test (a text)') + run(executor, 'insert into test values("abc")') + + sql = 'select * from test;' + runner = CliRunner() + result = runner.invoke(cli, args=CLI_ARGS + ['-e', sql] + ['--csv']) + expected = 'a\nabc\n\n' + + assert result.exit_code == 0 + assert result.output == expected + @dbtest def test_batch_mode(executor): From 00fa9b2c9e1ad0b1c04e8ec53dad9a58c0cb0072 Mon Sep 17 00:00:00 2001 From: Matheus Rosa Date: Sat, 5 Nov 2016 16:11:28 +0000 Subject: [PATCH 0253/1025] Fix tests for --execute command --- tests/test_main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index bed88945e..974eca74f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -53,7 +53,7 @@ def test_execute_arg(executor): expected = 'a\nabc\n' - assert result.output == expected + assert expected in result.output @dbtest @@ -67,7 +67,7 @@ def test_execute_arg_with_table(executor): expected = '+-----+\n| a |\n|-----|\n| abc |\n+-----+\n' assert result.exit_code == 0 - assert result.output == expected + assert expected in result.output @dbtest @@ -81,7 +81,7 @@ def test_execute_arg_with_csv(executor): expected = 'a\nabc\n\n' assert result.exit_code == 0 - assert result.output == expected + assert expected in result.output @dbtest From 4efa6e7072035901b5422276f046e1fe4beea5c0 Mon Sep 17 00:00:00 2001 From: cxbig Date: Fri, 11 Nov 2016 10:03:28 +0100 Subject: [PATCH 0254/1025] Added 'REGEXP' keyword in completer list --- mycli/sqlcompleter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index c12ada771..e79806771 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -30,7 +30,7 @@ class SQLCompleter(Completer): 'LEVEL', 'LIKE', 'LIMIT', 'LOCK', 'LOGS', 'LONG', 'MASTER', 'MODE', 'MODIFY', 'NOT', 'NULL', 'NUMBER', 'OFFSET', 'ON', 'OPTION', 'OR', 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', 'PORT', 'PRIMARY', - 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'RENAME', 'REPAIR', 'RESET', + 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'REGEXP', 'RENAME', 'REPAIR', 'RESET', 'REVOKE', 'RIGHT', 'ROLLBACK','ROW', 'ROWS', 'SELECT', 'SESSION', 'SET', 'SHARE', 'SHOW', 'SLAVE', 'SMALLINT', 'START', 'STOP', 'TABLE', 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UNION', 'UNIQUE', 'UPDATE', From 05a13b7bb327c379d1c43d932ac6ed67426ba255 Mon Sep 17 00:00:00 2001 From: Jialong Liu Date: Tue, 20 Dec 2016 00:04:45 +0800 Subject: [PATCH 0255/1025] Fixes #202. --- mycli/main.py | 273 +++++++++++++++++++++++++------------------------- 1 file changed, 137 insertions(+), 136 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 5691ac01b..f585c84b5 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -465,6 +465,142 @@ def get_continuation_tokens(cli, width): continuation_prompt = self.get_prompt(self.prompt_continuation_format) return [(Token.Continuation, ' ' * (width - len(continuation_prompt)) + continuation_prompt)] + def one_iteration(document=None): + if document is None: + document = self.cli.run(reset_current_buffer=True) + + special.set_expanded_output(False) + + # The reason we check here instead of inside the sqlexecute is + # because we want to raise the Exit exception which will be + # caught by the try/except block that wraps the + # sqlexecute.run() statement. + if quit_command(document.text): + raise EOFError + + try: + document = self.handle_editor_command(self.cli, document) + except RuntimeError as e: + logger.error("sql: %r, error: %r", document.text, e) + logger.error("traceback: %r", traceback.format_exc()) + self.output(str(e), err=True, fg='red') + return + + if self.destructive_warning: + destroy = confirm_destructive_query(document.text) + if destroy is None: + pass # Query was not destructive. Nothing to do here. + elif destroy is True: + self.output('Your call!') + else: + self.output('Wise choice!') + return + + # Keep track of whether or not the query is mutating. In case + # of a multi-statement query, the overall query is considered + # mutating if any one of the component statements is mutating + mutating = False + + try: + logger.debug('sql: %r', document.text) + + if self.logfile: + self.logfile.write('\n# %s\n' % datetime.now()) + self.logfile.write(document.text) + self.logfile.write('\n') + + successful = False + start = time() + res = sqlexecute.run(document.text) + successful = True + output = [] + total = 0 + for title, cur, headers, status in res: + logger.debug("headers: %r", headers) + logger.debug("rows: %r", cur) + logger.debug("status: %r", status) + threshold = 1000 + if (is_select(status) and + cur and cur.rowcount > threshold): + self.output('The result set has more than %s rows.' + % threshold, fg='red') + if not click.confirm('Do you want to continue?'): + self.output("Aborted!", err=True, fg='red') + break + + if self.auto_vertical_output: + max_width = self.cli.output.get_size().columns + else: + max_width = None + + formatted = format_output(title, cur, headers, + status, self.table_format, + special.is_expanded_output(), max_width) + + output.extend(formatted) + end = time() + total += end - start + mutating = mutating or is_mutating(status) + except UnicodeDecodeError as e: + import pymysql + if pymysql.VERSION < (0, 6, 7): + message = ('You are running an older version of pymysql.\n' + 'Please upgrade to 0.6.7 or above to view binary data.\n' + 'Try \'pip install -U pymysql\'.') + self.output(message) + else: + raise e + except KeyboardInterrupt: + # Restart connection to the database + sqlexecute.connect() + logger.debug("cancelled query, sql: %r", document.text) + self.output("cancelled query", err=True, fg='red') + except NotImplementedError: + self.output('Not Yet Implemented.', fg="yellow") + except OperationalError as e: + logger.debug("Exception: %r", e) + if (e.args[0] in (2003, 2006, 2013)): + logger.debug('Attempting to reconnect.') + self.output('Reconnecting...', fg='yellow') + try: + sqlexecute.connect() + logger.debug('Reconnected successfully.') + one_iteration(document) + return # OK to just return, cuz the recursion call runs to the end. + except OperationalError as e: + logger.debug('Reconnect failed. e: %r', e) + self.output(str(e), err=True, fg='red') + return # If reconnection failed, don't proceed further. + else: + logger.error("sql: %r, error: %r", document.text, e) + logger.error("traceback: %r", traceback.format_exc()) + self.output(str(e), err=True, fg='red') + except Exception as e: + logger.error("sql: %r, error: %r", document.text, e) + logger.error("traceback: %r", traceback.format_exc()) + self.output(str(e), err=True, fg='red') + else: + try: + if special.is_pager_enabled(): + self.output_via_pager('\n'.join(output)) + else: + self.output('\n'.join(output)) + except KeyboardInterrupt: + pass + if special.is_timing_enabled(): + self.output('Time: %0.03fs' % total) + + # Refresh the table names and column names if necessary. + if need_completion_refresh(document.text): + self.refresh_completions( + reset=need_completion_reset(document.text)) + finally: + if self.logfile is False: + self.output("Warning: This query was not logged.", err=True, fg='red') + query = Query(document.text, successful, mutating) + self.query_history.append(query) + + get_toolbar_tokens = create_toolbar_tokens_func(self.completion_refresher.is_refreshing) layout = create_prompt_layout(lexer=MyCliLexer, @@ -500,142 +636,7 @@ def get_continuation_tokens(cli, width): try: while True: - document = self.cli.run(reset_current_buffer=True) - - special.set_expanded_output(False) - - # The reason we check here instead of inside the sqlexecute is - # because we want to raise the Exit exception which will be - # caught by the try/except block that wraps the - # sqlexecute.run() statement. - if quit_command(document.text): - raise EOFError - - try: - document = self.handle_editor_command(self.cli, document) - except RuntimeError as e: - logger.error("sql: %r, error: %r", document.text, e) - logger.error("traceback: %r", traceback.format_exc()) - self.output(str(e), err=True, fg='red') - continue - if self.destructive_warning: - destroy = confirm_destructive_query(document.text) - if destroy is None: - pass # Query was not destructive. Nothing to do here. - elif destroy is True: - self.output('Your call!') - else: - self.output('Wise choice!') - continue - - # Keep track of whether or not the query is mutating. In case - # of a multi-statement query, the overall query is considered - # mutating if any one of the component statements is mutating - mutating = False - - try: - logger.debug('sql: %r', document.text) - - if self.logfile: - self.logfile.write('\n# %s\n' % datetime.now()) - self.logfile.write(document.text) - self.logfile.write('\n') - - successful = False - start = time() - res = sqlexecute.run(document.text) - successful = True - output = [] - total = 0 - for title, cur, headers, status in res: - logger.debug("headers: %r", headers) - logger.debug("rows: %r", cur) - logger.debug("status: %r", status) - threshold = 1000 - if (is_select(status) and - cur and cur.rowcount > threshold): - self.output('The result set has more than %s rows.' - % threshold, fg='red') - if not click.confirm('Do you want to continue?'): - self.output("Aborted!", err=True, fg='red') - break - - if self.auto_vertical_output: - max_width = self.cli.output.get_size().columns - else: - max_width = None - - formatted = format_output(title, cur, headers, - status, self.table_format, - special.is_expanded_output(), max_width) - - output.extend(formatted) - end = time() - total += end - start - mutating = mutating or is_mutating(status) - except UnicodeDecodeError as e: - import pymysql - if pymysql.VERSION < (0, 6, 7): - message = ('You are running an older version of pymysql.\n' - 'Please upgrade to 0.6.7 or above to view binary data.\n' - 'Try \'pip install -U pymysql\'.') - self.output(message) - else: - raise e - except KeyboardInterrupt: - # Restart connection to the database - sqlexecute.connect() - logger.debug("cancelled query, sql: %r", document.text) - self.output("cancelled query", err=True, fg='red') - except NotImplementedError: - self.output('Not Yet Implemented.', fg="yellow") - except OperationalError as e: - logger.debug("Exception: %r", e) - reconnect = True - if (e.args[0] in (2003, 2006, 2013)): - reconnect = click.prompt('Connection reset. Reconnect (Y/n)', - show_default=False, type=bool, default=True) - if reconnect: - logger.debug('Attempting to reconnect.') - try: - sqlexecute.connect() - logger.debug('Reconnected successfully.') - self.output('Reconnected!\nTry the command again.', fg='green') - except OperationalError as e: - logger.debug('Reconnect failed. e: %r', e) - self.output(str(e), err=True, fg='red') - continue # If reconnection failed, don't proceed further. - else: # If user chooses not to reconnect, don't proceed further. - continue - else: - logger.error("sql: %r, error: %r", document.text, e) - logger.error("traceback: %r", traceback.format_exc()) - self.output(str(e), err=True, fg='red') - except Exception as e: - logger.error("sql: %r, error: %r", document.text, e) - logger.error("traceback: %r", traceback.format_exc()) - self.output(str(e), err=True, fg='red') - else: - try: - if special.is_pager_enabled(): - self.output_via_pager('\n'.join(output)) - else: - self.output('\n'.join(output)) - except KeyboardInterrupt: - pass - if special.is_timing_enabled(): - self.output('Time: %0.03fs' % total) - - # Refresh the table names and column names if necessary. - if need_completion_refresh(document.text): - self.refresh_completions( - reset=need_completion_reset(document.text)) - finally: - if self.logfile is False: - self.output("Warning: This query was not logged.", err=True, fg='red') - query = Query(document.text, successful, mutating) - self.query_history.append(query) - + one_iteration() except EOFError: if not self.less_chatty: self.output('Goodbye!') From fa39a021b83628cae5330bcae98e74eb52ecbc5b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 24 Jan 2017 07:30:53 -0600 Subject: [PATCH 0256/1025] Honor smart_completion setting. --- mycli/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index f585c84b5..e0560e85e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -155,8 +155,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.query_history = [] # Initialize completer. - smart_completion = c['main'].as_bool('smart_completion') - self.completer = SQLCompleter(smart_completion) + self.smart_completion = c['main'].as_bool('smart_completion') + self.completer = SQLCompleter(self.smart_completion) self._completer_lock = threading.Lock() # Register custom special commands. @@ -443,7 +443,8 @@ def run_cli(self): logger = self.logger self.configure_pager() - self.refresh_completions() + if self.smart_completion: + self.refresh_completions() project_root = os.path.dirname(PACKAGE_ROOT) author_file = os.path.join(project_root, 'AUTHORS') From 1e71e5b97cb2d0313b21fccdbd5b291bd63afd9b Mon Sep 17 00:00:00 2001 From: chainkite Date: Sun, 29 Jan 2017 20:49:20 +0800 Subject: [PATCH 0257/1025] Fix #333 ctrl-c issue --- mycli/main.py | 9 ++++++++- mycli/sqlexecute.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index e0560e85e..b100d9c82 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -552,9 +552,16 @@ def one_iteration(document=None): else: raise e except KeyboardInterrupt: + # get last connection id + connection_id_to_kill = sqlexecute.connection_id + logger.debug("connection id to kill: %r", connection_id_to_kill) # Restart connection to the database sqlexecute.connect() - logger.debug("cancelled query, sql: %r", document.text) + for title, cur, headers, status in sqlexecute.run('kill %s' % connection_id_to_kill): + status_str = str(status).lower() + if status_str.find('ok') > -1: + logger.debug("cancelled query, connection id: %r, sql: %r", + connection_id_to_kill, document.text) self.output("cancelled query", err=True, fg='red') except NotImplementedError: self.output('Not Yet Implemented.', fg="yellow") diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index d44a11fec..40274827e 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -41,6 +41,7 @@ def __init__(self, database, user, password, host, port, socket, charset, self.local_infile = local_infile self.ssl = ssl self._server_type = None + self.connection_id = None self.connect() def connect(self, database=None, user=None, password=None, host=None, @@ -90,6 +91,8 @@ def connect(self, database=None, user=None, password=None, host=None, self.socket = socket self.charset = charset self.ssl = ssl + # retrieve connection id + self.reset_connection_id() def run(self, statement): """Execute the sql in the database and return the results. The results @@ -221,3 +224,16 @@ def server_type(self): self._server_type = (product_type, version) return self._server_type + + def get_connection_id(self): + if not self.connection_id: + self.reset_connection_id() + return self.connection_id + + def reset_connection_id(self): + # Remember current connection id + _logger.debug('Get current connection id') + res = self.run('select connection_id()') + for title, cur, headers, status in res: + self.connection_id = list(cur)[0][0] + _logger.debug('Current connection id: %s', self.connection_id) From 32fbfb88ac7a8a861962d816f47b57811230c784 Mon Sep 17 00:00:00 2001 From: chainkite Date: Thu, 2 Feb 2017 21:39:48 +0800 Subject: [PATCH 0258/1025] Fix #333 use fetchone() when get connection id --- mycli/sqlexecute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 40274827e..b28008e82 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -235,5 +235,5 @@ def reset_connection_id(self): _logger.debug('Get current connection id') res = self.run('select connection_id()') for title, cur, headers, status in res: - self.connection_id = list(cur)[0][0] + self.connection_id = cur.fetchone()[0] _logger.debug('Current connection id: %s', self.connection_id) From 9044017530a7c3edc2f2d91988bc9bfe5062b938 Mon Sep 17 00:00:00 2001 From: chainkite Date: Mon, 6 Feb 2017 14:25:24 +0800 Subject: [PATCH 0259/1025] catch exception when kill query --- mycli/main.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index b100d9c82..8bb7721a6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -557,12 +557,15 @@ def one_iteration(document=None): logger.debug("connection id to kill: %r", connection_id_to_kill) # Restart connection to the database sqlexecute.connect() - for title, cur, headers, status in sqlexecute.run('kill %s' % connection_id_to_kill): - status_str = str(status).lower() - if status_str.find('ok') > -1: - logger.debug("cancelled query, connection id: %r, sql: %r", - connection_id_to_kill, document.text) - self.output("cancelled query", err=True, fg='red') + try: + for title, cur, headers, status in sqlexecute.run('kill %s' % connection_id_to_kill): + status_str = str(status).lower() + if status_str.find('ok') > -1: + logger.debug("cancelled query, connection id: %r, sql: %r", + connection_id_to_kill, document.text) + self.output("cancelled query", err=True, fg='red') + except Exception as e: + self.output("cancel query with error: %r" % e, err=True, fg='red') except NotImplementedError: self.output('Not Yet Implemented.', fg="yellow") except OperationalError as e: From 07e800b030c4a5b6935dea9e44941d673c522192 Mon Sep 17 00:00:00 2001 From: chainkite Date: Thu, 9 Feb 2017 15:05:13 +0800 Subject: [PATCH 0260/1025] revise error message --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 8bb7721a6..b7ac62f5e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -565,7 +565,7 @@ def one_iteration(document=None): connection_id_to_kill, document.text) self.output("cancelled query", err=True, fg='red') except Exception as e: - self.output("cancel query with error: %r" % e, err=True, fg='red') + self.output('Encountered error while cancelling query: %s' % str(e), err=True, fg='red') except NotImplementedError: self.output('Not Yet Implemented.', fg="yellow") except OperationalError as e: From feaf3295c0cc395da56d4436ab35457e853d7d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Tue, 14 Feb 2017 19:43:09 +0100 Subject: [PATCH 0261/1025] Change link to pymysql to a working one --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 016c5396f..a15cbc033 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ of this app. [Click](http://click.pocoo.org/3/) is used for command line option parsing and printing error messages. -Thanks to [PyMysql](http://www.pymysql.org/) for a pure python adapter to MySQL database. +Thanks to [PyMysql](https://github.com/PyMySQL/PyMySQL) for a pure python adapter to MySQL database. [Tabulate](https://pypi.python.org/pypi/tabulate) library is used for pretty printing the output of tables. From 99cc5cf2c5b7a560c4f9ee459e3286ca7b554f96 Mon Sep 17 00:00:00 2001 From: John Sterling Date: Tue, 14 Feb 2017 17:18:28 -0500 Subject: [PATCH 0262/1025] 316: users needs to be cleared prior to refresh to prevent repeatedly appending the same users every time a refreh is triggered --- mycli/sqlcompleter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index e79806771..da7f34f87 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -192,6 +192,7 @@ def set_dbname(self, dbname): def reset_completions(self): self.databases = [] + self.users = [] self.show_items = [] self.dbname = '' self.dbmetadata = {'tables': {}, 'views': {}, 'functions': {}} From a2e63ddd6e0eb143380781dbaaf40f412441d0fb Mon Sep 17 00:00:00 2001 From: Gilbert Consellado Date: Thu, 16 Feb 2017 16:37:57 +0800 Subject: [PATCH 0263/1025] Add some keywords on the keywords list --- mycli/sqlcompleter.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index da7f34f87..c31f3ed02 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -19,21 +19,21 @@ class SQLCompleter(Completer): keywords = ['ACCESS', 'ADD', 'ALL', 'ALTER TABLE', 'AND', 'ANY', 'AS', - 'ASC', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', - 'CASE', 'CHAR', 'CHECK', 'COLUMN', 'COMMENT', 'COMMIT', - 'CHANGE MASTER TO', 'CREATE', 'CURRENT', 'DATABASE', 'DATE', + 'ASC', 'AUTO_INCREMENT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BINARY', 'BY', + 'CASE', 'CHAR', 'CHECK', 'COLUMN', 'COMMENT', 'COMMIT', 'CONSTRAINT', + 'CHANGE MASTER TO', 'CHARACTER SET', 'COLLATE', 'CREATE', 'CURRENT', 'CURRENT_TIMESTAMP', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', 'DELETE FROM', 'DELIMITER', 'DESC', - 'DESCRIBE', 'DROP', 'ELSE', 'END', 'ESCAPE', 'EXISTS', - 'FILE', 'FLOAT', 'FOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', + 'DESCRIBE', 'DROP', 'ELSE', 'END', 'ENGINE', 'ESCAPE', 'EXISTS', + 'FILE', 'FLOAT', 'FOR', 'FOREIGN', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', 'INCREMENT', 'INDEX', - 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'LEFT', + 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'KEY', 'LEFT', 'LEVEL', 'LIKE', 'LIMIT', 'LOCK', 'LOGS', 'LONG', 'MASTER', 'MODE', 'MODIFY', 'NOT', 'NULL', 'NUMBER', 'OFFSET', 'ON', 'OPTION', 'OR', 'ORDER BY', 'OUTER', 'OWNER', 'PASSWORD', 'PORT', 'PRIMARY', - 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'REGEXP', 'RENAME', 'REPAIR', 'RESET', - 'REVOKE', 'RIGHT', 'ROLLBACK','ROW', 'ROWS', 'SELECT', 'SESSION', 'SET', + 'PRIVILEGES', 'PROCESSLIST', 'PURGE', 'REFERENCES', 'REGEXP', 'RENAME', 'REPAIR', 'RESET', + 'REVOKE', 'RIGHT', 'ROLLBACK','ROW', 'ROWS', 'ROW_FORMAT', 'SELECT', 'SESSION', 'SET', 'SHARE', 'SHOW', 'SLAVE', 'SMALLINT', 'START', 'STOP', 'TABLE', 'THEN', - 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UNION', 'UNIQUE', 'UPDATE', + 'TO', 'TRANSACTION', 'TRIGGER', 'TRUNCATE', 'UNION', 'UNIQUE', 'UNSIGNED', 'UPDATE', 'USE', 'USER', 'USING', 'VALUES', 'VARCHAR', 'VIEW', 'WHEN', 'WHERE', 'WITH'] From e943050fc78ce2f5ec73ac4e8505271c934a43f5 Mon Sep 17 00:00:00 2001 From: Gilbert Consellado Date: Thu, 16 Feb 2017 19:46:21 +0800 Subject: [PATCH 0264/1025] Change the "FOREIGN" keyword to "FOREIGN KEY" --- mycli/sqlcompleter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index c31f3ed02..f5fc3c903 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -24,7 +24,7 @@ class SQLCompleter(Completer): 'CHANGE MASTER TO', 'CHARACTER SET', 'COLLATE', 'CREATE', 'CURRENT', 'CURRENT_TIMESTAMP', 'DATABASE', 'DATE', 'DECIMAL', 'DEFAULT', 'DELETE FROM', 'DELIMITER', 'DESC', 'DESCRIBE', 'DROP', 'ELSE', 'END', 'ENGINE', 'ESCAPE', 'EXISTS', - 'FILE', 'FLOAT', 'FOR', 'FOREIGN', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', + 'FILE', 'FLOAT', 'FOR', 'FOREIGN KEY', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', 'GRANT', 'GROUP BY', 'HAVING', 'HOST', 'IDENTIFIED', 'IN', 'INCREMENT', 'INDEX', 'INSERT INTO', 'INTEGER', 'INTO', 'INTERVAL', 'IS', 'JOIN', 'KEY', 'LEFT', 'LEVEL', 'LIKE', 'LIMIT', 'LOCK', 'LOGS', 'LONG', 'MASTER', 'MODE', From aeac636cddb8ace3054901e2f55f9025c5b80c2b Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 18 Feb 2017 09:24:37 -0800 Subject: [PATCH 0265/1025] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a15cbc033..6b682a2c4 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ If you already know how to install python packages, then you can install it via You might need sudo on linux. ``` -$ pip install mycli +$ pip install -U mycli ``` or From 247b24a7c53c9d8312227b5bef5d28610afd0058 Mon Sep 17 00:00:00 2001 From: 0x4ec7 <0x4ec7@gmail.com> Date: Tue, 21 Feb 2017 19:05:51 +0800 Subject: [PATCH 0266/1025] Fix: Crash if statement starts with # --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index b7ac62f5e..a91eba5bb 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -954,7 +954,7 @@ def query_starts_with(query, prefixes): """Check if the query starts with any item from *prefixes*.""" prefixes = [prefix.lower() for prefix in prefixes] formatted_sql = sqlparse.format(query.lower(), strip_comments=True) - return formatted_sql.split()[0] in prefixes + return bool(formatted_sql) and formatted_sql.split()[0] in prefixes def queries_start_with(queries, prefixes): """Check if any queries start with any item from *prefixes*.""" From 75166ae1bfcc2c58b0da36b29ffbfe91fdb7d7b1 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 21 Feb 2017 20:06:17 +0100 Subject: [PATCH 0267/1025] FIX: disable align (space padding) for table format tsv #348 --- mycli/packages/tabulate.py | 46 +++++++++++++++++++++----------------- tests/test_main.py | 4 ++-- tests/test_tabulate.py | 8 +++++++ 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index f0874d525..978f89439 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -87,7 +87,7 @@ def _is_file(f): TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", "linebetweenrows", "linebelow", "headerrow", "datarow", - "padding", "with_header_hide"]) + "padding", "with_header_hide", "with_align"]) def _pipe_segment_with_colons(align, colwidth): @@ -162,13 +162,13 @@ def escape_char(c): headerrow=DataRow("", " ", ""), datarow=DataRow("", " ", ""), padding=0, - with_header_hide=["lineabove", "linebelow"]), + with_header_hide=["lineabove", "linebelow"], with_align=True), "plain": TableFormat(lineabove=None, linebelowheader=None, linebetweenrows=None, linebelow=None, headerrow=DataRow("", " ", ""), datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None), + padding=0, with_header_hide=None, with_align=True), "grid": TableFormat(lineabove=Line("+", "-", "+", "+"), linebelowheader=Line("+", "=", "+", "+"), @@ -176,7 +176,7 @@ def escape_char(c): linebelow=Line("+", "-", "+", "+"), headerrow=DataRow("|", "|", "|"), datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=True), "fancy_grid": TableFormat(lineabove=Line("╒", "═", "╤", "╕"), linebelowheader=Line("╞", "═", "╪", "╡"), @@ -184,7 +184,7 @@ def escape_char(c): linebelow=Line("╘", "═", "╧", "╛"), headerrow=DataRow("│", "│", "│"), datarow=DataRow("│", "│", "│"), - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=True), "pipe": TableFormat(lineabove=_pipe_line_with_colons, linebelowheader=_pipe_line_with_colons, @@ -193,7 +193,7 @@ def escape_char(c): headerrow=DataRow("|", "|", "|"), datarow=DataRow("|", "|", "|"), padding=1, - with_header_hide=["lineabove"]), + with_header_hide=["lineabove"], with_align=True), "orgtbl": TableFormat(lineabove=None, linebelowheader=Line("|", "-", "+", "|"), @@ -201,7 +201,7 @@ def escape_char(c): linebelow=None, headerrow=DataRow("|", "|", "|"), datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=True), "psql": TableFormat(lineabove=Line("+", "-", "+", "+"), linebelowheader=Line("|", "-", "+", "|"), @@ -209,7 +209,7 @@ def escape_char(c): linebelow=Line("+", "-", "+", "+"), headerrow=DataRow("|", "|", "|"), datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=True), "rst": TableFormat(lineabove=Line("", "=", " ", ""), linebelowheader=Line("", "=", " ", ""), @@ -217,7 +217,7 @@ def escape_char(c): linebelow=Line("", "=", " ", ""), headerrow=DataRow("", " ", ""), datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None), + padding=0, with_header_hide=None, with_align=True), "mediawiki": TableFormat(lineabove=Line("{| class=\"wikitable\" style=\"text-align: left;\"", "", "", "\n|+ \n|-"), @@ -226,7 +226,7 @@ def escape_char(c): linebelow=Line("|}", "", "", ""), headerrow=partial(_mediawiki_row_with_attrs, "!"), datarow=partial(_mediawiki_row_with_attrs, "|"), - padding=0, with_header_hide=None), + padding=0, with_header_hide=None, with_align=True), "html": TableFormat(lineabove=Line("", "", "", ""), linebelowheader=None, @@ -234,7 +234,7 @@ def escape_char(c): linebelow=Line("
", "", "", ""), headerrow=partial(_html_row_with_attrs, "th"), datarow=partial(_html_row_with_attrs, "td"), - padding=0, with_header_hide=None), + padding=0, with_header_hide=None, with_align=False), "latex": TableFormat(lineabove=_latex_line_begin_tabular, linebelowheader=Line("\\hline", "", "", ""), @@ -242,7 +242,7 @@ def escape_char(c): linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), headerrow=_latex_row, datarow=_latex_row, - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=False), "latex_booktabs": TableFormat(lineabove=partial(_latex_line_begin_tabular, booktabs=True), linebelowheader=Line("\\midrule", "", "", ""), @@ -250,13 +250,13 @@ def escape_char(c): linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", ""), headerrow=_latex_row, datarow=_latex_row, - padding=1, with_header_hide=None), + padding=1, with_header_hide=None, with_align=False), "tsv": TableFormat(lineabove=None, linebelowheader=None, linebetweenrows=None, linebelow=None, headerrow=DataRow("", "\t", ""), datarow=DataRow("", "\t", ""), - padding=0, with_header_hide=None)} + padding=0, with_header_hide=None, with_align=False)} tabulate_formats = list(sorted(_table_formats.keys())) @@ -904,8 +904,14 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", else: width_fn = wcswidth - # align columns - aligns = [numalign if ct in [int,float] else stralign for ct in coltypes] + if not isinstance(tablefmt, TableFormat): + tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) + + if tablefmt.with_align: + # align columns + aligns = [numalign if ct in [int,float] else stralign for ct in coltypes] + else: + aligns = [False for ct in coltypes] minwidths = [width_fn(h) + MIN_PADDING for h in headers] if headers else [0]*len(cols) cols = [_align_column(c, a, minw, has_invisible) for c, a, minw in zip(cols, aligns, minwidths)] @@ -913,7 +919,10 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", if headers: # align headers and add headers t_cols = cols or [['']] * len(headers) - t_aligns = aligns or [stralign] * len(headers) + if tablefmt.with_align: + t_aligns = aligns or [stralign] * len(headers) + else: + t_aligns = [False for ct in coltypes] minwidths = [max(minw, width_fn(c[0])) for minw, c in zip(minwidths, t_cols)] headers = [_align_header(h, a, minw) for h, a, minw in zip(headers, t_aligns, minwidths)] @@ -922,9 +931,6 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", minwidths = [width_fn(c[0]) for c in cols] rows = list(zip(*cols)) - if not isinstance(tablefmt, TableFormat): - tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) - return _format_table(tablefmt, headers, rows, minwidths, aligns), rows diff --git a/tests/test_main.py b/tests/test_main.py index 974eca74f..49e8be3ff 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -31,7 +31,7 @@ def test_format_output_no_table(): results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], 'test status', None) - expected = ['Title', u'head1 \thead2\nabc \tdef', 'test status'] + expected = ['Title', u'head1\thead2\nabc\tdef', 'test status'] assert results == expected @dbtest @@ -98,7 +98,7 @@ def test_batch_mode(executor): result = runner.invoke(cli, args=CLI_ARGS, input=sql) assert result.exit_code == 0 - assert ' count(*)\n 3\na\nabc\n' in result.output + assert 'count(*)\n3\na\nabc\n' in result.output @dbtest def test_batch_mode_table(executor): diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py index 351261971..e0ddc407d 100644 --- a/tests/test_tabulate.py +++ b/tests/test_tabulate.py @@ -12,3 +12,11 @@ def test_dont_strip_leading_whitespace(): |---------| | abc | +---------+ ''').strip() +def test_dont_add_whitespace(): + data = [[3, 4]] + headers = ['1', '2'] + tbl, _ = tabulate(data, headers, tablefmt='tsv') + assert tbl == dedent(''' + 1\t2 + 3\t4 + ''').strip() From c86568ef3b6931e0cb5df186be000da23acbde06 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sun, 26 Feb 2017 17:02:22 +0100 Subject: [PATCH 0268/1025] fix doctest --- conftest.py | 13 ++++++- mycli/packages/tabulate.py | 78 +++++++++++++++++++------------------- pytest.ini | 2 + tests/pytest.ini | 2 - 4 files changed, 53 insertions(+), 42 deletions(-) create mode 100644 pytest.ini delete mode 100644 tests/pytest.ini diff --git a/conftest.py b/conftest.py index cee5a17e4..5f72d7107 100644 --- a/conftest.py +++ b/conftest.py @@ -1,2 +1,11 @@ -# https://pytest.org/latest/example/pythoncollection.html -collect_ignore = ["setup.py"] +import sys +collect_ignore = [ + "setup.py", + "mycli/magic.py", + "mycli/packages/parseutils.py", +] +if sys.version_info[0] > 2: + collect_ignore.extend([ + "mycli/packages/counter.py", + "mycli/packages/ordereddict.py", + ]) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index 978f89439..fa9718267 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -270,14 +270,15 @@ def simple_separated_format(separator): """Construct a simple TableFormat with columns separated by a separator. >>> tsv = simple_separated_format("\\t") ; \ - tabulate([["foo", 1], ["spam", 23]], tablefmt=tsv) == 'foo \\t 1\\nspam\\t23' - True - + print(tabulate([["foo", 1], ["spam", 23]], tablefmt=tsv)[0].replace('\\t', r'\\t')) + foo\\t1 + spam\\t23 """ return TableFormat(None, None, None, None, headerrow=DataRow('', separator, ''), datarow=DataRow('', separator, ''), - padding=0, with_header_hide=None) + padding=0, with_header_hide=None, + with_align=False) def _isconvertible(conv, string): @@ -498,16 +499,17 @@ def _column_type(strings, has_invisible=True): def _format(val, valtype, floatfmt, missingval=""): - """Format a value accoding to its type. + u"""Format a value accoding to its type. Unicode is supported: >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', '\u0446\u0438\u0444\u0440\u0430'] ; \ tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] ; \ - good_result = '\\u0431\\u0443\\u043a\\u0432\\u0430 \\u0446\\u0438\\u0444\\u0440\\u0430\\n------- -------\\n\\u0430\\u0437 2\\n\\u0431\\u0443\\u043a\\u0438 4' ; \ - tabulate(tbl, headers=hrow) == good_result - True - + print(tabulate(tbl, headers=hrow)[0]) + буква цифра + ------- ------- + аз 2 + буки 4 """ if val is None: return missingval @@ -656,7 +658,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", missingval=""): """Format a fixed width table for pretty printing. - >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])) + >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])[0]) --- --------- 1 2.34 -56 8.999 @@ -686,7 +688,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", with the plain-text format of R and Pandas' dataframes. >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], - ... headers="firstrow")) + ... headers="firstrow")[0]) sex age ----- ----- ----- Alice F 24 @@ -714,7 +716,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", >>> print(tabulate([["spam", 1, None], ... ["eggs", 42, 3.14], - ... ["other", None, 2.7]], missingval="?")) + ... ["other", None, 2.7]], missingval="?")[0]) ----- -- ---- spam 1 ? eggs 42 3.14 @@ -730,25 +732,25 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", it separates columns with a double space: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "plain")) + ... ["strings", "numbers"], "plain")[0]) strings numbers spam 41.9999 eggs 451 - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="plain")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="plain")[0]) spam 41.9999 eggs 451 "simple" format is like Pandoc simple_tables: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "simple")) + ... ["strings", "numbers"], "simple")[0]) strings numbers --------- --------- spam 41.9999 eggs 451 - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="simple")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="simple")[0]) ---- -------- spam 41.9999 eggs 451 @@ -758,7 +760,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", Pandoc grid_tables: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "grid")) + ... ["strings", "numbers"], "grid")[0]) +-----------+-----------+ | strings | numbers | +===========+===========+ @@ -767,7 +769,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", | eggs | 451 | +-----------+-----------+ - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="grid")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="grid")[0]) +------+----------+ | spam | 41.9999 | +------+----------+ @@ -777,7 +779,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", "fancy_grid" draws a grid using box-drawing characters: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "fancy_grid")) + ... ["strings", "numbers"], "fancy_grid")[0]) ╒═══════════╤═══════════╕ │ strings │ numbers │ ╞═══════════╪═══════════╡ @@ -790,13 +792,13 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", pipe_tables: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "pipe")) + ... ["strings", "numbers"], "pipe")[0]) | strings | numbers | |:----------|----------:| | spam | 41.9999 | | eggs | 451 | - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="pipe")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="pipe")[0]) |:-----|---------:| | spam | 41.9999 | | eggs | 451 | @@ -807,14 +809,14 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", intersections: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "orgtbl")) + ... ["strings", "numbers"], "orgtbl")[0]) | strings | numbers | |-----------+-----------| | spam | 41.9999 | | eggs | 451 | - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="orgtbl")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="orgtbl")[0]) | spam | 41.9999 | | eggs | 451 | @@ -822,7 +824,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", note that reStructuredText accepts also "grid" tables: >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "rst")) + ... ["strings", "numbers"], "rst")[0]) ========= ========= strings numbers ========= ========= @@ -830,7 +832,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", eggs 451 ========= ========= - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")) + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")[0]) ==== ======== spam 41.9999 eggs 451 @@ -840,7 +842,7 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", MediaWiki-based sites: >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="mediawiki")) + ... headers="firstrow", tablefmt="mediawiki")[0]) {| class="wikitable" style="text-align: left;" |+ |- @@ -854,31 +856,31 @@ def tabulate(tabular_data, headers=[], tablefmt="simple", "html" produces HTML markup: >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="html")) + ... headers="firstrow", tablefmt="html")[0]) - - - + + +
strings numbers
spam 41.9999
eggs 451
stringsnumbers
spam41.9999
eggs451
"latex" produces a tabular environment of LaTeX document markup: - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex")) - \\begin{tabular}{lr} + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex")[0]) + \\begin{tabular}{ll} \\hline - spam & 41.9999 \\\\ - eggs & 451 \\\\ + spam & 41.9999 \\\\ + eggs & 451 \\\\ \\hline \\end{tabular} "latex_booktabs" produces a tabular environment of LaTeX document markup using the booktabs.sty package: - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex_booktabs")) - \\begin{tabular}{lr} + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex_booktabs")[0]) + \\begin{tabular}{ll} \\toprule - spam & 41.9999 \\\\ - eggs & 451 \\\\ + spam & 41.9999 \\\\ + eggs & 451 \\\\ \\bottomrule \end{tabular} diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..7c5b52b77 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts=--capture=sys --showlocals --doctest-modules diff --git a/tests/pytest.ini b/tests/pytest.ini deleted file mode 100644 index f78774051..000000000 --- a/tests/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts=--capture=sys --showlocals \ No newline at end of file From c3ed9a67aeb1a30ddf0ed67203def3aa4c9753ab Mon Sep 17 00:00:00 2001 From: John Sterling Date: Wed, 1 Mar 2017 21:15:18 -0500 Subject: [PATCH 0269/1025] #288: only use the default less flags if they are not already set --- mycli/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index a91eba5bb..6bb91b755 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -665,8 +665,9 @@ def output_via_pager(self, text): click.echo_via_pager(text) def configure_pager(self): - # Provide sane defaults for less. - os.environ['LESS'] = '-RXF' + # Provide sane defaults for less if they are empty. + if not os.environ.get('LESS'): + os.environ['LESS'] = '-RXF' cnf = self.read_my_cnf_files(self.cnf_files, ['pager', 'skip-pager']) if cnf['pager']: From 221b50ce19dc1a28ac79cfc39a54f2e9e18b2878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Thu, 2 Mar 2017 18:46:47 +0100 Subject: [PATCH 0270/1025] Replace obsolete pycrypto with pycryptodome fork The pycrypto library is abandoned since 2014 with a lot of bugs open. It has been forked into pycryptodome which is mostly API-compatible with pycrypto. FWICS, the bits used by mycli work fine with the new library. Therefore, switch to using pycryptodome instead of obsolete pycrypto. I've also enabled it for Windows since the latter library officially supports this system and provides binary wheels for it, so the dependency should no longer be a problem. Since the actual Python code isn't changed, mycli can still formally work with pycrypto or without the library at all. However, pip will now install the new library. --- mycli/config.py | 4 ++-- mycli/main.py | 2 +- setup.py | 7 +------ tests/test_config.py | 12 ++++++------ 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 284a8b400..97ead1fcd 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -19,7 +19,7 @@ class CryptoError(Exception): """ - Exception to signal about pycrypto not available. + Exception to signal about pycrypto(dome) not available. """ pass @@ -126,7 +126,7 @@ def read_and_decrypt_mylogin_cnf(f): :rtype: io.BytesIO or None """ if AES is None: - raise CryptoError('pycrypto is not available.') + raise CryptoError('pycrypto(dome) is not available.') # Number of bytes used to store the length of ciphertext. MAX_CIPHER_STORE_LEN = 4 diff --git a/mycli/main.py b/mycli/main.py index 6bb91b755..f0e175fd2 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -174,7 +174,7 @@ def __init__(self, sqlexecute=None, prompt=None, # There was an error reading the login path file. print('Error: Unable to read login path file.') except CryptoError: - click.secho('Warning: .mylogin.cnf was not read: pycrypto ' + click.secho('Warning: .mylogin.cnf was not read: pycrypto(dome) ' 'module is not available.') self.cli = None diff --git a/setup.py b/setup.py index c3693572e..caa140647 100644 --- a/setup.py +++ b/setup.py @@ -18,14 +18,9 @@ 'PyMySQL >= 0.6.2', 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.6', + 'pycryptodome', ] -# pycrypto is a hard package to install on Windows, so we make it an optional -# dependency. When it's installed, we can read mylogin.cnf, when it is not -# available, we skip reading mylogin.cnf and print a warning message. -if platform.system() != 'Windows': - install_requirements.append('pycrypto >= 2.6.1') - setup( name='mycli', author='Amjith Ramanujam', diff --git a/tests/test_config.py b/tests/test_config.py index efd66d017..7879dd109 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,7 +11,7 @@ open_mylogin_cnf, read_and_decrypt_mylogin_cnf, str_to_bool) -with_pycrypto = ['pycrypto' in set([package.project_name for package in +with_pycryptodome = ['pycryptodome' in set([package.project_name for package in pip.get_installed_distributions()])] LOGIN_PATH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -26,13 +26,13 @@ def open_bmylogin_cnf(name): return buf -@pytest.mark.skipif(with_pycrypto, reason='requires pycrypto missing') +@pytest.mark.skipif(with_pycryptodome, reason='requires pycryptodome missing') def test_read_mylogin_cnf_without_crypto(): with pytest.raises(CryptoError): mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) -@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_read_mylogin_cnf(): """Tests that a login path file can be read and decrypted.""" mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) @@ -44,14 +44,14 @@ def test_read_mylogin_cnf(): assert word in contents -@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_decrypt_blank_mylogin_cnf(): """Test that a blank login path file is handled correctly.""" mylogin_cnf = read_and_decrypt_mylogin_cnf(BytesIO()) assert mylogin_cnf is None -@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_corrupted_login_key(): """Test that a corrupted login path key is handled correctly.""" buf = open_bmylogin_cnf(LOGIN_PATH_FILE) @@ -68,7 +68,7 @@ def test_corrupted_login_key(): assert mylogin_cnf is None -@pytest.mark.skipif(not with_pycrypto, reason='requires pycrypto') +@pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_corrupted_pad(): """Tests that a login path file with a corrupted pad is partially read.""" buf = open_bmylogin_cnf(LOGIN_PATH_FILE) From 557be1dacd703a171d9ac3f161b65fc1076664c5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 2 Mar 2017 22:57:53 -0600 Subject: [PATCH 0271/1025] Add Python 3.6 to tests and setup file. --- .travis.yml | 1 + setup.py | 1 + tox.ini | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5450b982b..826c01d53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" env: - PYMYSQL_VERSION=0.6.7 diff --git a/setup.py b/setup.py index caa140647..54a2404d3 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Programming Language :: SQL', 'Topic :: Database', 'Topic :: Database :: Front-Ends', diff --git a/tox.ini b/tox.ini index a2d769118..d3f9aef14 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34, py35 +envlist = py26, py27, py33, py34, py35, py36 [testenv] deps = pytest mock From 12c94227ee0dc920d18806b60de11d44b2a2376d Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 25 Feb 2017 21:00:44 +0100 Subject: [PATCH 0272/1025] tests for mycli.packages.special.iocommands --- tests/test_special_iocommands.py | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/test_special_iocommands.py diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py new file mode 100644 index 000000000..5dbbc1602 --- /dev/null +++ b/tests/test_special_iocommands.py @@ -0,0 +1,37 @@ +import mycli.packages.special +import os +def test_set_get_pager(): + mycli.packages.special.set_pager_enabled(True) + assert mycli.packages.special.is_pager_enabled() + mycli.packages.special.set_pager_enabled(False) + assert not mycli.packages.special.is_pager_enabled() + mycli.packages.special.set_pager('less') + assert os.environ['PAGER'] == "less" + mycli.packages.special.set_pager(False) + assert os.environ['PAGER'] == "less" + del os.environ['PAGER'] + mycli.packages.special.set_pager(False) + mycli.packages.special.disable_pager() + assert not mycli.packages.special.is_pager_enabled() + +def test_set_get_timing(): + mycli.packages.special.set_timing_enabled(True) + assert mycli.packages.special.is_timing_enabled() + mycli.packages.special.set_timing_enabled(False) + assert not mycli.packages.special.is_timing_enabled() + +def test_set_get_expanded_output(): + mycli.packages.special.set_expanded_output(True) + assert mycli.packages.special.is_expanded_output() + mycli.packages.special.set_expanded_output(False) + assert not mycli.packages.special.is_expanded_output() + +def test_editor_command(): + assert mycli.packages.special.editor_command(r'hello\e') + assert mycli.packages.special.editor_command(r'\ehello') + assert not mycli.packages.special.editor_command(r'hello') + + assert mycli.packages.special.get_filename(r'\e filename') == "filename" + + os.environ['EDITOR'] = 'true' + mycli.packages.special.open_external_editor(r'select 1') == "select 1" From 97d31af5f37e19e8a60ccdf24731913203e52315 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 5 Mar 2017 12:49:38 -0600 Subject: [PATCH 0273/1025] Drop Python 2.6 support. --- .travis.yml | 1 - conftest.py | 5 - debian/control | 2 +- debian/mycli.triggers | 1 - mycli/clistyle.py | 2 +- mycli/completion_refresher.py | 5 +- mycli/main.py | 10 +- mycli/packages/counter.py | 190 ---------------------------------- mycli/packages/ordereddict.py | 127 ----------------------- mycli/sqlcompleter.py | 7 +- setup.py | 1 - tox.ini | 2 +- 12 files changed, 8 insertions(+), 345 deletions(-) delete mode 100644 mycli/packages/counter.py delete mode 100644 mycli/packages/ordereddict.py diff --git a/.travis.yml b/.travis.yml index 826c01d53..516df2604 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.6" - "2.7" - "3.3" - "3.4" diff --git a/conftest.py b/conftest.py index 5f72d7107..d2cd1336c 100644 --- a/conftest.py +++ b/conftest.py @@ -4,8 +4,3 @@ "mycli/magic.py", "mycli/packages/parseutils.py", ] -if sys.version_info[0] > 2: - collect_ignore.extend([ - "mycli/packages/counter.py", - "mycli/packages/ordereddict.py", - ]) diff --git a/debian/control b/debian/control index f1602021a..418383263 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Standards-Version: 3.9.5 Package: mycli Architecture: any -Pre-Depends: dpkg (>= 1.16.1), python2.7-minimal | python2.6-minimal, ${misc:Pre-Depends} +Pre-Depends: dpkg (>= 1.16.1), python2.7-minimal, ${misc:Pre-Depends} Depends: ${python:Depends}, ${misc:Depends} Description: CLI for MySQL Database. With auto-completion and syntax highlighting. CLI for MySQL Database. With auto-completion and syntax highlighting. diff --git a/debian/mycli.triggers b/debian/mycli.triggers index 084bfd36d..b0b1d2184 100644 --- a/debian/mycli.triggers +++ b/debian/mycli.triggers @@ -1,7 +1,6 @@ # Register interest in Python interpreter changes (Python 2 for now); and # don't make the Python package dependent on the virtualenv package # processing (noawait) -interest-noawait /usr/bin/python2.6 interest-noawait /usr/bin/python2.7 # Also provide a symbolic trigger for all dh-virtualenv packages diff --git a/mycli/clistyle.py b/mycli/clistyle.py index a2a7b2a70..ef7c1c9d8 100644 --- a/mycli/clistyle.py +++ b/mycli/clistyle.py @@ -13,7 +13,7 @@ def style_factory(name, cli_style): styles = {} styles.update(style.styles) styles.update(default_style_extensions) - custom_styles = dict([(string_to_tokentype(x), y) for x, y in cli_style.items()]) + custom_styles = {string_to_tokentype(x): y for x, y in cli_style.items()} styles.update(custom_styles) return style_from_dict(styles) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index b5b4142ed..33afa009c 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -1,9 +1,6 @@ import threading from .packages.special.main import COMMANDS -try: - from collections import OrderedDict -except ImportError: - from .packages.ordereddict import OrderedDict +from collections import OrderedDict from .sqlcompleter import SQLCompleter from .sqlexecute import SQLExecute diff --git a/mycli/main.py b/mycli/main.py index f0e175fd2..e852fddc4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -271,11 +271,7 @@ def initialize_logging(self): root_logger.addHandler(handler) root_logger.setLevel(level_map[log_level.upper()]) - # Only capture warnings on Python 2.7 and later. - try: - logging.captureWarnings(True) - except AttributeError: - pass + logging.captureWarnings(True) root_logger.debug('Initializing mycli logging.') root_logger.debug('Log file %r.', log_file) @@ -309,7 +305,7 @@ def get(key): result = cnf[sect][key] return result - return dict([(x, get(x)) for x in keys]) + return {x: get(x) for x in keys} def merge_ssl_with_cnf(self, ssl, cnf): """Merge SSL configuration dict with cnf dict""" @@ -809,7 +805,7 @@ def cli(database, user, host, port, socket, password, dbname, } # remove empty ssl options - ssl = dict((k, v) for (k, v) in ssl.items() if v is not None) + ssl = {k: v for k, v in ssl.items() if v is not None} if database and '://' in database: mycli.connect_uri(database, local_infile, ssl) else: diff --git a/mycli/packages/counter.py b/mycli/packages/counter.py deleted file mode 100644 index e74db3116..000000000 --- a/mycli/packages/counter.py +++ /dev/null @@ -1,190 +0,0 @@ -from __future__ import print_function -from operator import itemgetter -from heapq import nlargest -from itertools import repeat, ifilter - -class Counter(dict): - '''Dict subclass for counting hashable objects. Sometimes called a bag - or multiset. Elements are stored as dictionary keys and their counts - are stored as dictionary values. - - >>> Counter('zyzygy') - Counter({'y': 3, 'z': 2, 'g': 1}) - - ''' - - def __init__(self, iterable=None, **kwds): - '''Create a new, empty Counter object. And if given, count elements - from an input iterable. Or, initialize the count from another mapping - of elements to their counts. - - >>> c = Counter() # a new, empty counter - >>> c = Counter('gallahad') # a new counter from an iterable - >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping - >>> c = Counter(a=4, b=2) # a new counter from keyword args - - ''' - self.update(iterable, **kwds) - - def __missing__(self, key): - return 0 - - def most_common(self, n=None): - '''List the n most common elements and their counts from the most - common to the least. If n is None, then list all element counts. - - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('r', 2), ('b', 2)] - - ''' - if n is None: - return sorted(self.iteritems(), key=itemgetter(1), reverse=True) - return nlargest(n, self.iteritems(), key=itemgetter(1)) - - def elements(self): - '''Iterator over elements repeating each as many times as its count. - - >>> c = Counter('ABCABC') - >>> sorted(c.elements()) - ['A', 'A', 'B', 'B', 'C', 'C'] - - If an element's count has been set to zero or is a negative number, - elements() will ignore it. - - ''' - for elem, count in self.iteritems(): - for _ in repeat(None, count): - yield elem - - # Override dict methods where the meaning changes for Counter objects. - - @classmethod - def fromkeys(cls, iterable, v=None): - raise NotImplementedError( - 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') - - def update(self, iterable=None, **kwds): - '''Like dict.update() but add counts instead of replacing them. - - Source can be an iterable, a dictionary, or another Counter instance. - - >>> c = Counter('which') - >>> c.update('witch') # add elements from another iterable - >>> d = Counter('watch') - >>> c.update(d) # add elements from another counter - >>> c['h'] # four 'h' in which, witch, and watch - 4 - - ''' - if iterable is not None: - if hasattr(iterable, 'iteritems'): - if self: - self_get = self.get - for elem, count in iterable.iteritems(): - self[elem] = self_get(elem, 0) + count - else: - dict.update(self, iterable) # fast path when counter is empty - else: - self_get = self.get - for elem in iterable: - self[elem] = self_get(elem, 0) + 1 - if kwds: - self.update(kwds) - - def copy(self): - 'Like dict.copy() but returns a Counter instance instead of a dict.' - return Counter(self) - - def __delitem__(self, elem): - 'Like dict.__delitem__() but does not raise KeyError for missing values.' - if elem in self: - dict.__delitem__(self, elem) - - def __repr__(self): - if not self: - return '%s()' % self.__class__.__name__ - items = ', '.join(map('%r: %r'.__mod__, self.most_common())) - return '%s({%s})' % (self.__class__.__name__, items) - - # Multiset-style mathematical operations discussed in: - # Knuth TAOCP Volume II section 4.6.3 exercise 19 - # and at http://en.wikipedia.org/wiki/Multiset - # - # Outputs guaranteed to only include positive counts. - # - # To strip negative and zero counts, add-in an empty counter: - # c += Counter() - - def __add__(self, other): - '''Add counts from two counters. - - >>> Counter('abbb') + Counter('bcc') - Counter({'b': 4, 'c': 2, 'a': 1}) - - - ''' - if not isinstance(other, Counter): - return NotImplemented - result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] + other[elem] - if newcount > 0: - result[elem] = newcount - return result - - def __sub__(self, other): - ''' Subtract count, but keep only results with positive counts. - - >>> Counter('abbbc') - Counter('bccd') - Counter({'b': 2, 'a': 1}) - - ''' - if not isinstance(other, Counter): - return NotImplemented - result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] - other[elem] - if newcount > 0: - result[elem] = newcount - return result - - def __or__(self, other): - '''Union is the maximum of value in either of the input counters. - - >>> Counter('abbb') | Counter('bcc') - Counter({'b': 3, 'c': 2, 'a': 1}) - - ''' - if not isinstance(other, Counter): - return NotImplemented - _max = max - result = Counter() - for elem in set(self) | set(other): - newcount = _max(self[elem], other[elem]) - if newcount > 0: - result[elem] = newcount - return result - - def __and__(self, other): - ''' Intersection is the minimum of corresponding counts. - - >>> Counter('abbb') & Counter('bcc') - Counter({'b': 1}) - - ''' - if not isinstance(other, Counter): - return NotImplemented - _min = min - result = Counter() - if len(self) < len(other): - self, other = other, self - for elem in ifilter(self.__contains__, other): - newcount = _min(self[elem], other[elem]) - if newcount > 0: - result[elem] = newcount - return result - - -if __name__ == '__main__': - import doctest - print(doctest.testmod()) diff --git a/mycli/packages/ordereddict.py b/mycli/packages/ordereddict.py deleted file mode 100644 index 5b0303f5a..000000000 --- a/mycli/packages/ordereddict.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (c) 2009 Raymond Hettinger -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -from UserDict import DictMixin - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - if last: - key = reversed(self).next() - else: - key = iter(self).next() - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - if len(self) != len(other): - return False - for p, q in zip(self.items(), other.items()): - if p != q: - return False - return True - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index f5fc3c903..41364756c 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -7,12 +7,7 @@ from .packages.special.favoritequeries import favoritequeries from re import compile, escape from .packages.tabulate import table_formats - -try: - from collections import Counter -except ImportError: - # python 2.6 - from .packages.counter import Counter +from collections import Counter _logger = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index 54a2404d3..5c70b9e3e 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,6 @@ 'License :: OSI Approved :: BSD License', 'Operating System :: Unix', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', diff --git a/tox.ini b/tox.ini index d3f9aef14..8d578d8fa 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34, py35, py36 +envlist = py27, py33, py34, py35, py36 [testenv] deps = pytest mock From 0daa21b90419da9fa4e04c7e6fd92be878c8e81f Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 6 Mar 2017 20:25:41 +0100 Subject: [PATCH 0274/1025] tee special command --- mycli/main.py | 2 ++ mycli/packages/special/iocommands.py | 34 ++++++++++++++++++++++++++++ tests/test_special_iocommands.py | 18 +++++++++++++++ tests/test_sqlexecute.py | 2 +- 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 6bb91b755..e83e7eba8 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -592,6 +592,7 @@ def one_iteration(document=None): self.output(str(e), err=True, fg='red') else: try: + special.write_tee(output) if special.is_pager_enabled(): self.output_via_pager('\n'.join(output)) else: @@ -649,6 +650,7 @@ def one_iteration(document=None): while True: one_iteration() except EOFError: + special.close_tee() if not self.less_chatty: self.output('Goodbye!') diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 415e33d3e..c5f7f8f4d 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -16,6 +16,7 @@ TIMING_ENABLED = False use_expanded_output = False PAGER_ENABLED = True +tee_file = None @export def set_timing_enabled(val): @@ -240,3 +241,36 @@ def execute_system_command(arg, **_): return [(None, None, None, response)] except OSError as e: return [(None, None, None, 'OSError: %s' % e.strerror)] + +@special_command('tee', 'tee [-o] filename', 'write to a output file (optionally override using -o)') +def set_tee(arg, **_): + global tee_file + if arg.startswith('-o '): + mode = "w" + filename = arg[3:] + else: + mode = 'a' + filename = arg + tee_file = open(filename, mode) + return [(None, None, None, "")] + +@export +def close_tee(): + global tee_file + if tee_file: + tee_file.close() + tee_file = None + +@special_command('notee', 'notee', 'stop writing to a output file') +def no_tee(arg, **_): + close_tee() + return [(None, None, None, "")] + +@export +def write_tee(output): + global tee_file + if tee_file: + for buf in output: + tee_file.write(buf) + tee_file.write(u"\n") + tee_file.flush() diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py index 5dbbc1602..98e4cc2f7 100644 --- a/tests/test_special_iocommands.py +++ b/tests/test_special_iocommands.py @@ -1,4 +1,5 @@ import mycli.packages.special +import tempfile import os def test_set_get_pager(): mycli.packages.special.set_pager_enabled(True) @@ -35,3 +36,20 @@ def test_editor_command(): os.environ['EDITOR'] = 'true' mycli.packages.special.open_external_editor(r'select 1') == "select 1" + +def test_tee_command(): + mycli.packages.special.write_tee([u"hello world"]) # write without file set + with tempfile.NamedTemporaryFile() as f: + mycli.packages.special.execute(None, u"tee "+f.name) + mycli.packages.special.write_tee([u"hello world"]) + assert f.read() == b"hello world\n" + + mycli.packages.special.execute(None, u"tee -o "+f.name) + mycli.packages.special.write_tee([u"hello world"]) + f.seek(0) + assert f.read() == b"hello world\n" + + mycli.packages.special.execute(None, u"notee") + mycli.packages.special.write_tee([u"hello world"]) + f.seek(0) + assert f.read() == b"hello world\n" diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index d9ed62532..903e8e6d8 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -213,7 +213,7 @@ def test_favorite_query_expanded_output(executor): @dbtest def test_special_command(executor): results = run(executor, '\\?') - expected_line = u'| help | \\? | Show this help. |\n' + expected_line = u'\n| help' assert len(results) == 1 assert expected_line in results[0] From ab2982ac22254b0a22b942e5abd11e361135cff8 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 6 Mar 2017 21:08:02 +0100 Subject: [PATCH 0275/1025] add queries to tee output --- mycli/main.py | 4 +++- mycli/packages/special/iocommands.py | 5 ++--- tests/test_special_iocommands.py | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index e83e7eba8..c79382fa4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -505,6 +505,7 @@ def one_iteration(document=None): try: logger.debug('sql: %r', document.text) + special.write_tee(self.get_prompt(self.prompt_format) + document.text) if self.logfile: self.logfile.write('\n# %s\n' % datetime.now()) self.logfile.write(document.text) @@ -592,7 +593,7 @@ def one_iteration(document=None): self.output(str(e), err=True, fg='red') else: try: - special.write_tee(output) + special.write_tee('\n'.join(output)) if special.is_pager_enabled(): self.output_via_pager('\n'.join(output)) else: @@ -655,6 +656,7 @@ def one_iteration(document=None): self.output('Goodbye!') def output(self, text, **kwargs): + special.write_tee(text) if self.logfile: self.logfile.write(utf8tounicode(text)) self.logfile.write('\n') diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index c5f7f8f4d..9582a574d 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -270,7 +270,6 @@ def no_tee(arg, **_): def write_tee(output): global tee_file if tee_file: - for buf in output: - tee_file.write(buf) - tee_file.write(u"\n") + tee_file.write(output) + tee_file.write(u"\n") tee_file.flush() diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py index 98e4cc2f7..78d29b0d3 100644 --- a/tests/test_special_iocommands.py +++ b/tests/test_special_iocommands.py @@ -38,18 +38,18 @@ def test_editor_command(): mycli.packages.special.open_external_editor(r'select 1') == "select 1" def test_tee_command(): - mycli.packages.special.write_tee([u"hello world"]) # write without file set + mycli.packages.special.write_tee(u"hello world") # write without file set with tempfile.NamedTemporaryFile() as f: mycli.packages.special.execute(None, u"tee "+f.name) - mycli.packages.special.write_tee([u"hello world"]) + mycli.packages.special.write_tee(u"hello world") assert f.read() == b"hello world\n" mycli.packages.special.execute(None, u"tee -o "+f.name) - mycli.packages.special.write_tee([u"hello world"]) + mycli.packages.special.write_tee(u"hello world") f.seek(0) assert f.read() == b"hello world\n" mycli.packages.special.execute(None, u"notee") - mycli.packages.special.write_tee([u"hello world"]) + mycli.packages.special.write_tee(u"hello world") f.seek(0) assert f.read() == b"hello world\n" From e53981db0d5a8617260761c11e5c9f38d8f4faa0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 7 Mar 2017 08:57:05 -0600 Subject: [PATCH 0276/1025] Add error messages for tee command. --- mycli/packages/special/iocommands.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 9582a574d..4d8206432 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -251,7 +251,15 @@ def set_tee(arg, **_): else: mode = 'a' filename = arg - tee_file = open(filename, mode) + + if not filename: + raise TypeError('You must provide a filename.') + + try: + tee_file = open(filename, mode) + except OSError as e: + raise OSError("Cannot write to file '{}': {}".format(e.filename, e.strerror)) + return [(None, None, None, "")] @export From d26f3d745459f8da7e62ca14a5840dcef396c730 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 7 Mar 2017 08:58:11 -0600 Subject: [PATCH 0277/1025] Grammar fix. --- mycli/packages/special/iocommands.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 4d8206432..650b4ff93 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -242,7 +242,8 @@ def execute_system_command(arg, **_): except OSError as e: return [(None, None, None, 'OSError: %s' % e.strerror)] -@special_command('tee', 'tee [-o] filename', 'write to a output file (optionally override using -o)') +@special_command('tee', 'tee [-o] filename', + 'write to an output file (optionally overwrite using -o)') def set_tee(arg, **_): global tee_file if arg.startswith('-o '): @@ -269,7 +270,7 @@ def close_tee(): tee_file.close() tee_file = None -@special_command('notee', 'notee', 'stop writing to a output file') +@special_command('notee', 'notee', 'stop writing to an output file') def no_tee(arg, **_): close_tee() return [(None, None, None, "")] From 7266570b1c327ee227064f1b1a75216feba1a09b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 7 Mar 2017 09:17:46 -0600 Subject: [PATCH 0278/1025] Add tee tests for raised errors. --- tests/test_special_iocommands.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py index 78d29b0d3..eddfbf8ac 100644 --- a/tests/test_special_iocommands.py +++ b/tests/test_special_iocommands.py @@ -1,6 +1,12 @@ -import mycli.packages.special -import tempfile import os +import stat +import tempfile + +import pytest + +import mycli.packages.special + + def test_set_get_pager(): mycli.packages.special.set_pager_enabled(True) assert mycli.packages.special.is_pager_enabled() @@ -53,3 +59,12 @@ def test_tee_command(): mycli.packages.special.write_tee(u"hello world") f.seek(0) assert f.read() == b"hello world\n" + +def test_tee_command_error(): + with pytest.raises(TypeError): + mycli.packages.special.execute(None, 'tee') + + with pytest.raises(OSError): + with tempfile.NamedTemporaryFile() as f: + os.chmod(f.name, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + mycli.packages.special.execute(None, 'tee {}'.format(f.name)) From 05190ec683b124f4da13c9c31a4c6b4669f2b9e0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 7 Mar 2017 09:23:01 -0600 Subject: [PATCH 0279/1025] Catch Python2 tee error. --- mycli/packages/special/iocommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 650b4ff93..4d4fcc655 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -258,7 +258,7 @@ def set_tee(arg, **_): try: tee_file = open(filename, mode) - except OSError as e: + except (IOError, OSError) as e: raise OSError("Cannot write to file '{}': {}".format(e.filename, e.strerror)) return [(None, None, None, "")] From 71cb83899b3ceaa88602445c27a7d117cec058f5 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 7 Mar 2017 20:14:30 +0100 Subject: [PATCH 0280/1025] Update Detailed Install Instructions for Fedora --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6b682a2c4..6ecca526c 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,17 @@ Twitter: [@amjithr](http://twitter.com/amjithr) ## Detailed Install Instructions: -### RHEL, Centos, Fedora: +### Fedora -I haven't built an RPM package for mycli yet. So please use `pip` to install `mycli`. You can install pip on your system using: +Fedora has a package available for mycli, install it using dnf: + +``` +$ sudo dnf install mycli +``` + +### RHEL, Centos + +I haven't built an RPM package for mycli for RHEL or Centos yet. So please use `pip` to install `mycli`. You can install pip on your system using: ``` $ sudo yum install python-pip From 20f743c3b1353816124cd7ac3d0d18af335ca440 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 11 Mar 2017 13:37:08 +0100 Subject: [PATCH 0281/1025] pytest environment variables use environment variables for pytest for alternative host/port/etc. --- tests/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index dda72dddf..dd7725cc0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,9 +3,11 @@ from mycli.packages import connection from os import getenv -# TODO: should this be somehow be divined from environment? -USER, HOST, PORT, CHARSET = 'root', 'localhost', 3306, 'utf8' -PASSWORD = getenv('PASSWORD') +PASSWORD = getenv('PYTEST_PASSWORD') +USER = getenv('PYTEST_USER', 'root') +HOST = getenv('PYTEST_HOST', 'localhost') +PORT = getenv('PYTEST_PORT', 3306) +CHARSET = getenv('PYTEST_CHARSET', 'utf8') def db_connection(dbname=None): conn = connection.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, From 14c0bd2177d2ece01bd820f16b97e5d964ab15f2 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 11 Mar 2017 08:08:58 -0800 Subject: [PATCH 0282/1025] Update AUTHORS file. --- AUTHORS | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 316fa8979..66fdb7cb1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,12 +1,16 @@ -Many thanks to the following contributors. +Project Lead: +------------- + * Thomas Roten + Core Developers: ---------------- - * Thomas Roten * Iryna Cherniavska * Matheus Rosa * Darik Gamble + * Dick Marinus + * Amjith Ramanujam Contributors: ------------- From b78747c4517d9925804113d7719af462642ef9ff Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 17 Mar 2017 20:21:04 -0500 Subject: [PATCH 0283/1025] Add 1.9.0 changes. --- changelog.md | 71 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/changelog.md b/changelog.md index 2bd345e87..8ee9db42f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,38 @@ +1.9.0: +====== + +Features: +--------- + +* Add tee/notee commands for outputing results to a file. (Thanks: [Dick Marinus]). +* Add date, port, and whitespace options to prompt configuration. (Thanks: [Matheus Rosa]). +* Allow user to specify LESS pager flags. (Thanks: [John Sterling]). +* Add support for auto-reconnect. (Thanks: [Jialong Liu]). +* Add CSV batch output. (Thanks: [Matheus Rosa]). +* Add auto_vertical_output config to myclirc. (Thanks: [Matheus Rosa]). +* Improve Fedora install instructions. (Thanks: [Dick Marinus]). + +Bug Fixes: +---------- + +* Fix crashes occuring from commands starting with #. (Thanks: [Zhidong]). +* Fix broken PyMySQL link in README. (Thanks: [Daniël van Eeden]). +* Add various missing keywords for highlighting and autocompletion. (Thanks: [zer09]). +* Add the missing REGEXP keyword for highlighting and autocompletion. (Thanks: [cxbig]). +* Fix duplicate username entries in completion list. (Thanks: [John Sterling]). +* Remove extra spaces in TSV table format output. (Thanks: [Dick Marinus]). +* Kill running query when interrupted via Ctrl-C. (Thanks: [chainkite]). +* Read the smart_completion config from myclirc. (Thanks: [Thomas Roten]). + +Internal Changes: +----------------- + +* Improve handling of test database credentials. (Thanks: [Dick Marinus]). +* Add Python 3.6 to test environments and PyPI metadata. (Thanks: [Thomas Roten]). +* Drop Python 2.6 support. (Thanks: [Thomas Roten]). +* Swap pycrypto dependency for pycryptodome. (Thanks: [Michał Górny]). +* Bump sqlparse version so pgcli and mycli can be installed together. (Thanks: [darikg]). + 1.8.1: ====== @@ -60,7 +95,7 @@ Internal Changes: Features: --------- -* Change continuation prompt for multi-line mode to match default mysql. +* Change continuation prompt for multi-line mode to match default mysql. * Add `status` command to match mysql's `status` command. (Thanks: [Thomas Roten]). * Add SSL support for `mycli`. (Thanks: [Artem Bezsmertnyi]). * Add auto-completion and highlight support for OFFSET keyword. (Thanks: [Matheus Rosa]). @@ -102,7 +137,7 @@ Bug Fixes: Bug Fixes: ---------- -* Cast the value of port read from my.cnf to int. +* Cast the value of port read from my.cnf to int. 1.5.0: ====== @@ -115,7 +150,7 @@ Features: This feature is only available when `pycrypto` package is installed. * Register the special command `prompt` with the `\R` as alias. (Thanks: [Matheus Rosa]). Users can now change the mysql prompt at runtime using `prompt` command. - eg: + eg: ``` mycli> prompt \u@\h> Changed prompt format to \u@\h> @@ -162,9 +197,9 @@ Internal Changes: Features: --------- -* Add `source` command. This allows running sql statement from a file. +* Add `source` command. This allows running sql statement from a file. - eg: + eg: ``` mycli> source filename.sql ``` @@ -175,13 +210,13 @@ Features: disable the warning before running `DROP` commands. * Add completion support for CHANGE TO and other master/slave commands. This is - still preliminary and it will be enhanced in the future. + still preliminary and it will be enhanced in the future. -* Add custom styles to color the menus and toolbars. +* Add custom styles to color the menus and toolbars. -* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders]) +* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders]) - Multi-line queries are automatically indented. + Multi-line queries are automatically indented. Bug Fixes: ---------- @@ -196,7 +231,7 @@ Bug Fixes: Features: --------- * Add a new special command (\T) to change the table format on the fly. (Thanks: [Jonathan Bruno](https://github.com/brewneaux)) - eg: + eg: ``` mycli> \T tsv ``` @@ -217,7 +252,7 @@ Features: [clientamjith] user = 'amjith' database = 'user_management' - + $ mycli --defaults-group-suffix=amjith # uses the [clientamjith] section in my.cnf ``` @@ -225,7 +260,7 @@ Features: `my.cnf` to use at launch. This also makes it play nice with mysql sandbox. * Make `-p` and `--password` take the password in commandline. This makes mycli - a drop in replacement for mysql. + a drop in replacement for mysql. 1.2.0: ====== @@ -240,7 +275,7 @@ Features: Bug Fixes: --------- -* Prevent Ctrl-C from quitting mycli while the pager is active. +* Prevent Ctrl-C from quitting mycli while the pager is active. * Refresh auto-completions after the database is changed via a CONNECT command. Internal Changes: @@ -289,7 +324,7 @@ Features: * Customizable prompt. (Thanks [Steve Robbins](https://github.com/steverobbins)) * Make `\G` formatting to behave more like mysql. - + Bug Fixes: ---------- @@ -302,7 +337,7 @@ Bug Fixes: Features: --------- -* Upgrade prompt_toolkit to 0.38. This improves the performance of pasting long queries. +* Upgrade prompt_toolkit to 0.38. This improves the performance of pasting long queries. * Add support for reading my.cnf files. * Add editor command \e. * Replace ConfigParser with ConfigObj. @@ -327,12 +362,12 @@ Features: * Add support for connecting via socket. * Add completion for SQL functions. * Add completion support for SHOW statements. -* Made the timing of sql statements human friendly. +* Made the timing of sql statements human friendly. * Automatically prompt for a password if needed. Bug Fixes: ---------- -* Fixed the installation issues with PyMySQL dependency on case-sensitive file systems. +* Fixed the installation issues with PyMySQL dependency on case-sensitive file systems. [Daniel West]: http://github.com/danieljwest [Iryna Cherniavska]: https://github.com/j-bennet @@ -342,7 +377,7 @@ Bug Fixes: [Shoma Suzuki]: https://github.com/shoma [spacewander]: https://github.com/spacewander [Thomas Roten]: https://github.com/tsroten -[Artem Bezsmertnyi]: https://github.com/mrdeathless +[Artem Bezsmertnyi]: https://github.com/mrdeathless [Mikhail Borisov]: https://github.com/borman [Casper Langemeijer]: Casper Langemeijer [Lennart Weller]: https://github.com/lhw From be590c2473b31bc4d585aacc2547000fff5b028b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 17 Mar 2017 20:21:12 -0500 Subject: [PATCH 0284/1025] Add new contributor names. --- AUTHORS | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/AUTHORS b/AUTHORS index 66fdb7cb1..b5c88bdc6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -42,6 +42,14 @@ Contributors: * jbruno * mrdeathless * Abirami P + * John Sterling + * Jialong Liu + * Zhidong + * Daniël van Eeden + * zer09 + * cxbig + * chainkite + * Michał Górny Creator: -------- From 8aa055896d267ffbb27e87b818816a1ffaba3187 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Fri, 17 Mar 2017 20:21:26 -0500 Subject: [PATCH 0285/1025] Remove duplicate name. --- AUTHORS | 1 - 1 file changed, 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b5c88bdc6..2eb4af10b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -19,7 +19,6 @@ Contributors: * Shoma Suzuki * Daniel West * Scrappy Soft - * Dick Marinus * Daniel Black * Jonathan Bruno * Casper Langemeijer From d5b5e8280d778d57da4b7e659853d05cbf5da3d2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 13:28:37 -0500 Subject: [PATCH 0286/1025] Create PULL_REQUEST_TEMPLATE.md Adds a pull request template with sections for: - description - checklist --- .github/PULL_REQUEST_TEMPLATE.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..6ea05678c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +## Description + + + + +## Checklist + +- [ ] I've added this contribution to the `changelog.md`. +- [ ] I've added my name to the `AUTHORS` file (or it's already there). From b789fd30acf4b4063dbe4b257a3a0ac68e90e29d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 13:29:39 -0500 Subject: [PATCH 0287/1025] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6ea05678c..8d498abcd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,5 @@ ## Description - + From afa6a0f4e35802566b50c00bc1f146fe63b86ba3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 18:41:19 -0500 Subject: [PATCH 0288/1025] Use input for Python 3. --- release.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/release.py b/release.py index 622056274..845c83f8f 100644 --- a/release.py +++ b/release.py @@ -6,6 +6,11 @@ import sys from optparse import OptionParser +try: + input = raw_input +except NameError: + pass + DEBUG = False CONFIRM_STEPS = False DRY_RUN = False @@ -19,7 +24,7 @@ def skip_step(): global CONFIRM_STEPS if CONFIRM_STEPS: - choice = raw_input("--- Confirm step? (y/N) [y] ") + choice = input("--- Confirm step? (y/N) [y] ") if choice.lower() == 'n': return True return False @@ -87,7 +92,7 @@ def push_tags_to_github(): def checklist(questions): for question in questions: - choice = raw_input(question + ' (y/N) [n] ') + choice = input(question + ' (y/N) [n] ') if choice.lower() != 'y': sys.exit(1) @@ -119,7 +124,7 @@ def checklist(questions): CONFIRM_STEPS = popts.confirm_steps DRY_RUN = popts.dry_run - choice = raw_input('Are you sure? (y/N) [n] ') + choice = input('Are you sure? (y/N) [n] ') if choice.lower() != 'y': sys.exit(1) From 885b0b3643da479eed508dcc30ac76200972aeae Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 18:42:13 -0500 Subject: [PATCH 0289/1025] Add extra newlines for PEP8. --- release.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release.py b/release.py index 845c83f8f..4682b0517 100644 --- a/release.py +++ b/release.py @@ -75,9 +75,11 @@ def register_with_pypi(): def create_source_tarball(): run_step('python', 'setup.py', 'sdist') + def create_python_wheel(): run_step('python', 'setup.py', 'sdist', 'bdist_wheel') + def upload_source_tarball(): run_step('python', 'setup.py', 'sdist', 'upload') From a1eb31f2dc78a18172df29f86416c73d787e32f1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 18:57:28 -0500 Subject: [PATCH 0290/1025] Make the release script executable. --- release.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 release.py diff --git a/release.py b/release.py old mode 100644 new mode 100755 From 49c23eaadee84a62cf2b81f115c3be189945e755 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 20:07:00 -0500 Subject: [PATCH 0291/1025] Releasing version 1.9.0 --- mycli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/__init__.py b/mycli/__init__.py index e8b6b090b..e5102d301 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.8.1' +__version__ = '1.9.0' From 2016c6d5b4c42c6c2db9a79f19f78a6bd7052fed Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 20:35:30 -0500 Subject: [PATCH 0292/1025] Use twine for uploading to PyPI. --- release.py | 9 +++++---- requirements-dev.txt | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/release.py b/release.py index 622056274..37865b590 100644 --- a/release.py +++ b/release.py @@ -71,10 +71,11 @@ def create_source_tarball(): run_step('python', 'setup.py', 'sdist') def create_python_wheel(): - run_step('python', 'setup.py', 'sdist', 'bdist_wheel') + run_step('python', 'setup.py', 'bdist_wheel') -def upload_source_tarball(): - run_step('python', 'setup.py', 'sdist', 'upload') + +def upload_distribution_files(): + run_step('twine', 'upload', 'dist/*') def push_to_github(): @@ -130,4 +131,4 @@ def checklist(questions): create_python_wheel() push_to_github() push_tags_to_github() - upload_source_tarball() + upload_distribution_files() diff --git a/requirements-dev.txt b/requirements-dev.txt index ca4019c29..e54eabb9c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ mock pytest tox +twine==1.8.1 From 74d76991aa5e7d27b638654c278fb81e5ce063d0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 20:40:49 -0500 Subject: [PATCH 0293/1025] Add twine change to change log. --- changelog.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/changelog.md b/changelog.md index 8ee9db42f..f8573009d 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,12 @@ +TBD +=== + +Internal Changes: +----------------- + +* Upload mycli distributions in a safer manner (using twine). (Thanks: [Thomas + Roten]). + 1.9.0: ====== From 127eb6ead4d4399eaa641de633322b74f3cc6a34 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 18 Mar 2017 22:38:26 -0500 Subject: [PATCH 0294/1025] Merge the distribution functions. --- release.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/release.py b/release.py index 23316b324..9a5d399f9 100755 --- a/release.py +++ b/release.py @@ -72,12 +72,8 @@ def register_with_pypi(): run_step('python', 'setup.py', 'register') -def create_source_tarball(): - run_step('python', 'setup.py', 'sdist') - - -def create_python_wheel(): - run_step('python', 'setup.py', 'bdist_wheel') +def create_distribution_files(): + run_step('python', 'setup.py', 'sdist', 'bdist_wheel') def upload_distribution_files(): @@ -133,8 +129,7 @@ def checklist(questions): commit_for_release('mycli/__init__.py', ver) create_git_tag('v%s' % ver) register_with_pypi() - create_source_tarball() - create_python_wheel() + create_distribution_files() push_to_github() push_tags_to_github() upload_distribution_files() From 9a61fa165678184947e51a2e39a54409d01b12c3 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Mon, 20 Mar 2017 10:45:16 -0700 Subject: [PATCH 0295/1025] Fixes editor bug with prompt_toolkit 1.0.13. --- mycli/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index db38fa2bc..05456546c 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -422,6 +422,10 @@ def handle_editor_command(self, cli, document): :param document: Document :return: Document """ + # FIXME: using application.pre_run_callables like this here is not the best solution. + # It's internal api of prompt_toolkit that may change. This was added to fix + # https://github.com/dbcli/pgcli/issues/668. We may find a better way to do it in the future. + saved_callables = cli.application.pre_run_callables while special.editor_command(document.text): filename = special.get_filename(document.text) sql, message = special.open_external_editor(filename, @@ -430,8 +434,10 @@ def handle_editor_command(self, cli, document): # Something went wrong. Raise an exception and bail. raise RuntimeError(message) cli.current_buffer.document = Document(sql, cursor_position=len(sql)) - document = cli.run(False) + cli.application.pre_run_callables = [] + document = cli.run() continue + cli.application.pre_run_callables = saved_callables return document def run_cli(self): @@ -464,7 +470,7 @@ def get_continuation_tokens(cli, width): def one_iteration(document=None): if document is None: - document = self.cli.run(reset_current_buffer=True) + document = self.cli.run() special.set_expanded_output(False) From 86d0f9e916439cd5fc1ad5bf6a7f92a06b346f1f Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Mon, 20 Mar 2017 10:49:53 -0700 Subject: [PATCH 0296/1025] Added fix to changelog. --- changelog.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index f8573009d..d6f4ec96a 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Bug Fixes: +---------- + +* Fix external editor bug (issue #377). (Thanks: [Irina Truong]). + Internal Changes: ----------------- @@ -194,9 +199,9 @@ Bug Fixes: Internal Changes: ----------------- -* Make pycrypto optional and only install it in \*nix systems. (Thanks: [Iryna Cherniavska]). +* Make pycrypto optional and only install it in \*nix systems. (Thanks: [Irina Truong]). * Add badge for PyPI version to README. (Thanks: [Shoma Suzuki]). -* Updated release script with a --dry-run and --confirm-steps option. (Thanks: [Iryna Cherniavska]). +* Updated release script with a --dry-run and --confirm-steps option. (Thanks: [Irina Truong]). * Adds support for PyMySQL 0.6.2 and above. This is useful for debian package builders. (Thanks: [Thomas Roten]). * Disable click warning. @@ -245,7 +250,7 @@ Features: mycli> \T tsv ``` * Add `--defaults-group-suffix` to the command line. This lets the user specify - a group to use in the my.cnf files. (Thanks: [Iryna Cherniavska](http://github.com/j-bennet)) + a group to use in the my.cnf files. (Thanks: [Irina Truong](http://github.com/j-bennet)) In the my.cnf file a user can specify credentials for different databases and invoke mycli with the group name to use the appropriate credentials. @@ -310,7 +315,7 @@ Features: * Fuzzy completion is now case-insensitive. (Thanks: [bjarnagin](https://github.com/bjarnagin)) * Added new-line (`\n`) to the list of special characters to use in prompt. (Thanks: [brewneaux](https://github.com/brewneaux)) -* Honor the `pager` setting in my.cnf files. (Thanks: [Iryna Cherniavska](http://github.com/j-bennet)) +* Honor the `pager` setting in my.cnf files. (Thanks: [Irina Truong](http://github.com/j-bennet)) Bug Fixes: ---------- @@ -379,7 +384,7 @@ Bug Fixes: * Fixed the installation issues with PyMySQL dependency on case-sensitive file systems. [Daniel West]: http://github.com/danieljwest -[Iryna Cherniavska]: https://github.com/j-bennet +[Irina Truong]: https://github.com/j-bennet [Kacper Kwapisz]: https://github.com/KKKas [Martijn Engler]: https://github.com/martijnengler [Matheus Rosa]: https://github.com/mdsrosa From ba0f5bffefcc5767b84a08e5b691c6f4c4dddea2 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Tue, 21 Mar 2017 21:21:37 -0700 Subject: [PATCH 0297/1025] Bumped minimal requirement of prompt-toolkit to 1.0.10, where reset_current_buffer was deprecated. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5c70b9e3e..5c08d1b59 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ install_requirements = [ 'click >= 4.1', 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? - 'prompt_toolkit>=1.0.0,<1.1.0', + 'prompt_toolkit>=1.0.10,<1.1.0', 'PyMySQL >= 0.6.2', 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.6', From 29f28e414aac665c3185221480db61955df53e31 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 21 Mar 2017 23:22:51 -0500 Subject: [PATCH 0298/1025] Open config file using UTF8 encoding. --- mycli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/config.py b/mycli/config.py index 97ead1fcd..df669fe24 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -40,7 +40,7 @@ def read_config_file(f): f = os.path.expanduser(f) try: - config = ConfigObj(f, interpolation=False) + config = ConfigObj(f, interpolation=False, encoding='utf8') except ConfigObjError as e: log(logger, logging.ERROR, "Unable to parse line {0} of config file " "'{1}'.".format(e.line_number, f)) From f44d7453e83c04251651a63f08fab522821569ca Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 21 Mar 2017 23:25:24 -0500 Subject: [PATCH 0299/1025] Add config utf-8 change to change log. --- changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index d6f4ec96a..1ae1a13f0 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,8 @@ Bug Fixes: ---------- * Fix external editor bug (issue #377). (Thanks: [Irina Truong]). +* Fixed bug so that favorite queries can include unicode characters. (Thanks: + [Thomas Roten]). Internal Changes: ----------------- From f0694e01421852adf66eff05863036cfb2edda48 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Wed, 22 Mar 2017 07:33:56 +0100 Subject: [PATCH 0300/1025] Add unit test for unicode in favorite queries --- tests/test_special_iocommands.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_special_iocommands.py b/tests/test_special_iocommands.py index eddfbf8ac..2ed2dbad8 100644 --- a/tests/test_special_iocommands.py +++ b/tests/test_special_iocommands.py @@ -1,3 +1,4 @@ +# coding: utf-8 import os import stat import tempfile @@ -5,6 +6,7 @@ import pytest import mycli.packages.special +import utils def test_set_get_pager(): @@ -68,3 +70,9 @@ def test_tee_command_error(): with tempfile.NamedTemporaryFile() as f: os.chmod(f.name, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) mycli.packages.special.execute(None, 'tee {}'.format(f.name)) + +def test_favorite_query(): + with utils.db_connection().cursor() as cur: + query = u'select "✔"' + mycli.packages.special.execute(cur, u'\\fs check {0}'.format(query)) + assert next(mycli.packages.special.execute(cur, u'\\f check'))[0] == "> " + query From 3c95e185a4f51895a3079b6799bd0d73c183f3f3 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Wed, 22 Mar 2017 17:02:03 +0100 Subject: [PATCH 0301/1025] Fix requirements Pygments lowered to >= 1.6 PyMySQL raised to >= 0.6.7 configobj lowered to 5.0.5 pycryptodome set minimum to 3 --- .travis.yml | 5 +--- changelog.md | 19 +++++++------- mycli/config.py | 13 +-------- mycli/main.py | 29 ++++++-------------- mycli/packages/connection.py | 51 ------------------------------------ mycli/sqlexecute.py | 6 ++--- setup.py | 8 +++--- tests/test_config.py | 11 ++------ tests/test_sqlexecute.py | 4 --- tests/utils.py | 6 ++--- 10 files changed, 32 insertions(+), 120 deletions(-) delete mode 100644 mycli/packages/connection.py diff --git a/.travis.yml b/.travis.yml index 516df2604..1a7991836 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,8 @@ python: - "3.5" - "3.6" -env: - - PYMYSQL_VERSION=0.6.7 - install: - - pip install PyMySQL==$PYMYSQL_VERSION . pytest mock codecov + - pip install PyMySQL . pytest mock codecov script: - coverage run --source mycli -m py.test diff --git a/changelog.md b/changelog.md index 1ae1a13f0..ae59ec7ac 100644 --- a/changelog.md +++ b/changelog.md @@ -7,6 +7,7 @@ Bug Fixes: * Fix external editor bug (issue #377). (Thanks: [Irina Truong]). * Fixed bug so that favorite queries can include unicode characters. (Thanks: [Thomas Roten]). +* Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) Internal Changes: ----------------- @@ -25,7 +26,7 @@ Features: * Allow user to specify LESS pager flags. (Thanks: [John Sterling]). * Add support for auto-reconnect. (Thanks: [Jialong Liu]). * Add CSV batch output. (Thanks: [Matheus Rosa]). -* Add auto_vertical_output config to myclirc. (Thanks: [Matheus Rosa]). +* Add `auto_vertical_output` config to myclirc. (Thanks: [Matheus Rosa]). * Improve Fedora install instructions. (Thanks: [Dick Marinus]). Bug Fixes: @@ -38,7 +39,7 @@ Bug Fixes: * Fix duplicate username entries in completion list. (Thanks: [John Sterling]). * Remove extra spaces in TSV table format output. (Thanks: [Dick Marinus]). * Kill running query when interrupted via Ctrl-C. (Thanks: [chainkite]). -* Read the smart_completion config from myclirc. (Thanks: [Thomas Roten]). +* Read the `smart_completion` config from myclirc. (Thanks: [Thomas Roten]). Internal Changes: ----------------- @@ -72,7 +73,7 @@ Features: * Add support for --execute/-e commandline arg. (Thanks: [Matheus Rosa]). * Add `less_chatty` config option to skip the intro messages. (Thanks: [Scrappy Soft]). -* Support MYCLI_HISTFILE environment variable to specify where to write the history file. (Thanks: [Scrappy Soft]). +* Support `MYCLI_HISTFILE` environment variable to specify where to write the history file. (Thanks: [Scrappy Soft]). * Add `prompt_continuation` config option to allow configuring the continuation prompt for multi-line queries. (Thanks: [Scrappy Soft]). * Display login-path instead of host in prompt. (Thanks: [Irina Truong]). @@ -103,7 +104,7 @@ Bug Fixes: Internal Changes: ----------------- -* Upgrade prompt_toolkit to 1.0.0. (Thanks: [Jonathan Slenders]) +* Upgrade `prompt_toolkit` to 1.0.0. (Thanks: [Jonathan Slenders]) 1.6.0: ====== @@ -131,7 +132,7 @@ Bug Fixes: Internal Changes: ----------------- -* Upgrade prompt_toolkit to 0.60. +* Upgrade `prompt_toolkit` to 0.60. * Add Python 3.5 to test environments. (Thanks: [Thomas Roten]). * Remove license meta-data. (Thanks: [Thomas Roten]). * Skip binary tests if PyMySQL version does not support it. (Thanks: [Thomas Roten]). @@ -230,7 +231,7 @@ Features: * Add custom styles to color the menus and toolbars. -* Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders]) +* Upgrade `prompt_toolkit` to 0.46. (Thanks: [Jonathan Slenders]) Multi-line queries are automatically indented. @@ -297,7 +298,7 @@ Bug Fixes: Internal Changes: ----------------- -* Upgrade prompt_toolkit dependency version to 0.45. +* Upgrade `prompt_toolkit` dependency version to 0.45. * Added Travis CI to run the tests automatically. 1.1.1: @@ -329,7 +330,7 @@ Internal Changes: ----------------- * Changed pymysql version to be greater than 0.6.6. -* Upgrade prompt_toolkit version to 0.42. (Thanks: [Yasuhiro Matsumoto](https://github.com/mattn)) +* Upgrade `prompt_toolkit` version to 0.42. (Thanks: [Yasuhiro Matsumoto](https://github.com/mattn)) * Removed the explicit dependency on six. 2015/06/10: @@ -353,7 +354,7 @@ Bug Fixes: Features: --------- -* Upgrade prompt_toolkit to 0.38. This improves the performance of pasting long queries. +* Upgrade `prompt_toolkit` to 0.38. This improves the performance of pasting long queries. * Add support for reading my.cnf files. * Add editor command \e. * Replace ConfigParser with ConfigObj. diff --git a/mycli/config.py b/mycli/config.py index df669fe24..7f5e0cb25 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -11,17 +11,8 @@ basestring except NameError: basestring = str -try: - from Crypto.Cipher import AES -except ImportError: - AES = None - +from Crypto.Cipher import AES -class CryptoError(Exception): - """ - Exception to signal about pycrypto(dome) not available. - """ - pass logger = logging.getLogger(__name__) @@ -125,8 +116,6 @@ def read_and_decrypt_mylogin_cnf(f): :return: the decrypted login path file :rtype: io.BytesIO or None """ - if AES is None: - raise CryptoError('pycrypto(dome) is not available.') # Number of bytes used to store the length of ciphertext. MAX_CIPHER_STORE_LEN = 4 diff --git a/mycli/main.py b/mycli/main.py index 05456546c..04bd26462 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -46,7 +46,7 @@ from .clibuffer import CLIBuffer from .completion_refresher import CompletionRefresher from .config import (write_default_config, get_mylogin_cnf_path, - open_mylogin_cnf, CryptoError, read_config_file, + open_mylogin_cnf, read_config_file, read_config_files, str_to_bool) from .key_bindings import mycli_bindings from .encodingutils import utf8tounicode @@ -165,17 +165,13 @@ def __init__(self, sqlexecute=None, prompt=None, # Load .mylogin.cnf if it exists. mylogin_cnf_path = get_mylogin_cnf_path() if mylogin_cnf_path: - try: - mylogin_cnf = open_mylogin_cnf(mylogin_cnf_path) - if mylogin_cnf_path and mylogin_cnf: - # .mylogin.cnf gets read last, even if defaults_file is specified. - self.cnf_files.append(mylogin_cnf) - elif mylogin_cnf_path and not mylogin_cnf: - # There was an error reading the login path file. - print('Error: Unable to read login path file.') - except CryptoError: - click.secho('Warning: .mylogin.cnf was not read: pycrypto(dome) ' - 'module is not available.') + mylogin_cnf = open_mylogin_cnf(mylogin_cnf_path) + if mylogin_cnf_path and mylogin_cnf: + # .mylogin.cnf gets read last, even if defaults_file is specified. + self.cnf_files.append(mylogin_cnf) + elif mylogin_cnf_path and not mylogin_cnf: + # There was an error reading the login path file. + print('Error: Unable to read login path file.') self.cli = None @@ -545,15 +541,6 @@ def one_iteration(document=None): end = time() total += end - start mutating = mutating or is_mutating(status) - except UnicodeDecodeError as e: - import pymysql - if pymysql.VERSION < (0, 6, 7): - message = ('You are running an older version of pymysql.\n' - 'Please upgrade to 0.6.7 or above to view binary data.\n' - 'Try \'pip install -U pymysql\'.') - self.output(message) - else: - raise e except KeyboardInterrupt: # get last connection id connection_id_to_kill = sqlexecute.connection_id diff --git a/mycli/packages/connection.py b/mycli/packages/connection.py deleted file mode 100644 index 5ece3f9eb..000000000 --- a/mycli/packages/connection.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Connection and cursor wrappers around PyMySQL. - -This module effectively backports PyMySQL functionality and error handling -so that mycli will support Debian's python-pymysql version (0.6.2). -""" - -import pymysql - -Cursor = pymysql.cursors.Cursor -connect = pymysql.connect - - -if pymysql.VERSION[1] == 6 and pymysql.VERSION[2] < 5: - class Cursor(pymysql.cursors.Cursor): - """Makes Cursor a context manager in PyMySQL < 0.6.5.""" - - def __enter__(self): - return self - - def __exit__(self, *exc_info): - del exc_info - self.close() - - -if pymysql.VERSION[1] == 6 and pymysql.VERSION[2] < 3: - class Connection(pymysql.connections.Connection): - """Adds error handling to Connection in PyMySQL < 0.6.3.""" - - def __del__(self): - if self.socket: - try: - self.socket.close() - except: - pass - self.socket = None - self._rfile = None - - def connect(*args, **kwargs): - """Makes connect() use our custom Connection class. - - PyMySQL < 0.6.3 uses the *passwd* argument instead of *password*. This - function renames that keyword or assigns it the default value of '', - which is the same default value PyMySQL gives it. - - See pymysql.connections.Connection.__init__() for more information - about calling this function. - """ - - kwargs['passwd'] = kwargs.pop('password', '') - - return Connection(*args, **kwargs) diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index b28008e82..786c8f0ec 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -1,7 +1,7 @@ import logging import pymysql import sqlparse -from .packages import connection, special +from .packages import special from pymysql.constants import FIELD_TYPE from pymysql.converters import (convert_mysql_timestamp, convert_datetime, convert_timedelta, convert_date) @@ -72,11 +72,11 @@ def connect(self, database=None, user=None, password=None, host=None, FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), } - conn = connection.connect(database=db, user=user, password=password, + conn = pymysql.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, use_unicode=True, charset=charset, autocommit=True, client_flag=pymysql.constants.CLIENT.INTERACTIVE, - cursorclass=connection.Cursor, local_infile=local_infile, + local_infile=local_infile, conv=conv, ssl=ssl) if hasattr(self, 'conn'): self.conn.close() diff --git a/setup.py b/setup.py index 5c08d1b59..82cdef6e1 100644 --- a/setup.py +++ b/setup.py @@ -13,12 +13,12 @@ install_requirements = [ 'click >= 4.1', - 'Pygments >= 2.0', # Pygments has to be Capitalcased. WTF? + 'Pygments >= 1.6', 'prompt_toolkit>=1.0.10,<1.1.0', - 'PyMySQL >= 0.6.2', + 'PyMySQL >= 0.6.7', 'sqlparse>=0.2.2,<0.3.0', - 'configobj >= 5.0.6', - 'pycryptodome', + 'configobj >= 5.0.5', + 'pycryptodome >= 3', ] setup( diff --git a/tests/test_config.py b/tests/test_config.py index 7879dd109..2a0d26c18 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,9 +7,8 @@ import tempfile import pytest -from mycli.config import (CryptoError, get_mylogin_cnf_path, - open_mylogin_cnf, read_and_decrypt_mylogin_cnf, - str_to_bool) +from mycli.config import (get_mylogin_cnf_path, open_mylogin_cnf, + read_and_decrypt_mylogin_cnf, str_to_bool) with_pycryptodome = ['pycryptodome' in set([package.project_name for package in pip.get_installed_distributions()])] @@ -26,12 +25,6 @@ def open_bmylogin_cnf(name): return buf -@pytest.mark.skipif(with_pycryptodome, reason='requires pycryptodome missing') -def test_read_mylogin_cnf_without_crypto(): - with pytest.raises(CryptoError): - mylogin_cnf = open_mylogin_cnf(LOGIN_PATH_FILE) - - @pytest.mark.skipif(not with_pycryptodome, reason='requires pycryptodome') def test_read_mylogin_cnf(): """Tests that a login path file can be read and decrypted.""" diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index 903e8e6d8..e9929a70f 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -7,8 +7,6 @@ from utils import run, dbtest, set_expanded_output -pymysql_support_binary = pymysql.VERSION >= (0, 6, 7) - @dbtest def test_conn(executor): run(executor, '''create table test(a text)''') @@ -36,7 +34,6 @@ def test_bools(executor): 1 row in set""") @dbtest -@pytest.mark.skipif(not pymysql_support_binary, reason='pymysql < 0.6.7') def test_binary(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') run(executor, '''INSERT INTO bt VALUES (GeomFromText('LINESTRING(116.37604 39.73979,116.375 39.73965)'));''') @@ -50,7 +47,6 @@ def test_binary(executor): 1 row in set""") @dbtest -@pytest.mark.skipif(not pymysql_support_binary, reason='pymysql < 0.6.7') def test_binary_expanded(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') run(executor, '''INSERT INTO bt VALUES (GeomFromText('LINESTRING(116.37604 39.73979,116.375 39.73965)'));''') diff --git a/tests/utils.py b/tests/utils.py index dd7725cc0..c76cb7ad3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,6 @@ import pytest +import pymysql from mycli.main import format_output, special -from mycli.packages import connection from os import getenv PASSWORD = getenv('PYTEST_PASSWORD') @@ -10,8 +10,8 @@ CHARSET = getenv('PYTEST_CHARSET', 'utf8') def db_connection(dbname=None): - conn = connection.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, - charset=CHARSET, cursorclass=connection.Cursor, + conn = pymysql.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, + charset=CHARSET, local_infile=False) conn.autocommit = True return conn From 59e1c003f9401ec51fdd0c4bd297bf0b3793dc51 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 23 Mar 2017 10:05:44 -0500 Subject: [PATCH 0302/1025] Open thanks files using UTF-8 encoding. --- changelog.md | 2 ++ mycli/main.py | 2 +- tests/test_main.py | 18 +++++++++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index ae59ec7ac..924d8efa4 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,8 @@ Bug Fixes: * Fixed bug so that favorite queries can include unicode characters. (Thanks: [Thomas Roten]). * Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) +* Fix bug where mycli would not start due to the thanks/credit intro text. + (Thanks: [Thomas Roten]). Internal Changes: ----------------- diff --git a/mycli/main.py b/mycli/main.py index 04bd26462..ba3443955 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -981,7 +981,7 @@ def quit_command(sql): def thanks_picker(files=()): for filename in files: - with open(filename) as f: + with open(filename, encoding='utf-8') as f: contents = f.readlines() return choice([x.split('*')[1].strip() for x in contents if x.startswith('*')]) diff --git a/tests/test_main.py b/tests/test_main.py index 49e8be3ff..14f1c10e2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,10 +1,18 @@ +import os + import click from click.testing import CliRunner from mycli.main import (cli, confirm_destructive_query, format_output, - is_destructive, query_starts_with, queries_start_with) + is_destructive, query_starts_with, queries_start_with, + thanks_picker, PACKAGE_ROOT) from utils import USER, HOST, PORT, PASSWORD, dbtest, run +try: + text_type = basestring +except NameError: + text_type = str + CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, '--password', PASSWORD, '_test_db'] @@ -171,3 +179,11 @@ def test_confirm_destructive_query_notty(executor): sql = 'drop database foo;' assert confirm_destructive_query(sql) is None + +def test_thanks_picker_utf8(): + project_root = os.path.dirname(PACKAGE_ROOT) + author_file = os.path.join(project_root, 'AUTHORS') + sponsor_file = os.path.join(project_root, 'SPONSORS') + + name = thanks_picker((author_file, sponsor_file)) + assert isinstance(name, text_type) From 6ada4d468a6b33e2f6af8a3c0af4fc71ca4cc603 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:27:32 -0500 Subject: [PATCH 0303/1025] Add basic output formatter class functionality. --- mycli/main.py | 111 ++-- mycli/myclirc | 2 +- mycli/packages/expanded.py | 2 +- mycli/packages/tabulate.py | 1078 ------------------------------------ mycli/sqlcompleter.py | 11 +- setup.py | 1 + tests/test_main.py | 28 +- tests/test_tabulate.py | 22 - tests/utils.py | 17 +- 9 files changed, 81 insertions(+), 1191 deletions(-) delete mode 100644 mycli/packages/tabulate.py delete mode 100644 tests/test_tabulate.py diff --git a/mycli/main.py b/mycli/main.py index ba3443955..b9a3cebe4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -35,8 +35,6 @@ from pygments.token import Token from configobj import ConfigObj, ConfigObjError -from .packages.tabulate import tabulate, table_formats -from .packages.expanded import expanded_table from .packages.special.main import (COMMANDS, NO_QUERY) import mycli.packages.special as special from .sqlcompleter import SQLCompleter @@ -49,6 +47,7 @@ open_mylogin_cnf, read_config_file, read_config_files, str_to_bool) from .key_bindings import mycli_bindings +from .output_formatter import OutputFormatter from .encodingutils import utf8tounicode from .lexer import MyCliLexer from .__init__ import __version__ @@ -144,6 +143,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.completion_refresher = CompletionRefresher() + self.formatter = OutputFormatter() + self.logger = logging.getLogger(__name__) self.initialize_logging() @@ -191,9 +192,9 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - if not arg in table_formats(): + if not arg in self.formatter.supported_formats(): msg = "Table type %s not yet implemented. Allowed types:" % arg - for table_type in table_formats(): + for table_type in self.formatter.supported_formats(): msg += "\n\t%s" % table_type yield (None, None, None, msg) else: @@ -533,9 +534,8 @@ def one_iteration(document=None): else: max_width = None - formatted = format_output(title, cur, headers, - status, self.table_format, - special.is_expanded_output(), max_width) + formatted = self.format_output(title, cur, headers, + status, special.is_expanded_output(), max_width) output.extend(formatted) end = time() @@ -715,15 +715,55 @@ def get_prompt(self, string): string = string.replace('\\_', ' ') return string - def run_query(self, query, table_format=None, new_line=True): + def run_query(self, query, new_line=True): """Runs query""" results = self.sqlexecute.run(query) for result in results: title, cur, headers, status = result - output = format_output(title, cur, headers, None, table_format) + output = self.format_output(title, cur, headers, None) for line in output: click.echo(line, nl=new_line) + def format_output(self, title, cur, headers, status, expanded=False, + max_width=None): + output = [] + if title: # Only print the title if it's not None. + output.append(title) + if cur: + headers = [utf8tounicode(x) for x in headers] + + if expanded: + output.append(self.formatter.format_output(cur, headers, + 'expanded')) + elif self.table_format == 'csv': + content = StringIO() + writer = csv.writer(content) + writer.writerow(headers) + + for row in cur: + row = ['null' if val is None else str(val) for val in row] + writer.writerow(row) + + output.append(content.getvalue()) + content.close() + else: + rows = list(cur) + formatted = self.formatter.format_output(rows, headers, + self.table_format) + if (self.table_format != 'expanded' and + max_width and rows and + content_exceeds_width(rows[0], max_width) and + headers): + output.append(self.formatter.format_output(cur, headers, + 'expanded')) + else: + output.append(formatted) + if status: # Only print the status if it's not None. + output.append(status) + + return output + + @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') @click.option('-P', '--port', envvar='MYSQL_TCP_PORT', type=int, help='Port number to use for connection. Honors ' @@ -818,12 +858,11 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - table_format = None - if table: - table_format = mycli.table_format - elif csv: + table_format = 'tsv' + if csv: table_format = 'csv' - mycli.run_query(execute, table_format=table_format) + mycli.table_format = table_format or mycli.table_format + mycli.run_query(execute) exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') @@ -844,58 +883,22 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - table_format = None + table_format = 'tsv' new_line = True if csv: table_format = 'csv' new_line = False - elif table: - table_format = mycli.table_format - mycli.run_query(stdin_text, table_format=table_format, new_line=new_line) + mycli.table_format = table_format or mycli.table_format + + mycli.run_query(stdin_text, new_line=new_line) exit(0) except Exception as e: click.secho(str(e), err=True, fg='red') exit(1) -def format_output(title, cur, headers, status, table_format, expanded=False, max_width=None): - output = [] - if title: # Only print the title if it's not None. - output.append(title) - if cur: - headers = [utf8tounicode(x) for x in headers] - table_format = 'tsv' if table_format is None else table_format - - if expanded: - output.append(expanded_table(cur, headers)) - elif table_format == 'csv': - content = StringIO() - writer = csv.writer(content) - writer.writerow(headers) - - for row in cur: - row = ['null' if val is None else str(val) for val in row] - writer.writerow(row) - - output.append(content.getvalue()) - content.close() - else: - rows = list(cur) - tabulated, frows = tabulate(rows, headers, tablefmt=table_format, - missingval='') - if (max_width and rows and - content_exceeds_width(frows[0], max_width) and - headers): - output.append(expanded_table(rows, headers)) - else: - output.append(tabulated) - if status: # Only print the status if it's not None. - output.append(status) - - return output - def content_exceeds_width(row, width): # Account for 3 characters between each column separator_space = (len(row)*3) diff --git a/mycli/myclirc b/mycli/myclirc index baf505106..ff6e8f734 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -66,7 +66,7 @@ less_chatty = False login_path_as_host = False # Cause result sets to be displayed vertically if they are too wide for the current window, -# and using normal tabular format otherwise. (This applies to statements terminated by ; or \G.) +# and using normal tabular format otherwise. (This applies to statements terminated by ; or \G.) auto_vertical_output = False # Custom colors for the completion menu, toolbar, etc. diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 128e9c690..245e4ca72 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,4 +1,4 @@ -from .tabulate import _text_type +from tabulate import _text_type import binascii def pad(field, total, char=u" "): diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py deleted file mode 100644 index fa9718267..000000000 --- a/mycli/packages/tabulate.py +++ /dev/null @@ -1,1078 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Pretty-print tabular data.""" - -from __future__ import print_function -from __future__ import unicode_literals -from collections import namedtuple -from decimal import Decimal -from platform import python_version_tuple -from wcwidth import wcswidth -import re -import binascii - - -if python_version_tuple()[0] < "3": - from itertools import izip_longest - from functools import partial - _none_type = type(None) - _int_type = int - _long_type = long - _float_type = float - _text_type = unicode - _binary_type = str - - def _is_file(f): - return isinstance(f, file) - -else: - from itertools import zip_longest as izip_longest - from functools import reduce, partial - _none_type = type(None) - _int_type = int - _long_type = int - _float_type = float - _text_type = str - _binary_type = bytes - - import io - def _is_file(f): - return isinstance(f, io.IOBase) - - -__all__ = ["tabulate", "tabulate_formats", "simple_separated_format"] -__version__ = "0.7.4" - - -MIN_PADDING = 2 - - -Line = namedtuple("Line", ["begin", "hline", "sep", "end"]) - - -DataRow = namedtuple("DataRow", ["begin", "sep", "end"]) - - -# A table structure is suppposed to be: -# -# --- lineabove --------- -# headerrow -# --- linebelowheader --- -# datarow -# --- linebewteenrows --- -# ... (more datarows) ... -# --- linebewteenrows --- -# last datarow -# --- linebelow --------- -# -# TableFormat's line* elements can be -# -# - either None, if the element is not used, -# - or a Line tuple, -# - or a function: [col_widths], [col_alignments] -> string. -# -# TableFormat's *row elements can be -# -# - either None, if the element is not used, -# - or a DataRow tuple, -# - or a function: [cell_values], [col_widths], [col_alignments] -> string. -# -# padding (an integer) is the amount of white space around data values. -# -# with_header_hide: -# -# - either None, to display all table elements unconditionally, -# - or a list of elements not to be displayed if the table has column headers. -# -TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", - "linebetweenrows", "linebelow", - "headerrow", "datarow", - "padding", "with_header_hide", "with_align"]) - - -def _pipe_segment_with_colons(align, colwidth): - """Return a segment of a horizontal line with optional colons which - indicate column's alignment (as in `pipe` output format).""" - w = colwidth - if align in ["right", "decimal"]: - return ('-' * (w - 1)) + ":" - elif align == "center": - return ":" + ('-' * (w - 2)) + ":" - elif align == "left": - return ":" + ('-' * (w - 1)) - else: - return '-' * w - - -def _pipe_line_with_colons(colwidths, colaligns): - """Return a horizontal line with optional colons to indicate column's - alignment (as in `pipe` output format).""" - segments = [_pipe_segment_with_colons(a, w) for a, w in zip(colaligns, colwidths)] - return "|" + "|".join(segments) + "|" - - -def _mediawiki_row_with_attrs(separator, cell_values, colwidths, colaligns): - alignment = { "left": '', - "right": 'align="right"| ', - "center": 'align="center"| ', - "decimal": 'align="right"| ' } - # hard-coded padding _around_ align attribute and value together - # rather than padding parameter which affects only the value - values_with_attrs = [' ' + alignment.get(a, '') + c + ' ' - for c, a in zip(cell_values, colaligns)] - colsep = separator*2 - return (separator + colsep.join(values_with_attrs)).rstrip() - - -def _html_row_with_attrs(celltag, cell_values, colwidths, colaligns): - alignment = { "left": '', - "right": ' style="text-align: right;"', - "center": ' style="text-align: center;"', - "decimal": ' style="text-align: right;"' } - values_with_attrs = ["<{0}{1}>{2}".format(celltag, alignment.get(a, ''), c) - for c, a in zip(cell_values, colaligns)] - return "" + "".join(values_with_attrs).rstrip() + "" - - -def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): - alignment = { "left": "l", "right": "r", "center": "c", "decimal": "r" } - tabular_columns_fmt = "".join([alignment.get(a, "l") for a in colaligns]) - return "\n".join(["\\begin{tabular}{" + tabular_columns_fmt + "}", - "\\toprule" if booktabs else "\hline"]) - -LATEX_ESCAPE_RULES = {r"&": r"\&", r"%": r"\%", r"$": r"\$", r"#": r"\#", - r"_": r"\_", r"^": r"\^{}", r"{": r"\{", r"}": r"\}", - r"~": r"\textasciitilde{}", "\\": r"\textbackslash{}", - r"<": r"\ensuremath{<}", r">": r"\ensuremath{>}"} - - -def _latex_row(cell_values, colwidths, colaligns): - def escape_char(c): - return LATEX_ESCAPE_RULES.get(c, c) - escaped_values = ["".join(map(escape_char, cell)) for cell in cell_values] - rowfmt = DataRow("", "&", "\\\\") - return _build_simple_row(escaped_values, rowfmt) - - -_table_formats = {"simple": - TableFormat(lineabove=Line("", "-", " ", ""), - linebelowheader=Line("", "-", " ", ""), - linebetweenrows=None, - linebelow=Line("", "-", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, - with_header_hide=["lineabove", "linebelow"], with_align=True), - "plain": - TableFormat(lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None, with_align=True), - "grid": - TableFormat(lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("+", "=", "+", "+"), - linebetweenrows=Line("+", "-", "+", "+"), - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None, with_align=True), - "fancy_grid": - TableFormat(lineabove=Line("╒", "═", "╤", "╕"), - linebelowheader=Line("╞", "═", "╪", "╡"), - linebetweenrows=Line("├", "─", "┼", "┤"), - linebelow=Line("╘", "═", "╧", "╛"), - headerrow=DataRow("│", "│", "│"), - datarow=DataRow("│", "│", "│"), - padding=1, with_header_hide=None, with_align=True), - "pipe": - TableFormat(lineabove=_pipe_line_with_colons, - linebelowheader=_pipe_line_with_colons, - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, - with_header_hide=["lineabove"], with_align=True), - "orgtbl": - TableFormat(lineabove=None, - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None, with_align=True), - "psql": - TableFormat(lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None, with_align=True), - "rst": - TableFormat(lineabove=Line("", "=", " ", ""), - linebelowheader=Line("", "=", " ", ""), - linebetweenrows=None, - linebelow=Line("", "=", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None, with_align=True), - "mediawiki": - TableFormat(lineabove=Line("{| class=\"wikitable\" style=\"text-align: left;\"", - "", "", "\n|+ \n|-"), - linebelowheader=Line("|-", "", "", ""), - linebetweenrows=Line("|-", "", "", ""), - linebelow=Line("|}", "", "", ""), - headerrow=partial(_mediawiki_row_with_attrs, "!"), - datarow=partial(_mediawiki_row_with_attrs, "|"), - padding=0, with_header_hide=None, with_align=True), - "html": - TableFormat(lineabove=Line("", "", "", ""), - linebelowheader=None, - linebetweenrows=None, - linebelow=Line("
", "", "", ""), - headerrow=partial(_html_row_with_attrs, "th"), - datarow=partial(_html_row_with_attrs, "td"), - padding=0, with_header_hide=None, with_align=False), - "latex": - TableFormat(lineabove=_latex_line_begin_tabular, - linebelowheader=Line("\\hline", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None, with_align=False), - "latex_booktabs": - TableFormat(lineabove=partial(_latex_line_begin_tabular, booktabs=True), - linebelowheader=Line("\\midrule", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None, with_align=False), - "tsv": - TableFormat(lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("", "\t", ""), - datarow=DataRow("", "\t", ""), - padding=0, with_header_hide=None, with_align=False)} - - -tabulate_formats = list(sorted(_table_formats.keys())) - - -_invisible_codes = re.compile(r"\x1b\[\d*m|\x1b\[\d*\;\d*\;\d*m") # ANSI color codes -_invisible_codes_bytes = re.compile(b"\x1b\[\d*m|\x1b\[\d*\;\d*\;\d*m") # ANSI color codes - - -def simple_separated_format(separator): - """Construct a simple TableFormat with columns separated by a separator. - - >>> tsv = simple_separated_format("\\t") ; \ - print(tabulate([["foo", 1], ["spam", 23]], tablefmt=tsv)[0].replace('\\t', r'\\t')) - foo\\t1 - spam\\t23 - """ - return TableFormat(None, None, None, None, - headerrow=DataRow('', separator, ''), - datarow=DataRow('', separator, ''), - padding=0, with_header_hide=None, - with_align=False) - - -def _isconvertible(conv, string): - try: - n = conv(string) - return True - except (ValueError, TypeError): - return False - - -def _isnumber(string): - """ - >>> _isnumber("123.45") - True - >>> _isnumber("123") - True - >>> _isnumber("spam") - False - """ - return _isconvertible(float, string) - - -def _isint(string): - """ - >>> _isint("123") - True - >>> _isint("123.45") - False - """ - return type(string) is _int_type or type(string) is _long_type or \ - (isinstance(string, _binary_type) or isinstance(string, _text_type)) and \ - _isconvertible(int, string) - - -def _type(string, has_invisible=True): - """The least generic type (type(None), int, float, str, unicode). - - >>> _type(None) is type(None) - True - >>> _type("foo") is type("") - True - >>> _type("1") is type(1) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - - """ - - if has_invisible and \ - (isinstance(string, _text_type) or isinstance(string, _binary_type)): - string = _strip_invisible(string) - - if string is None: - return _none_type - if isinstance(string, (bool, Decimal,)): - return _text_type - elif hasattr(string, "isoformat"): # datetime.datetime, date, and time - return _text_type - elif _isint(string): - return int - elif _isnumber(string): - return float - elif isinstance(string, _binary_type): - return _binary_type - else: - return _text_type - - -def _afterpoint(string): - """Symbols after a decimal point, -1 if the string lacks the decimal point. - - >>> _afterpoint("123.45") - 2 - >>> _afterpoint("1001") - -1 - >>> _afterpoint("eggs") - -1 - >>> _afterpoint("123e45") - 2 - - """ - if _isnumber(string): - if _isint(string): - return -1 - else: - pos = string.rfind(".") - pos = string.lower().rfind("e") if pos < 0 else pos - if pos >= 0: - return len(string) - pos - 1 - else: - return -1 # no point - else: - return -1 # not a number - - -def _padleft(width, s, has_invisible=True): - """Flush right. - - >>> _padleft(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430' - True - - """ - lwidth = width - wcswidth(_strip_invisible(s) if has_invisible else s) - return ' ' * lwidth + s - - -def _padright(width, s, has_invisible=True): - """Flush left. - - >>> _padright(6, '\u044f\u0439\u0446\u0430') == '\u044f\u0439\u0446\u0430 ' - True - - """ - rwidth = width - wcswidth(_strip_invisible(s) if has_invisible else s) - return s + ' ' * rwidth - - -def _padboth(width, s, has_invisible=True): - """Center string. - - >>> _padboth(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430 ' - True - - """ - xwidth = width - wcswidth(_strip_invisible(s) if has_invisible else s) - lwidth = xwidth // 2 - rwidth = 0 if xwidth <= 0 else lwidth + xwidth % 2 - return ' ' * lwidth + s + ' ' * rwidth - - -def _strip_invisible(s): - "Remove invisible ANSI color codes." - if isinstance(s, _text_type): - return re.sub(_invisible_codes, "", s) - else: # a bytestring - return re.sub(_invisible_codes_bytes, "", s) - - -def _visible_width(s): - """Visible width of a printed string. ANSI color codes are removed. - - >>> _visible_width('\x1b[31mhello\x1b[0m'), _visible_width("world") - (5, 5) - - """ - if isinstance(s, _text_type) or isinstance(s, _binary_type): - return wcswidth(_strip_invisible(s)) - else: - return wcswidth(_text_type(s)) - - -def _align_column(strings, alignment, minwidth=0, has_invisible=True): - """[string] -> [padded_string] - - >>> list(map(str,_align_column(["12.345", "-1234.5", "1.23", "1234.5", "1e+234", "1.0e234"], "decimal"))) - [' 12.345 ', '-1234.5 ', ' 1.23 ', ' 1234.5 ', ' 1e+234 ', ' 1.0e234'] - - >>> list(map(str,_align_column(['123.4', '56.7890'], None))) - ['123.4', '56.7890'] - - """ - if alignment == "right": - padfn = _padleft - elif alignment == "center": - padfn = _padboth - elif alignment == "decimal": - decimals = [_afterpoint(s) for s in strings] - maxdecimals = max(decimals) - strings = [s + (maxdecimals - decs) * " " - for s, decs in zip(strings, decimals)] - padfn = _padleft - elif not alignment: - return strings - else: - padfn = _padright - - if has_invisible: - width_fn = _visible_width - else: - width_fn = wcswidth - - maxwidth = max(max(map(width_fn, strings)), minwidth) - padded_strings = [padfn(maxwidth, s, has_invisible) for s in strings] - return padded_strings - - -def _more_generic(type1, type2): - types = { _none_type: 0, int: 1, float: 2, _binary_type: 3, _text_type: 4 } - invtypes = { 4: _text_type, 3: _binary_type, 2: float, 1: int, 0: _none_type } - moregeneric = max(types.get(type1, 4), types.get(type2, 4)) - return invtypes[moregeneric] - - -def _column_type(strings, has_invisible=True): - """The least generic type all column values are convertible to. - - >>> _column_type(["1", "2"]) is _int_type - True - >>> _column_type(["1", "2.3"]) is _float_type - True - >>> _column_type(["1", "2.3", "four"]) is _text_type - True - >>> _column_type(["four", '\u043f\u044f\u0442\u044c']) is _text_type - True - >>> _column_type([None, "brux"]) is _text_type - True - >>> _column_type([1, 2, None]) is _int_type - True - >>> import datetime as dt - >>> _column_type([dt.datetime(1991,2,19), dt.time(17,35)]) is _text_type - True - - """ - types = [_type(s, has_invisible) for s in strings ] - return reduce(_more_generic, types, int) - - -def _format(val, valtype, floatfmt, missingval=""): - u"""Format a value accoding to its type. - - Unicode is supported: - - >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', '\u0446\u0438\u0444\u0440\u0430'] ; \ - tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] ; \ - print(tabulate(tbl, headers=hrow)[0]) - буква цифра - ------- ------- - аз 2 - буки 4 - """ - if val is None: - return missingval - - if valtype in [int, _text_type]: - return "{0}".format(val) - elif valtype is _binary_type: - try: - return _text_type(val, "ascii") - except UnicodeDecodeError: - return _text_type('0x' + binascii.hexlify(val).decode('ascii')) - except TypeError: - return _text_type(val) - elif valtype is float: - return format(float(val), floatfmt) - else: - return "{0}".format(val) - - -def _align_header(header, alignment, width): - if alignment == "left": - return _padright(width, header) - elif alignment == "center": - return _padboth(width, header) - elif not alignment: - return "{0}".format(header) - else: - return _padleft(width, header) - - -def _normalize_tabular_data(tabular_data, headers): - """Transform a supported data type to a list of lists, and a list of headers. - - Supported tabular data types: - - * list-of-lists or another iterable of iterables - - * list of named tuples (usually used with headers="keys") - - * list of dicts (usually used with headers="keys") - - * list of OrderedDicts (usually used with headers="keys") - - * 2D NumPy arrays - - * NumPy record arrays (usually used with headers="keys") - - * dict of iterables (usually used with headers="keys") - - * pandas.DataFrame (usually used with headers="keys") - - The first row can be used as headers if headers="firstrow", - column indices can be used as headers if headers="keys". - - """ - - if hasattr(tabular_data, "keys") and hasattr(tabular_data, "values"): - # dict-like and pandas.DataFrame? - if hasattr(tabular_data.values, "__call__"): - # likely a conventional dict - keys = tabular_data.keys() - rows = list(izip_longest(*tabular_data.values())) # columns have to be transposed - elif hasattr(tabular_data, "index"): - # values is a property, has .index => it's likely a pandas.DataFrame (pandas 0.11.0) - keys = tabular_data.keys() - vals = tabular_data.values # values matrix doesn't need to be transposed - names = tabular_data.index - rows = [[v]+list(row) for v,row in zip(names, vals)] - else: - raise ValueError("tabular data doesn't appear to be a dict or a DataFrame") - - if headers == "keys": - headers = list(map(_text_type,keys)) # headers should be strings - - else: # it's a usual an iterable of iterables, or a NumPy array - rows = list(tabular_data) - - if (headers == "keys" and - hasattr(tabular_data, "dtype") and - getattr(tabular_data.dtype, "names")): - # numpy record array - headers = tabular_data.dtype.names - elif (headers == "keys" - and len(rows) > 0 - and isinstance(rows[0], tuple) - and hasattr(rows[0], "_fields")): - # namedtuple - headers = list(map(_text_type, rows[0]._fields)) - elif (len(rows) > 0 - and isinstance(rows[0], dict)): - # dict or OrderedDict - uniq_keys = set() # implements hashed lookup - keys = [] # storage for set - if headers == "firstrow": - firstdict = rows[0] if len(rows) > 0 else {} - keys.extend(firstdict.keys()) - uniq_keys.update(keys) - rows = rows[1:] - for row in rows: - for k in row.keys(): - #Save unique items in input order - if k not in uniq_keys: - keys.append(k) - uniq_keys.add(k) - if headers == 'keys': - headers = keys - elif isinstance(headers, dict): - # a dict of headers for a list of dicts - headers = [headers.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - elif headers == "firstrow": - if len(rows) > 0: - headers = [firstdict.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - else: - headers = [] - elif headers: - raise ValueError('headers for a list of dicts is not a dict or a keyword') - rows = [[row.get(k) for k in keys] for row in rows] - elif headers == "keys" and len(rows) > 0: - # keys are column indices - headers = list(map(_text_type, range(len(rows[0])))) - - # take headers from the first row if necessary - if headers == "firstrow" and len(rows) > 0: - headers = list(map(_text_type, rows[0])) # headers should be strings - rows = rows[1:] - - headers = list(map(_text_type,headers)) - rows = list(map(list,rows)) - - # pad with empty headers for initial columns if necessary - if headers and len(rows) > 0: - nhs = len(headers) - ncols = len(rows[0]) - if nhs < ncols: - headers = [""]*(ncols - nhs) + headers - - return rows, headers - -def table_formats(): - return _table_formats.keys() - -def tabulate(tabular_data, headers=[], tablefmt="simple", - floatfmt="g", numalign="decimal", stralign="left", - missingval=""): - """Format a fixed width table for pretty printing. - - >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])[0]) - --- --------- - 1 2.34 - -56 8.999 - 2 10001 - --- --------- - - The first required argument (`tabular_data`) can be a - list-of-lists (or another iterable of iterables), a list of named - tuples, a dictionary of iterables, an iterable of dictionaries, - a two-dimensional NumPy array, NumPy record array, or a Pandas' - dataframe. - - - Table headers - ------------- - - To print nice column headers, supply the second argument (`headers`): - - - `headers` can be an explicit list of column headers - - if `headers="firstrow"`, then the first row of data is used - - if `headers="keys"`, then dictionary keys or column indices are used - - Otherwise a headerless table is produced. - - If the number of headers is less than the number of columns, they - are supposed to be names of the last columns. This is consistent - with the plain-text format of R and Pandas' dataframes. - - >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], - ... headers="firstrow")[0]) - sex age - ----- ----- ----- - Alice F 24 - Bob M 19 - - - Column alignment - ---------------- - - `tabulate` tries to detect column types automatically, and aligns - the values properly. By default it aligns decimal points of the - numbers (or flushes integer numbers to the right), and flushes - everything else to the left. Possible column alignments - (`numalign`, `stralign`) are: "right", "center", "left", "decimal" - (only for `numalign`), and None (to disable alignment). - - - Table formats - ------------- - - `floatfmt` is a format specification used for columns which - contain numeric data with a decimal point. - - `None` values are replaced with a `missingval` string: - - >>> print(tabulate([["spam", 1, None], - ... ["eggs", 42, 3.14], - ... ["other", None, 2.7]], missingval="?")[0]) - ----- -- ---- - spam 1 ? - eggs 42 3.14 - other ? 2.7 - ----- -- ---- - - Various plain-text table formats (`tablefmt`) are supported: - 'plain', 'simple', 'grid', 'pipe', 'orgtbl', 'rst', 'mediawiki', - 'latex', and 'latex_booktabs'. Variable `tabulate_formats` contains the list of - currently supported formats. - - "plain" format doesn't use any pseudographics to draw tables, - it separates columns with a double space: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "plain")[0]) - strings numbers - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="plain")[0]) - spam 41.9999 - eggs 451 - - "simple" format is like Pandoc simple_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "simple")[0]) - strings numbers - --------- --------- - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="simple")[0]) - ---- -------- - spam 41.9999 - eggs 451 - ---- -------- - - "grid" is similar to tables produced by Emacs table.el package or - Pandoc grid_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "grid")[0]) - +-----------+-----------+ - | strings | numbers | - +===========+===========+ - | spam | 41.9999 | - +-----------+-----------+ - | eggs | 451 | - +-----------+-----------+ - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="grid")[0]) - +------+----------+ - | spam | 41.9999 | - +------+----------+ - | eggs | 451 | - +------+----------+ - - "fancy_grid" draws a grid using box-drawing characters: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "fancy_grid")[0]) - ╒═══════════╤═══════════╕ - │ strings │ numbers │ - ╞═══════════╪═══════════╡ - │ spam │ 41.9999 │ - ├───────────┼───────────┤ - │ eggs │ 451 │ - ╘═══════════╧═══════════╛ - - "pipe" is like tables in PHP Markdown Extra extension or Pandoc - pipe_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "pipe")[0]) - | strings | numbers | - |:----------|----------:| - | spam | 41.9999 | - | eggs | 451 | - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="pipe")[0]) - |:-----|---------:| - | spam | 41.9999 | - | eggs | 451 | - - "orgtbl" is like tables in Emacs org-mode and orgtbl-mode. They - are slightly different from "pipe" format by not using colons to - define column alignment, and using a "+" sign to indicate line - intersections: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "orgtbl")[0]) - | strings | numbers | - |-----------+-----------| - | spam | 41.9999 | - | eggs | 451 | - - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="orgtbl")[0]) - | spam | 41.9999 | - | eggs | 451 | - - "rst" is like a simple table format from reStructuredText; please - note that reStructuredText accepts also "grid" tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "rst")[0]) - ========= ========= - strings numbers - ========= ========= - spam 41.9999 - eggs 451 - ========= ========= - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")[0]) - ==== ======== - spam 41.9999 - eggs 451 - ==== ======== - - "mediawiki" produces a table markup used in Wikipedia and on other - MediaWiki-based sites: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="mediawiki")[0]) - {| class="wikitable" style="text-align: left;" - |+ - |- - ! strings !! align="right"| numbers - |- - | spam || align="right"| 41.9999 - |- - | eggs || align="right"| 451 - |} - - "html" produces HTML markup: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], ["eggs", "451.0"]], - ... headers="firstrow", tablefmt="html")[0]) - - - - -
stringsnumbers
spam41.9999
eggs451
- - "latex" produces a tabular environment of LaTeX document markup: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex")[0]) - \\begin{tabular}{ll} - \\hline - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\hline - \\end{tabular} - - "latex_booktabs" produces a tabular environment of LaTeX document markup - using the booktabs.sty package: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="latex_booktabs")[0]) - \\begin{tabular}{ll} - \\toprule - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\bottomrule - \end{tabular} - - Also returns a tuple of the raw rows pulled from tabular_data - """ - if tabular_data is None: - tabular_data = [] - list_of_lists, headers = _normalize_tabular_data(tabular_data, headers) - - # format rows and columns, convert numeric values to strings - cols = list(zip(*list_of_lists)) - coltypes = list(map(_column_type, cols)) - cols = [[_format(v, ct, floatfmt, missingval) for v in c] - for c,ct in zip(cols, coltypes)] - - # optimization: look for ANSI control codes once, - # enable smart width functions only if a control code is found - plain_text = '\n'.join(['\t'.join(map(_text_type, headers))] + \ - ['\t'.join(map(_text_type, row)) for row in cols]) - has_invisible = re.search(_invisible_codes, plain_text) - if has_invisible: - width_fn = _visible_width - else: - width_fn = wcswidth - - if not isinstance(tablefmt, TableFormat): - tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) - - if tablefmt.with_align: - # align columns - aligns = [numalign if ct in [int,float] else stralign for ct in coltypes] - else: - aligns = [False for ct in coltypes] - minwidths = [width_fn(h) + MIN_PADDING for h in headers] if headers else [0]*len(cols) - cols = [_align_column(c, a, minw, has_invisible) - for c, a, minw in zip(cols, aligns, minwidths)] - - if headers: - # align headers and add headers - t_cols = cols or [['']] * len(headers) - if tablefmt.with_align: - t_aligns = aligns or [stralign] * len(headers) - else: - t_aligns = [False for ct in coltypes] - minwidths = [max(minw, width_fn(c[0])) for minw, c in zip(minwidths, t_cols)] - headers = [_align_header(h, a, minw) - for h, a, minw in zip(headers, t_aligns, minwidths)] - rows = list(zip(*cols)) - else: - minwidths = [width_fn(c[0]) for c in cols] - rows = list(zip(*cols)) - - return _format_table(tablefmt, headers, rows, minwidths, aligns), rows - - -def _build_simple_row(padded_cells, rowfmt): - "Format row according to DataRow format without padding." - begin, sep, end = rowfmt - return (begin + sep.join(padded_cells) + end).rstrip() - - -def _build_row(padded_cells, colwidths, colaligns, rowfmt): - "Return a string which represents a row of data cells." - if not rowfmt: - return None - if hasattr(rowfmt, "__call__"): - return rowfmt(padded_cells, colwidths, colaligns) - else: - return _build_simple_row(padded_cells, rowfmt) - - -def _build_line(colwidths, colaligns, linefmt): - "Return a string which represents a horizontal line." - if not linefmt: - return None - if hasattr(linefmt, "__call__"): - return linefmt(colwidths, colaligns) - else: - begin, fill, sep, end = linefmt - cells = [fill*w for w in colwidths] - return _build_simple_row(cells, (begin, sep, end)) - - -def _pad_row(cells, padding): - if cells: - pad = " "*padding - padded_cells = [pad + cell + pad for cell in cells] - return padded_cells - else: - return cells - - -def _format_table(fmt, headers, rows, colwidths, colaligns): - """Produce a plain-text representation of the table.""" - lines = [] - hidden = fmt.with_header_hide if (headers and fmt.with_header_hide) else [] - pad = fmt.padding - headerrow = fmt.headerrow - - padded_widths = [(w + 2*pad) for w in colwidths] - padded_headers = _pad_row(headers, pad) - padded_rows = [_pad_row(row, pad) for row in rows] - - if fmt.lineabove and "lineabove" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.lineabove)) - - if padded_headers: - lines.append(_build_row(padded_headers, padded_widths, colaligns, headerrow)) - if fmt.linebelowheader and "linebelowheader" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.linebelowheader)) - - if padded_rows and fmt.linebetweenrows and "linebetweenrows" not in hidden: - # initial rows with a line below - for row in padded_rows[:-1]: - lines.append(_build_row(row, padded_widths, colaligns, fmt.datarow)) - lines.append(_build_line(padded_widths, colaligns, fmt.linebetweenrows)) - # the last row without a line below - lines.append(_build_row(padded_rows[-1], padded_widths, colaligns, fmt.datarow)) - else: - for row in padded_rows: - lines.append(_build_row(row, padded_widths, colaligns, fmt.datarow)) - - if fmt.linebelow and "linebelow" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.linebelow)) - - return "\n".join(lines) - - -def _main(): - """\ - Usage: tabulate [options] [FILE ...] - - Pretty-print tabular data. See also https://bitbucket.org/astanin/python-tabulate - - FILE a filename of the file with tabular data; - if "-" or missing, read data from stdin. - - Options: - - -h, --help show this message - -1, --header use the first row of data as a table header - -s REGEXP, --sep REGEXP use a custom column separator (default: whitespace) - -f FMT, --format FMT set output table format; supported formats: - plain, simple, grid, fancy_grid, pipe, orgtbl, - rst, mediawiki, html, latex, latex_booktabs, tsv - (default: simple) - """ - import getopt - import sys - import textwrap - usage = textwrap.dedent(_main.__doc__) - try: - opts, args = getopt.getopt(sys.argv[1:], - "h1f:s:", - ["help", "header", "format", "separator"]) - except getopt.GetoptError as e: - print(e) - print(usage) - sys.exit(2) - headers = [] - tablefmt = "simple" - sep = r"\s+" - for opt, value in opts: - if opt in ["-1", "--header"]: - headers = "firstrow" - elif opt in ["-f", "--format"]: - if value not in tabulate_formats: - print("%s is not a supported table format" % value) - print(usage) - sys.exit(3) - tablefmt = value - elif opt in ["-s", "--sep"]: - sep = value - elif opt in ["-h", "--help"]: - print(usage) - sys.exit(0) - files = [sys.stdin] if not args else args - for f in files: - if f == "-": - f = sys.stdin - if _is_file(f): - _pprint_file(f, headers=headers, tablefmt=tablefmt, sep=sep) - else: - with open(f) as fobj: - _pprint_file(fobj) - - -def _pprint_file(fobject, headers, tablefmt, sep): - rows = fobject.readlines() - table = [re.split(sep, r.rstrip()) for r in rows] - print(tabulate(table, headers, tablefmt)) - - -if __name__ == "__main__": - _main() diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 41364756c..656e5d531 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -1,13 +1,15 @@ from __future__ import print_function from __future__ import unicode_literals import logging +from re import compile, escape +from collections import Counter + from prompt_toolkit.completion import Completer, Completion + +from .output_formatter import OutputFormatter from .packages.completion_engine import suggest_type from .packages.parseutils import last_word from .packages.special.favoritequeries import favoritequeries -from re import compile, escape -from .packages.tabulate import table_formats -from collections import Counter _logger = logging.getLogger(__name__) @@ -57,7 +59,8 @@ def __init__(self, smart_completion=True): self.name_pattern = compile("^[_a-z][_a-z0-9\$]*$") self.special_commands = [] - self.table_formats = table_formats() + formatter = OutputFormatter() + self.table_formats = formatter.supported_formats() self.reset_completions() def escape_name(self, name): diff --git a/setup.py b/setup.py index 82cdef6e1..3ecc21c48 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', 'pycryptodome >= 3', + 'tabulate >= 0.7.6', ] setup( diff --git a/tests/test_main.py b/tests/test_main.py index 14f1c10e2..cb769a0be 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,7 +3,7 @@ import click from click.testing import CliRunner -from mycli.main import (cli, confirm_destructive_query, format_output, +from mycli.main import (cli, confirm_destructive_query, is_destructive, query_starts_with, queries_start_with, thanks_picker, PACKAGE_ROOT) from utils import USER, HOST, PORT, PASSWORD, dbtest, run @@ -16,32 +16,6 @@ CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, '--password', PASSWORD, '_test_db'] -def test_format_output(): - results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], - 'test status', 'psql') - expected = ['Title', '+---------+---------+\n| head1 | head2 |\n|---------+---------|\n| abc | def |\n+---------+---------+', 'test status'] - assert results == expected - -def test_format_output_auto_expand(): - table_results = format_output('Title', [('abc', 'def')], - ['head1', 'head2'], 'test status', 'psql', - max_width=100) - table = ['Title', '+---------+---------+\n| head1 | head2 |\n|---------+---------|\n| abc | def |\n+---------+---------+', 'test status'] - assert table_results == table - - expanded_results = format_output('Title', [('abc', 'def')], - ['head1', 'head2'], 'test status', 'psql', - max_width=1) - expanded = ['Title', u'***************************[ 1. row ]***************************\nhead1 | abc\nhead2 | def\n', 'test status'] - assert expanded_results == expanded - -def test_format_output_no_table(): - results = format_output('Title', [('abc', 'def')], ['head1', 'head2'], - 'test status', None) - - expected = ['Title', u'head1\thead2\nabc\tdef', 'test status'] - assert results == expected - @dbtest def test_execute_arg(executor): run(executor, 'create table test (a text)') diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py deleted file mode 100644 index e0ddc407d..000000000 --- a/tests/test_tabulate.py +++ /dev/null @@ -1,22 +0,0 @@ -from mycli.packages.tabulate import tabulate -from textwrap import dedent - - -def test_dont_strip_leading_whitespace(): - data = [[' abc']] - headers = ['xyz'] - tbl, _ = tabulate(data, headers, tablefmt='psql') - assert tbl == dedent(''' - +---------+ - | xyz | - |---------| - | abc | - +---------+ ''').strip() -def test_dont_add_whitespace(): - data = [[3, 4]] - headers = ['1', '2'] - tbl, _ = tabulate(data, headers, tablefmt='tsv') - assert tbl == dedent(''' - 1\t2 - 3\t4 - ''').strip() diff --git a/tests/utils.py b/tests/utils.py index c76cb7ad3..5e9e0abb0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,11 @@ -import pytest -import pymysql -from mycli.main import format_output, special from os import getenv +import pymysql +import pytest + +from mycli.main import MyCli, special +from mycli.output_formatter import OutputFormatter + PASSWORD = getenv('PYTEST_PASSWORD') USER = getenv('PYTEST_USER', 'root') HOST = getenv('PYTEST_HOST', 'localhost') @@ -37,8 +40,14 @@ def create_db(dbname): def run(executor, sql, join=False): " Return string output for the sql to be run " result = [] + + # TODO: this needs to go away. `run()` should not test formatted output. + # It should test raw results. + mycli = MyCli() + mycli.table_format = 'psql' for title, rows, headers, status in executor.run(sql): - result.extend(format_output(title, rows, headers, status, 'psql', special.is_expanded_output())) + result.extend(mycli.format_output(title, rows, headers, status, special.is_expanded_output())) + if join: result = '\n'.join(result) return result From 2bc4facb4ae978c7465dcb4128b895224348acb1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:30:40 -0500 Subject: [PATCH 0304/1025] Add missing class. --- mycli/output_formatter.py | 57 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 mycli/output_formatter.py diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py new file mode 100644 index 000000000..c939aa512 --- /dev/null +++ b/mycli/output_formatter.py @@ -0,0 +1,57 @@ +"""A generic output formatter interface.""" + +from __future__ import unicode_literals + +from tabulate import tabulate + +from .packages.expanded import expanded_table + + +def tabulate_wrapper(data, headers, table_format=None, missing_value=None): + """Wrap tabulate inside a standard function for OutputFormatter.""" + return tabulate(data, headers, tablefmt=table_format, + missingval=missing_value) + + +class OutputFormatter(object): + """A class with a standard interface for various formatting libraries.""" + + def __init__(self): + """Register the supported output formats.""" + self._output_formats = {} + + tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', + 'orgtbl', 'jira', 'psql', 'rst', 'tsv', + 'mediawiki', 'moinmoin', 'html', 'latex', + 'latex_booktabs', 'textile') + for tabulate_format in tabulate_formats: + self.register_output_format(tabulate_format, tabulate_wrapper, + table_format=tabulate_format) + + self.register_output_format('expanded', expanded_table) + + def register_output_format(self, name, function, **kwargs): + """Register a new output format. + + *function* should be a callable that accepts the following arguments: + - *headers*: A list of headers for the output data. + - *data*: The data that needs formatting. + - *kwargs*: Any other keyword arguments for controlling the output. + It should return the formatted output as a string. + """ + self._output_formats[name] = (function, kwargs) + + def supported_formats(self): + """Return the supported output format names.""" + return tuple(self._output_formats.keys()) + + def format_output(self, data, headers, format_name, **kwargs): + """Format the headers and data using a specific formatter. + + *format_name* must be a formatter available in `supported_formats()`. + + All keyword arguments are passed to the specified formatter. + """ + function, fkwargs = self._output_formats[format_name] + fkwargs.update(kwargs) + return function(data, headers, **fkwargs) From 3c783213fa0c2c63668a709644ddded28f2dfad7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:34:10 -0500 Subject: [PATCH 0305/1025] Make logic more readable. --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index b9a3cebe4..46de4cbff 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -192,7 +192,7 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - if not arg in self.formatter.supported_formats(): + if arg not in self.formatter.supported_formats(): msg = "Table type %s not yet implemented. Allowed types:" % arg for table_type in self.formatter.supported_formats(): msg += "\n\t%s" % table_type From 26ff1f3f23fac653de47e51d4494c494a1fe4cc3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:37:31 -0500 Subject: [PATCH 0306/1025] Move CSV to output formatter. --- mycli/main.py | 17 ----------------- mycli/output_formatter.py | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 46de4cbff..389f176c6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -15,12 +15,6 @@ from random import choice from io import open -# support StringIO for Python 2 and 3 -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - import click import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction @@ -735,17 +729,6 @@ def format_output(self, title, cur, headers, status, expanded=False, if expanded: output.append(self.formatter.format_output(cur, headers, 'expanded')) - elif self.table_format == 'csv': - content = StringIO() - writer = csv.writer(content) - writer.writerow(headers) - - for row in cur: - row = ['null' if val is None else str(val) for val in row] - writer.writerow(row) - - output.append(content.getvalue()) - content.close() else: rows = list(cur) formatted = self.formatter.format_output(rows, headers, diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index c939aa512..5081a5d61 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -2,6 +2,12 @@ from __future__ import unicode_literals +import csv +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + from tabulate import tabulate from .packages.expanded import expanded_table @@ -13,6 +19,21 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value=None): missingval=missing_value) +def csv_wrapper(data, headers): + content = StringIO() + writer = csv.writer(content) + writer.writerow(headers) + + for row in data: + row = ['null' if val is None else str(val) for val in row] + writer.writerow(row) + + output = content.getvalue() + content.close() + + return output + + class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" @@ -28,6 +49,7 @@ def __init__(self): self.register_output_format(tabulate_format, tabulate_wrapper, table_format=tabulate_format) + self.register_output_format('csv', csv_wrapper) self.register_output_format('expanded', expanded_table) def register_output_format(self, name, function, **kwargs): From 5da417568292477e3603c8124ed733e7be0dd840 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:51:47 -0500 Subject: [PATCH 0307/1025] Simply format output. --- mycli/main.py | 27 ++++++++++++--------------- mycli/output_formatter.py | 2 +- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 389f176c6..c3485dfc4 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -720,27 +720,24 @@ def run_query(self, query, new_line=True): def format_output(self, title, cur, headers, status, expanded=False, max_width=None): + table_format = 'expanded' if expanded else self.table_format output = [] + if title: # Only print the title if it's not None. output.append(title) + if cur: headers = [utf8tounicode(x) for x in headers] - if expanded: - output.append(self.formatter.format_output(cur, headers, - 'expanded')) - else: - rows = list(cur) - formatted = self.formatter.format_output(rows, headers, - self.table_format) - if (self.table_format != 'expanded' and - max_width and rows and - content_exceeds_width(rows[0], max_width) and - headers): - output.append(self.formatter.format_output(cur, headers, - 'expanded')) - else: - output.append(formatted) + rows = list(cur) + formatted = self.formatter.format_output(rows, headers, table_format) + + if (table_format != 'expanded' and max_width and rows and + content_exceeds_width(rows[0], max_width) and headers): + formatted = self.formatter.format_output(rows, headers, 'expanded') + + output.append(formatted) + if status: # Only print the status if it's not None. output.append(status) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5081a5d61..5c2678a0d 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -13,7 +13,7 @@ from .packages.expanded import expanded_table -def tabulate_wrapper(data, headers, table_format=None, missing_value=None): +def tabulate_wrapper(data, headers, table_format=None, missing_value=''): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, missingval=missing_value) From 8918a51db94b46ae9846946685860c51f2ad0061 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:55:39 -0500 Subject: [PATCH 0308/1025] Fix tsv/table logic. --- mycli/main.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index c3485dfc4..cb40aaf09 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -838,10 +838,12 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - table_format = 'tsv' - if csv: + table_format = None + if table: + table_format = mycli.table_format + elif csv: table_format = 'csv' - mycli.table_format = table_format or mycli.table_format + mycli.table_format = table_format or 'tsv' mycli.run_query(execute) exit(0) except Exception as e: @@ -863,14 +865,16 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - table_format = 'tsv' + table_format = None new_line = True if csv: table_format = 'csv' new_line = False + elif table: + table_format = mycli.table_format - mycli.table_format = table_format or mycli.table_format + mycli.table_format = table_format or 'tsv' mycli.run_query(stdin_text, new_line=new_line) exit(0) From 7b6398a14df84ebaa0801fe000b620a2c19a5949 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 22:59:01 -0500 Subject: [PATCH 0309/1025] Use missing_value keyword for csv wrapper. --- mycli/output_formatter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5c2678a0d..5a90709ac 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -19,13 +19,14 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value=''): missingval=missing_value) -def csv_wrapper(data, headers): +def csv_wrapper(data, headers, missing_value='null'): + """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() writer = csv.writer(content) writer.writerow(headers) for row in data: - row = ['null' if val is None else str(val) for val in row] + row = [missing_value if val is None else str(val) for val in row] writer.writerow(row) output = content.getvalue() From f5a305d45652d017e411c1f47cf9e87fe412b4f7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:05:22 -0500 Subject: [PATCH 0310/1025] Do not use tabulate for TSV formatting. --- mycli/output_formatter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5a90709ac..6b0d8f2a4 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -19,10 +19,10 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value=''): missingval=missing_value) -def csv_wrapper(data, headers, missing_value='null'): +def csv_wrapper(data, headers, missing_value='null', delimiter=','): """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() - writer = csv.writer(content) + writer = csv.writer(content, delimiter=delimiter) writer.writerow(headers) for row in data: @@ -43,14 +43,15 @@ def __init__(self): self._output_formats = {} tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'jira', 'psql', 'rst', 'tsv', - 'mediawiki', 'moinmoin', 'html', 'latex', - 'latex_booktabs', 'textile') + 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', + 'moinmoin', 'html', 'latex', 'latex_booktabs', + 'textile') for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, table_format=tabulate_format) self.register_output_format('csv', csv_wrapper) + self.register_output_format('tsv', csv_wrapper, delimiter='\t') self.register_output_format('expanded', expanded_table) def register_output_format(self, name, function, **kwargs): From 93c6306b16c6316e352c6f6307b45c49602258dd Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:07:11 -0500 Subject: [PATCH 0311/1025] Add space before expanded. --- mycli/output_formatter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 6b0d8f2a4..29922b866 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -52,6 +52,7 @@ def __init__(self): self.register_output_format('csv', csv_wrapper) self.register_output_format('tsv', csv_wrapper, delimiter='\t') + self.register_output_format('expanded', expanded_table) def register_output_format(self, name, function, **kwargs): From 3a881284339f71033d29e78cef288589efcb5fe1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:25:42 -0500 Subject: [PATCH 0312/1025] Add preprocessor. --- mycli/output_formatter.py | 32 +++++++++++++++++++++++--------- mycli/packages/expanded.py | 2 +- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 29922b866..9cc602671 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -13,20 +13,23 @@ from .packages.expanded import expanded_table -def tabulate_wrapper(data, headers, table_format=None, missing_value=''): +def override_missing_value(data, missing_value='', **_): + """Override missing values in the data with *missing_value*.""" + return [[missing_value if v is None else v for v in row] for row in data] + + +def tabulate_wrapper(data, headers, table_format=None, **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value) + return tabulate(data, headers, tablefmt=table_format) -def csv_wrapper(data, headers, missing_value='null', delimiter=','): +def csv_wrapper(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() writer = csv.writer(content, delimiter=delimiter) writer.writerow(headers) for row in data: - row = [missing_value if val is None else str(val) for val in row] writer.writerow(row) output = content.getvalue() @@ -48,12 +51,20 @@ def __init__(self): 'textile') for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, - table_format=tabulate_format) + table_format=tabulate_format, + preprocessor=override_missing_value, + missing_value='') - self.register_output_format('csv', csv_wrapper) - self.register_output_format('tsv', csv_wrapper, delimiter='\t') + self.register_output_format('csv', csv_wrapper, + preprocessor=override_missing_value, + missing_value='null') + self.register_output_format('tsv', csv_wrapper, delimiter='\t', + preprocessor=override_missing_value, + missing_value='null') - self.register_output_format('expanded', expanded_table) + self.register_output_format('expanded', expanded_table, + preprocessor=override_missing_value, + missing_value='') def register_output_format(self, name, function, **kwargs): """Register a new output format. @@ -79,4 +90,7 @@ def format_output(self, data, headers, format_name, **kwargs): """ function, fkwargs = self._output_formats[format_name] fkwargs.update(kwargs) + preprocessor = fkwargs.pop('preprocessor', None) + if preprocessor: + data = preprocessor(data, **fkwargs) return function(data, headers, **fkwargs) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 245e4ca72..2b0b7207f 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -19,7 +19,7 @@ def format_field(value): except UnicodeDecodeError: return _text_type('0x' + binascii.hexlify(value).decode('ascii')) -def expanded_table(rows, headers): +def expanded_table(rows, headers, **_): header_len = max([len(x) for x in headers]) max_row_len = 0 results = [] From cff2ddd8b198d4d47d45ef67f5d7493d95b073be Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:33:41 -0500 Subject: [PATCH 0313/1025] Simplify missing value. --- mycli/output_formatter.py | 6 +++--- mycli/packages/expanded.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 9cc602671..9c3d7d028 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -18,9 +18,10 @@ def override_missing_value(data, missing_value='', **_): return [[missing_value if v is None else v for v in row] for row in data] -def tabulate_wrapper(data, headers, table_format=None, **_): +def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format) + return tabulate(data, headers, tablefmt=table_format, + missingval=missing_value) def csv_wrapper(data, headers, delimiter=',', **_): @@ -52,7 +53,6 @@ def __init__(self): for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, table_format=tabulate_format, - preprocessor=override_missing_value, missing_value='') self.register_output_format('csv', csv_wrapper, diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 2b0b7207f..b39d9507f 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -35,7 +35,6 @@ def expanded_table(rows, headers, **_): max_row_len = row_len for header, value in zip(padded_headers, row): - if value is None: value = '' row_result.append(u"%s %s" % (header, value)) results.append('\n'.join(row_result)) From 5cbdf480e9cc3a65d6ac54619ec797a3cd4e41e0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:36:36 -0500 Subject: [PATCH 0314/1025] Disable number parsing. --- mycli/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 9c3d7d028..2d73e1a56 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -21,7 +21,7 @@ def override_missing_value(data, missing_value='', **_): def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value) + missingval=missing_value, disable_numparse=True) def csv_wrapper(data, headers, delimiter=',', **_): From cced4902517706581ec93237a4424e7be5a3b2c0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:44:23 -0500 Subject: [PATCH 0315/1025] Remove unused imports. --- mycli/main.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index cb40aaf09..beeeb7d23 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -5,9 +5,7 @@ import os import os.path import sys -import csv import traceback -import socket import logging import threading from time import time @@ -27,9 +25,8 @@ ConditionalProcessor) from prompt_toolkit.history import FileHistory from pygments.token import Token -from configobj import ConfigObj, ConfigObjError -from .packages.special.main import (COMMANDS, NO_QUERY) +from .packages.special.main import NO_QUERY import mycli.packages.special as special from .sqlcompleter import SQLCompleter from .clitoolbar import create_toolbar_tokens_func @@ -38,8 +35,7 @@ from .clibuffer import CLIBuffer from .completion_refresher import CompletionRefresher from .config import (write_default_config, get_mylogin_cnf_path, - open_mylogin_cnf, read_config_file, - read_config_files, str_to_bool) + open_mylogin_cnf, read_config_files, str_to_bool) from .key_bindings import mycli_bindings from .output_formatter import OutputFormatter from .encodingutils import utf8tounicode From d0f1dbf441ae823d56cadca456b1ff35a35abd88 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 28 Mar 2017 23:47:06 -0500 Subject: [PATCH 0316/1025] Remove tabulate license note. --- LICENSE.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 9a41a67d4..8afaa2657 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -27,9 +27,3 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------- - -This program also bundles with it python-tabulate -(https://pypi.python.org/pypi/tabulate) library. This library is licensed under -MIT License. - -------------------------------------------------------------------------------- From 369b5b0795d5391870f260c8563cb6e27e895308 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 11:39:29 +0200 Subject: [PATCH 0317/1025] Fix tests/test_sqlexecute unit tests --- mycli/output_formatter.py | 17 +++++++++++++++++ tests/test_sqlexecute.py | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 2d73e1a56..1397b54f6 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import csv +import binascii try: from cStringIO import StringIO except ImportError: @@ -18,6 +19,21 @@ def override_missing_value(data, missing_value='', **_): return [[missing_value if v is None else v for v in row] for row in data] +def bytes_to_unicode(data, **_): + results = [] + for row in data: + result = [] + for v in row: + if isinstance(v, bytes): + try: + conv = v.decode('utf8') + except: + v = '0x' + binascii.hexlify(v).decode('ascii') + result.append(v) + results.append(result) + return results + + def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, @@ -52,6 +68,7 @@ def __init__(self): 'textile') for tabulate_format in tabulate_formats: self.register_output_format(tabulate_format, tabulate_wrapper, + preprocessor=bytes_to_unicode, table_format=tabulate_format, missing_value='') diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index e9929a70f..d964f43ba 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -27,9 +27,9 @@ def test_bools(executor): results = run(executor, '''select * from test''', join=True) assert results == dedent("""\ +-----+ - | a | + | a | |-----| - | 1 | + | 1 | +-----+ 1 row in set""") From ae3712f764aaf22aa5a2fe3c9a277b375f541bc5 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 12:06:01 +0200 Subject: [PATCH 0318/1025] fix test_main unit tests --- tests/test_main.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index cb769a0be..29501f137 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,6 +8,8 @@ thanks_picker, PACKAGE_ROOT) from utils import USER, HOST, PORT, PASSWORD, dbtest, run +from textwrap import dedent + try: text_type = basestring except NameError: @@ -80,7 +82,7 @@ def test_batch_mode(executor): result = runner.invoke(cli, args=CLI_ARGS, input=sql) assert result.exit_code == 0 - assert 'count(*)\n3\na\nabc\n' in result.output + assert 'count(*)\n3\n\na\nabc\n' in result.output @dbtest def test_batch_mode_table(executor): @@ -95,10 +97,17 @@ def test_batch_mode_table(executor): runner = CliRunner() result = runner.invoke(cli, args=CLI_ARGS + ['-t'], input=sql) - expected = ( - '| count(*) |\n|------------|\n| 3 |\n+------------+\n' - '+-----+\n| a |\n|-----|\n| abc |\n+-----+' - ) + expected = (dedent("""\ + +------------+ + | count(*) | + |------------| + | 3 | + +------------+ + +-----+ + | a | + |-----| + | abc | + +-----+""")) assert result.exit_code == 0 assert expected in result.output From 3dbe5c01cb03fa42b6dc7718b3b4b8bfcfc5c92a Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 12:39:37 +0200 Subject: [PATCH 0319/1025] use str for python2 delimiter --- mycli/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 1397b54f6..178add2f7 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -43,7 +43,7 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): def csv_wrapper(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" content = StringIO() - writer = csv.writer(content, delimiter=delimiter) + writer = csv.writer(content, delimiter=str(delimiter)) writer.writerow(headers) for row in data: From 434aea0d3f31cde2f76309b1eea429e1b97f6527 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 1 Apr 2017 16:37:06 +0200 Subject: [PATCH 0320/1025] add docstring to bytes_to_unicode function --- mycli/output_formatter.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 178add2f7..2375c0dd6 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """A generic output formatter interface.""" from __future__ import unicode_literals @@ -20,6 +21,12 @@ def override_missing_value(data, missing_value='', **_): def bytes_to_unicode(data, **_): + """Convert bytes that cannot be decoded to utf8 to hexlified string + >>> result = bytes_to_unicode([[b"\\xff", "abc", "✌"]]) + >>> print(" ".join(result[0])) + 0xff abc ✌ + """ + results = [] for row in data: result = [] From be0e2ed85917ec4015bbde46f132b85d511e4e45 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 10:00:13 -0500 Subject: [PATCH 0321/1025] Add headers to preprocessor. --- mycli/output_formatter.py | 44 +++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 2375c0dd6..ba79e8037 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -15,30 +15,34 @@ from .packages.expanded import expanded_table -def override_missing_value(data, missing_value='', **_): +def override_missing_value(data, headers, missing_value='', **_): """Override missing values in the data with *missing_value*.""" - return [[missing_value if v is None else v for v in row] for row in data] + return ([[missing_value if v is None else v for v in row] for row in data], + headers) -def bytes_to_unicode(data, **_): - """Convert bytes that cannot be decoded to utf8 to hexlified string - >>> result = bytes_to_unicode([[b"\\xff", "abc", "✌"]]) - >>> print(" ".join(result[0])) - 0xff abc ✌ +def bytes_to_hex(b): + """Convert bytes that cannot be decoded to utf8 to hexlified string. + + >>> print(bytes_to_hex(b"\\xff")) + 0xff + >>> print(bytes_to_hex('abc')) + abc + >>> print(bytes_to_hex('✌')) + ✌ """ + if isinstance(b, bytes): + try: + b.decode('utf8') + except: + b = '0x' + binascii.hexlify(b).decode('ascii') + return b - results = [] - for row in data: - result = [] - for v in row: - if isinstance(v, bytes): - try: - conv = v.decode('utf8') - except: - v = '0x' + binascii.hexlify(v).decode('ascii') - result.append(v) - results.append(result) - return results + +def bytes_to_unicode(data, headers, **_): + """Convert all *data* and *headers* to unicode.""" + return ([[bytes_to_hex(v) for v in row] for row in data], + [bytes_to_hex(h) for h in headers]) def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): @@ -116,5 +120,5 @@ def format_output(self, data, headers, format_name, **kwargs): fkwargs.update(kwargs) preprocessor = fkwargs.pop('preprocessor', None) if preprocessor: - data = preprocessor(data, **fkwargs) + data, headers = preprocessor(data, headers, **fkwargs) return function(data, headers, **fkwargs) From f4d05278d7984e7d3cdc9d0361bf3a9f89d89cfc Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:10:20 -0500 Subject: [PATCH 0322/1025] Move mycli table format to output formatter. --- mycli/main.py | 48 +++++++++++++++++++-------------------- mycli/output_formatter.py | 26 ++++++++++++++++++--- tests/utils.py | 1 - 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index beeeb7d23..fa715d26a 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -106,7 +106,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.table_format = c['main']['table_format'] + self.formatter = OutputFormatter(format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] @@ -133,8 +133,6 @@ def __init__(self, sqlexecute=None, prompt=None, self.completion_refresher = CompletionRefresher() - self.formatter = OutputFormatter() - self.logger = logging.getLogger(__name__) self.initialize_logging() @@ -182,14 +180,16 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - if arg not in self.formatter.supported_formats(): - msg = "Table type %s not yet implemented. Allowed types:" % arg + print('change table: ' + arg) + try: + self.formatter.set_format_name(arg) + yield (None, None, None, + 'Changed table type to {}'.format(arg)) + except ValueError: + msg = 'Table type {} not yet implemented. Allowed types:'.format(arg) for table_type in self.formatter.supported_formats(): - msg += "\n\t%s" % table_type + msg += "\n\t{}".format(table_type) yield (None, None, None, msg) - else: - self.table_format = arg - yield (None, None, None, "Changed table Type to %s" % self.table_format) def change_db(self, arg, **_): if arg is None: @@ -716,7 +716,7 @@ def run_query(self, query, new_line=True): def format_output(self, title, cur, headers, status, expanded=False, max_width=None): - table_format = 'expanded' if expanded else self.table_format + expanded = expanded or self.formatter.get_format_name() == 'expanded' output = [] if title: # Only print the title if it's not None. @@ -726,11 +726,13 @@ def format_output(self, title, cur, headers, status, expanded=False, headers = [utf8tounicode(x) for x in headers] rows = list(cur) - formatted = self.formatter.format_output(rows, headers, table_format) + formatted = self.formatter.format_output( + rows, headers, format_name='expanded' if expanded else None) - if (table_format != 'expanded' and max_width and rows and + if (not expanded and max_width and rows and content_exceeds_width(rows[0], max_width) and headers): - formatted = self.formatter.format_output(rows, headers, 'expanded') + formatted = self.formatter.format_output( + rows, headers, format_name='expanded') output.append(formatted) @@ -834,12 +836,11 @@ def cli(database, user, host, port, socket, password, dbname, # --execute argument if execute: try: - table_format = None - if table: - table_format = mycli.table_format - elif csv: - table_format = 'csv' - mycli.table_format = table_format or 'tsv' + if csv: + mycli.formatter.set_format_name('csv') + elif not table: + mycli.formatter.set_format_name('tsv') + mycli.run_query(execute) exit(0) except Exception as e: @@ -861,16 +862,13 @@ def cli(database, user, host, port, socket, password, dbname, confirm_destructive_query(stdin_text) is False): exit(0) try: - table_format = None new_line = True if csv: - table_format = 'csv' + mycli.formatter.set_format_name('csv') new_line = False - elif table: - table_format = mycli.table_format - - mycli.table_format = table_format or 'tsv' + elif not table: + mycli.formatter.set_format_name('tsv') mycli.run_query(stdin_text, new_line=new_line) exit(0) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index ba79e8037..ebb92b1d7 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -3,8 +3,8 @@ from __future__ import unicode_literals -import csv import binascii +import csv try: from cStringIO import StringIO except ImportError: @@ -69,9 +69,10 @@ def csv_wrapper(data, headers, delimiter=',', **_): class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" - def __init__(self): + def __init__(self, format_name=None): """Register the supported output formats.""" self._output_formats = {} + self._format_name = None tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', @@ -94,6 +95,21 @@ def __init__(self): preprocessor=override_missing_value, missing_value='') + if format_name: + self.set_format_name(format_name) + + def set_format_name(self, format_name): + """Set the OutputFormatter's default format.""" + if format_name in self.supported_formats(): + self._format_name = format_name + else: + raise ValueError('unrecognized format_name: {}'.format( + format_name)) + + def get_format_name(self): + """Get the OutputFormatter's default format.""" + return self._format_name + def register_output_format(self, name, function, **kwargs): """Register a new output format. @@ -109,13 +125,17 @@ def supported_formats(self): """Return the supported output format names.""" return tuple(self._output_formats.keys()) - def format_output(self, data, headers, format_name, **kwargs): + def format_output(self, data, headers, format_name=None, **kwargs): """Format the headers and data using a specific formatter. *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. """ + format_name = format_name or self._format_name + if format_name not in self.supported_formats(): + raise ValueError('unrecognized format: {}'.format(format_name)) + function, fkwargs = self._output_formats[format_name] fkwargs.update(kwargs) preprocessor = fkwargs.pop('preprocessor', None) diff --git a/tests/utils.py b/tests/utils.py index 5e9e0abb0..64006324f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,7 +44,6 @@ def run(executor, sql, join=False): # TODO: this needs to go away. `run()` should not test formatted output. # It should test raw results. mycli = MyCli() - mycli.table_format = 'psql' for title, rows, headers, status in executor.run(sql): result.extend(mycli.format_output(title, rows, headers, status, special.is_expanded_output())) From 90fd5fe54e05b5a9f8e33af35f71a5fcae8ebce6 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:20:34 -0500 Subject: [PATCH 0323/1025] Use context manager for StringIO. --- mycli/output_formatter.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index ebb92b1d7..6a0564890 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import binascii +import contextlib import csv try: from cStringIO import StringIO @@ -53,17 +54,14 @@ def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): def csv_wrapper(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" - content = StringIO() - writer = csv.writer(content, delimiter=str(delimiter)) - writer.writerow(headers) + with contextlib.closing(StringIO()) as content: + writer = csv.writer(content, delimiter=str(delimiter)) - for row in data: - writer.writerow(row) + writer.writerow(headers) + for row in data: + writer.writerow(row) - output = content.getvalue() - content.close() - - return output + return content.getvalue() class OutputFormatter(object): From 9f0ace286f60875d708018a41b10c7015cebcac7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:27:34 -0500 Subject: [PATCH 0324/1025] Remove register output format method. --- mycli/output_formatter.py | 47 +++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 6a0564890..14339cb77 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -69,7 +69,21 @@ class OutputFormatter(object): def __init__(self, format_name=None): """Register the supported output formats.""" - self._output_formats = {} + self._output_formats = { + 'csv': (csv_wrapper, { + 'preprocessor': override_missing_value, + 'missing_value': '' + }), + 'tsv': (csv_wrapper, { + 'preprocessor': override_missing_value, + 'missing_value': '', + 'delimiter': '\t' + }), + 'expanded': (expanded_table, { + 'preprocessor': override_missing_value, + 'missing_value': '' + }) + } self._format_name = None tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', @@ -77,21 +91,11 @@ def __init__(self, format_name=None): 'moinmoin', 'html', 'latex', 'latex_booktabs', 'textile') for tabulate_format in tabulate_formats: - self.register_output_format(tabulate_format, tabulate_wrapper, - preprocessor=bytes_to_unicode, - table_format=tabulate_format, - missing_value='') - - self.register_output_format('csv', csv_wrapper, - preprocessor=override_missing_value, - missing_value='null') - self.register_output_format('tsv', csv_wrapper, delimiter='\t', - preprocessor=override_missing_value, - missing_value='null') - - self.register_output_format('expanded', expanded_table, - preprocessor=override_missing_value, - missing_value='') + self._output_formats[tabulate_format] = ( + tabulate_wrapper, {'preprocessor': bytes_to_unicode, + 'table_format': tabulate_format, + 'missing_value': ''} + ) if format_name: self.set_format_name(format_name) @@ -108,17 +112,6 @@ def get_format_name(self): """Get the OutputFormatter's default format.""" return self._format_name - def register_output_format(self, name, function, **kwargs): - """Register a new output format. - - *function* should be a callable that accepts the following arguments: - - *headers*: A list of headers for the output data. - - *data*: The data that needs formatting. - - *kwargs*: Any other keyword arguments for controlling the output. - It should return the formatted output as a string. - """ - self._output_formats[name] = (function, kwargs) - def supported_formats(self): """Return the supported output format names.""" return tuple(self._output_formats.keys()) From e7b93b8162b663ad745948ccbecb9f839c78d05f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:31:52 -0500 Subject: [PATCH 0325/1025] Remove print statement. --- mycli/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index fa715d26a..661dc65d2 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -180,7 +180,6 @@ def register_special_commands(self): '\\R', 'Change prompt format.', aliases=('\\R',), case_sensitive=True) def change_table_format(self, arg, **_): - print('change table: ' + arg) try: self.formatter.set_format_name(arg) yield (None, None, None, From b58cad04f7dda089612c5da03c93b87aa5b221a6 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:34:31 -0500 Subject: [PATCH 0326/1025] Update list of table formats. --- mycli/myclirc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/myclirc b/mycli/myclirc index ff6e8f734..36e9e409a 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -31,7 +31,8 @@ log_level = INFO timing = True # Table format. Possible values: psql, plain, simple, grid, fancy_grid, pipe, -# orgtbl, rst, mediawiki, html, latex, latex_booktabs, tsv. +# orgtbl, jira, rst, mediawiki, moinmoin, html, latex, latex_booktabs, +# textile, csv, tsv. # Recommended: psql, fancy_grid and grid. table_format = psql From cbd5ba0a58d699e402a1ef2c8b4665d3bfc375ae Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:43:03 -0500 Subject: [PATCH 0327/1025] Move bytes_to_hex to encoding utils package. --- mycli/encodingutils.py | 21 +++++++++++++++++++++ mycli/output_formatter.py | 20 +------------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index 29564d08e..ed8f9a158 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -1,8 +1,10 @@ +import binascii import sys PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 + def unicode2utf8(arg): """ Only in Python 2. Psycopg2 expects the args as bytes not unicode. @@ -13,6 +15,7 @@ def unicode2utf8(arg): return arg.encode('utf-8') return arg + def utf8tounicode(arg): """ Only in Python 2. Psycopg2 returns the error message as utf-8. @@ -22,3 +25,21 @@ def utf8tounicode(arg): if PY2 and isinstance(arg, str): return arg.decode('utf-8') return arg + + +def bytes_to_hex(b): + """Convert bytes that cannot be decoded to utf8 to hexlified string. + + >>> print(bytes_to_hex(b"\\xff")) + 0xff + >>> print(bytes_to_hex('abc')) + abc + >>> print(bytes_to_hex('✌')) + ✌ + """ + if isinstance(b, bytes): + try: + b.decode('utf8') + except: + b = '0x' + binascii.hexlify(b).decode('ascii') + return b diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 14339cb77..a1319fd8e 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals -import binascii import contextlib import csv try: @@ -13,6 +12,7 @@ from tabulate import tabulate +from .encodingutils import bytes_to_hex from .packages.expanded import expanded_table @@ -22,24 +22,6 @@ def override_missing_value(data, headers, missing_value='', **_): headers) -def bytes_to_hex(b): - """Convert bytes that cannot be decoded to utf8 to hexlified string. - - >>> print(bytes_to_hex(b"\\xff")) - 0xff - >>> print(bytes_to_hex('abc')) - abc - >>> print(bytes_to_hex('✌')) - ✌ - """ - if isinstance(b, bytes): - try: - b.decode('utf8') - except: - b = '0x' + binascii.hexlify(b).decode('ascii') - return b - - def bytes_to_unicode(data, headers, **_): """Convert all *data* and *headers* to unicode.""" return ([[bytes_to_hex(v) for v in row] for row in data], From ae648abb91f92645832c54675039ea3a7d8e0154 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 14:47:39 -0500 Subject: [PATCH 0328/1025] Fix encoding issue. --- mycli/encodingutils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index ed8f9a158..43ce6c7b6 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + import binascii import sys From 6ce7e158ed969a6d381f10cf0d1551ae63c04fd7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 15:27:56 -0500 Subject: [PATCH 0329/1025] Do not rely on tabulate for text type. --- mycli/encodingutils.py | 13 ++++++++++--- mycli/packages/expanded.py | 11 ++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index 43ce6c7b6..c5f32e9d2 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -7,6 +7,13 @@ PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 +if PY2: + text_type = unicode + binary_type = str +else: + text_type = str + binary_type = bytes + def unicode2utf8(arg): """ @@ -14,7 +21,7 @@ def unicode2utf8(arg): In Python 3 the args are expected as unicode. """ - if PY2 and isinstance(arg, unicode): + if PY2 and isinstance(arg, text_type): return arg.encode('utf-8') return arg @@ -25,7 +32,7 @@ def utf8tounicode(arg): In Python 3 the errors are returned as unicode. """ - if PY2 and isinstance(arg, str): + if PY2 and isinstance(arg, binary_type): return arg.decode('utf-8') return arg @@ -40,7 +47,7 @@ def bytes_to_hex(b): >>> print(bytes_to_hex('✌')) ✌ """ - if isinstance(b, bytes): + if isinstance(b, binary_type): try: b.decode('utf8') except: diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index b39d9507f..d79b9acd3 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,6 +1,7 @@ -from tabulate import _text_type import binascii +from .encodingutils import binary_type, text_type + def pad(field, total, char=u" "): return field + (char * (total - len(field))) @@ -12,12 +13,12 @@ def get_separator(num, header_len, data_len): def format_field(value): # Returns the field as a text type, otherwise will hexify the string try: - if isinstance(value, bytes): - return _text_type(value, "ascii") + if isinstance(value, binary_type): + return text_type(value, "ascii") else: - return _text_type(value) + return text_type(value) except UnicodeDecodeError: - return _text_type('0x' + binascii.hexlify(value).decode('ascii')) + return text_type('0x' + binascii.hexlify(value).decode('ascii')) def expanded_table(rows, headers, **_): header_len = max([len(x) for x in headers]) From 4a65f7bb98654f058e0f3cae1c06cf2f66bd33e2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 15:28:53 -0500 Subject: [PATCH 0330/1025] Fix import. --- mycli/packages/expanded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index d79b9acd3..d1108c2eb 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,6 +1,6 @@ import binascii -from .encodingutils import binary_type, text_type +from mycli.encodingutils import binary_type, text_type def pad(field, total, char=u" "): return field + (char * (total - len(field))) From 02f5c521495eec0968d99fdf603be958968a2a5e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:24:03 -0500 Subject: [PATCH 0331/1025] Move pre-processing to output_formatter. --- mycli/encodingutils.py | 16 +++++++------- mycli/main.py | 2 -- mycli/output_formatter.py | 45 +++++++++++++++++++++++++------------- mycli/packages/expanded.py | 29 ++++++++++-------------- tests/test_expanded.py | 3 ++- 5 files changed, 52 insertions(+), 43 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index c5f32e9d2..aac4aa07d 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -37,19 +37,19 @@ def utf8tounicode(arg): return arg -def bytes_to_hex(b): - """Convert bytes that cannot be decoded to utf8 to hexlified string. +def bytes_to_string(b): + """Convert bytes to a string. Hexlify bytes that can't be decoded. - >>> print(bytes_to_hex(b"\\xff")) + >>> print(bytes_to_string(b"\\xff")) 0xff - >>> print(bytes_to_hex('abc')) + >>> print(bytes_to_string('abc')) abc - >>> print(bytes_to_hex('✌')) + >>> print(bytes_to_string('✌')) ✌ """ if isinstance(b, binary_type): try: - b.decode('utf8') - except: - b = '0x' + binascii.hexlify(b).decode('ascii') + return b.decode('utf8') + except UnicodeDecodeError: + return '0x' + binascii.hexlify(b).decode('ascii') return b diff --git a/mycli/main.py b/mycli/main.py index 661dc65d2..faca8b37d 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -722,8 +722,6 @@ def format_output(self, title, cur, headers, status, expanded=False, output.append(title) if cur: - headers = [utf8tounicode(x) for x in headers] - rows = list(cur) formatted = self.formatter.format_output( rows, headers, format_name='expanded' if expanded else None) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index a1319fd8e..5474b41f6 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -12,20 +12,34 @@ from tabulate import tabulate -from .encodingutils import bytes_to_hex +from . import encodingutils from .packages.expanded import expanded_table +def to_string(value): + """Convert *value* to a string.""" + if isinstance(value, encodingutils.binary_type): + return encodingutils.bytes_to_string(value) + else: + return encodingutils.text_type(value) + + +def convert_to_string(data, headers, **_): + """Convert all *data* and *headers* to strings.""" + return ([[to_string(v) for v in row] for row in data], + [to_string(h) for h in headers]) + + def override_missing_value(data, headers, missing_value='', **_): """Override missing values in the data with *missing_value*.""" return ([[missing_value if v is None else v for v in row] for row in data], headers) -def bytes_to_unicode(data, headers, **_): - """Convert all *data* and *headers* to unicode.""" - return ([[bytes_to_hex(v) for v in row] for row in data], - [bytes_to_hex(h) for h in headers]) +def bytes_to_string(data, headers, **_): + """Convert all *data* and *headers* to strings.""" + return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], + [encodingutils.bytes_to_string(h) for h in headers]) def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): @@ -53,16 +67,16 @@ def __init__(self, format_name=None): """Register the supported output formats.""" self._output_formats = { 'csv': (csv_wrapper, { - 'preprocessor': override_missing_value, + 'preprocessor': (override_missing_value, bytes_to_string), 'missing_value': '' }), 'tsv': (csv_wrapper, { - 'preprocessor': override_missing_value, + 'preprocessor': (override_missing_value, bytes_to_string), 'missing_value': '', 'delimiter': '\t' }), 'expanded': (expanded_table, { - 'preprocessor': override_missing_value, + 'preprocessor': (override_missing_value, convert_to_string), 'missing_value': '' }) } @@ -73,11 +87,11 @@ def __init__(self, format_name=None): 'moinmoin', 'html', 'latex', 'latex_booktabs', 'textile') for tabulate_format in tabulate_formats: - self._output_formats[tabulate_format] = ( - tabulate_wrapper, {'preprocessor': bytes_to_unicode, - 'table_format': tabulate_format, - 'missing_value': ''} - ) + self._output_formats[tabulate_format] = (tabulate_wrapper, { + 'preprocessor': (bytes_to_string, ), + 'table_format': tabulate_format, + 'missing_value': '' + }) if format_name: self.set_format_name(format_name) @@ -111,7 +125,8 @@ def format_output(self, data, headers, format_name=None, **kwargs): function, fkwargs = self._output_formats[format_name] fkwargs.update(kwargs) - preprocessor = fkwargs.pop('preprocessor', None) + preprocessor = fkwargs.get('preprocessor', None) if preprocessor: - data, headers = preprocessor(data, headers, **fkwargs) + for f in preprocessor: + data, headers = f(data, headers, **fkwargs) return function(data, headers, **fkwargs) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index d1108c2eb..c3a8e9a53 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -1,42 +1,37 @@ -import binascii +"""Format data into a vertical, expanded table layout.""" -from mycli.encodingutils import binary_type, text_type +from __future__ import unicode_literals -def pad(field, total, char=u" "): + +def pad(field, total, char=' '): return field + (char * (total - len(field))) -def get_separator(num, header_len, data_len): - sep = u"***************************[ %d. row ]***************************\n" % (num + 1) +def get_separator(num, header_len, data_len): + sep = "***************************[ %d. row ]***************************\n" % (num + 1) return sep -def format_field(value): - # Returns the field as a text type, otherwise will hexify the string - try: - if isinstance(value, binary_type): - return text_type(value, "ascii") - else: - return text_type(value) - except UnicodeDecodeError: - return text_type('0x' + binascii.hexlify(value).decode('ascii')) def expanded_table(rows, headers, **_): + """Format *rows* and *headers* as an expanded table. + + The values in *rows* and *headers* must be strings. + """ header_len = max([len(x) for x in headers]) max_row_len = 0 results = [] - padded_headers = [pad(x, header_len) + u" |" for x in headers] + padded_headers = [pad(x, header_len) + ' |' for x in headers] header_len += 2 for row in rows: - row = [format_field(x) for x in row] row_len = max([len(x) for x in row]) row_result = [] if row_len > max_row_len: max_row_len = row_len for header, value in zip(padded_headers, row): - row_result.append(u"%s %s" % (header, value)) + row_result.append('{0} {1}'.format(header, value)) results.append('\n'.join(row_result)) diff --git a/tests/test_expanded.py b/tests/test_expanded.py index 9b2a6c5cf..cec940696 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -1,7 +1,8 @@ from mycli.packages.expanded import expanded_table +from mycli.encodingutils import text_type def test_expanded_table_renders(): - input = [("hello", 123), ("world", 456)] + input = [("hello", text_type(123)), ("world", text_type(456))] expected = """***************************[ 1. row ]*************************** name | hello From 5fe9d68c9679fe7e40bbec719a05a8c4d9053dfb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:46:23 -0500 Subject: [PATCH 0332/1025] Refactor expanded table. --- mycli/packages/expanded.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index c3a8e9a53..0b9e6ef1d 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -3,13 +3,16 @@ from __future__ import unicode_literals -def pad(field, total, char=' '): - return field + (char * (total - len(field))) +def get_separator(num, header_len): + """Get a row separator.""" + sep = "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) + return sep -def get_separator(num, header_len, data_len): - sep = "***************************[ %d. row ]***************************\n" % (num + 1) - return sep +def format_row(headers, row): + """Format a row.""" + formatted_row = [' '.join(field) for field in zip(headers, row)] + return '\n'.join(formatted_row) def expanded_table(rows, headers, **_): @@ -18,26 +21,12 @@ def expanded_table(rows, headers, **_): The values in *rows* and *headers* must be strings. """ header_len = max([len(x) for x in headers]) - max_row_len = 0 - results = [] - - padded_headers = [pad(x, header_len) + ' |' for x in headers] - header_len += 2 - - for row in rows: - row_len = max([len(x) for x in row]) - row_result = [] - if row_len > max_row_len: - max_row_len = row_len - - for header, value in zip(padded_headers, row): - row_result.append('{0} {1}'.format(header, value)) - - results.append('\n'.join(row_result)) + padded_headers = ['{} |'.format(x.ljust(header_len)) for x in headers] + results = [format_row(padded_headers, row) for row in rows] output = [] for i, result in enumerate(results): - output.append(get_separator(i, header_len, max_row_len)) + output.append(get_separator(i, header_len + 2)) output.append(result) output.append('\n') From 2052ae45925ce10412b1b1cb26a8cab593d51e0e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:48:22 -0500 Subject: [PATCH 0333/1025] Remove extra variable. --- mycli/packages/expanded.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 0b9e6ef1d..e4283d73f 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -5,8 +5,7 @@ def get_separator(num, header_len): """Get a row separator.""" - sep = "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) - return sep + return "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) def format_row(headers, row): From 328a7673e13ee40f1518edd3231b8a93471ece32 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:50:12 -0500 Subject: [PATCH 0334/1025] Remove extra argument to get_separator. --- mycli/packages/expanded.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index e4283d73f..83ad4276c 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -3,8 +3,8 @@ from __future__ import unicode_literals -def get_separator(num, header_len): - """Get a row separator.""" +def get_separator(num): + """Get a row separator for row *num*.""" return "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) @@ -25,7 +25,7 @@ def expanded_table(rows, headers, **_): output = [] for i, result in enumerate(results): - output.append(get_separator(i, header_len + 2)) + output.append(get_separator(i)) output.append(result) output.append('\n') From 041eafbca08a1729698b59f6fee252ef96c686eb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:52:09 -0500 Subject: [PATCH 0335/1025] Use named format strings for clarity. --- mycli/packages/expanded.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 83ad4276c..3ec938291 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -5,7 +5,8 @@ def get_separator(num): """Get a row separator for row *num*.""" - return "{0}[ {1}. row ]{2}\n".format('*' * 27, num + 1, '*' * 27) + return "{divider}[ {n}. row ]{divider}\n".format( + divider='*' * 27, n=num + 1) def format_row(headers, row): From 3a10f5bb0c4ca5b4d099541591fd9618ce5e6d23 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 16:56:36 -0500 Subject: [PATCH 0336/1025] Move divider to format_row function. --- mycli/packages/expanded.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mycli/packages/expanded.py b/mycli/packages/expanded.py index 3ec938291..2ab579944 100644 --- a/mycli/packages/expanded.py +++ b/mycli/packages/expanded.py @@ -11,7 +11,7 @@ def get_separator(num): def format_row(headers, row): """Format a row.""" - formatted_row = [' '.join(field) for field in zip(headers, row)] + formatted_row = [' | '.join(field) for field in zip(headers, row)] return '\n'.join(formatted_row) @@ -21,11 +21,11 @@ def expanded_table(rows, headers, **_): The values in *rows* and *headers* must be strings. """ header_len = max([len(x) for x in headers]) - padded_headers = ['{} |'.format(x.ljust(header_len)) for x in headers] - results = [format_row(padded_headers, row) for row in rows] + padded_headers = [x.ljust(header_len) for x in headers] + formatted_rows = [format_row(padded_headers, row) for row in rows] output = [] - for i, result in enumerate(results): + for i, result in enumerate(formatted_rows): output.append(get_separator(i)) output.append(result) output.append('\n') From 61bdbd98c5569b84a7b994bd67a457d4bfe62a2a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 21:03:17 -0500 Subject: [PATCH 0337/1025] Make the expanded format test more readable. --- tests/test_expanded.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/test_expanded.py b/tests/test_expanded.py index cec940696..db503cba8 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -1,14 +1,19 @@ +"""Test the vertical, expanded table formatter.""" +from textwrap import dedent + from mycli.packages.expanded import expanded_table from mycli.encodingutils import text_type + def test_expanded_table_renders(): - input = [("hello", text_type(123)), ("world", text_type(456))] - - expected = """***************************[ 1. row ]*************************** -name | hello -age | 123 -***************************[ 2. row ]*************************** -name | world -age | 456 -""" - assert expected == expanded_table(input, ["name", "age"]) + results = [('hello', text_type(123)), ('world', text_type(456))] + + expected = dedent("""\ + ***************************[ 1. row ]*************************** + name | hello + age | 123 + ***************************[ 2. row ]*************************** + name | world + age | 456 + """) + assert expected == expanded_table(results, ('name', 'age')) From 0f01a9d5767e1dc0d079633e806770099255af28 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 1 Apr 2017 21:08:44 -0500 Subject: [PATCH 0338/1025] Update docstring to be clearer. --- mycli/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5474b41f6..86db809c0 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -37,7 +37,7 @@ def override_missing_value(data, headers, missing_value='', **_): def bytes_to_string(data, headers, **_): - """Convert all *data* and *headers* to strings.""" + """Convert all *data* and *headers* bytes to strings.""" return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], [encodingutils.bytes_to_string(h) for h in headers]) From 4b47e43eb8b7d332d64bd6c2f6575e34d3451b67 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Thu, 23 Mar 2017 12:58:37 +0100 Subject: [PATCH 0339/1025] Use pymysql default conversions (instead of only our conversions) This will convert a MySQL DECIMAL to the Python's decimal.Decimal type. Tabulate handles Decimal as a string (and not a number) and will not remove the padding (fixes Issue#375) --- changelog.md | 1 + mycli/sqlexecute.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 924d8efa4..f87407e34 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,7 @@ Bug Fixes: * Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) * Fix bug where mycli would not start due to the thanks/credit intro text. (Thanks: [Thomas Roten]). +* Use pymysql default conversions (issue #375). (Thanks: [Dick Marinus]). Internal Changes: ----------------- diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 786c8f0ec..08030af47 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -4,7 +4,7 @@ from .packages import special from pymysql.constants import FIELD_TYPE from pymysql.converters import (convert_mysql_timestamp, convert_datetime, - convert_timedelta, convert_date) + convert_timedelta, convert_date, conversions) _logger = logging.getLogger(__name__) @@ -65,12 +65,13 @@ def connect(self, database=None, user=None, password=None, host=None, '\tlocal_infile: %r' '\tssl: %r', database, user, host, port, socket, charset, local_infile, ssl) - conv = { + conv = conversions.copy() + conv.update({ FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), - } + }) conn = pymysql.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, From 5839c11d641f97660a6cded8e6a354ec54d72ce1 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sun, 2 Apr 2017 18:42:54 +0200 Subject: [PATCH 0340/1025] align floats on decimal point, quote columns with trailing/starting whitespace --- mycli/output_formatter.py | 97 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5474b41f6..35e266749 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -15,6 +15,8 @@ from . import encodingutils from .packages.expanded import expanded_table +from decimal import Decimal + def to_string(value): """Convert *value* to a string.""" @@ -42,6 +44,87 @@ def bytes_to_string(data, headers, **_): [encodingutils.bytes_to_string(h) for h in headers]) +def intlen(value): + """Find (character) length + >>> intlen('11.1') + 2 + >>> intlen('11') + 2 + >>> intlen('1.1') + 1 + """ + pos = value.find('.') + if pos < 0: + pos = len(value) + return pos + +def align_decimals(data, headers, **_): + """Align decimals to decimal point + >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) + 1 + 11.1 + 1.1 + """ + pointpos = len(data[0]) * [0] + for row in data: + i = 0 + for v in row: + if isinstance(v, Decimal): + v = str(v) + pointpos[i] = max(intlen(v), pointpos[i]) + i += 1 + results = [] + for row in data: + i = 0 + result = [] + for v in row: + if isinstance(v, Decimal): + v = str(v) + result.append((pointpos[i]-intlen(v))*" "+v) + else: + result.append(v) + i += 1 + results.append(result) + return results, headers + + +def quote_whitespaces(data, headers, quotestyle="'", **_): + """Quote whitespace + >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) + ' before' + 'after ' + ' both ' + 'none' + >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) + abc + def + ghi + jkl + """ + """Convert all *data* and *headers* to strings.""" + quote = len(data[0])*[False] + for row in data: + i = 0 + for v in row: + v = encodingutils.text_type(v) + if v[0] == ' ' or v[-1] == ' ': + quote[i] = True + i += 1 + + results = [] + for row in data: + result = [] + i = 0 + for v in row: + if quote[i]: + result.append('{quotestyle}{value}{quotestyle}'.format(quotestyle=quotestyle, value=v)) + else: + result.append(v) + i += 1 + results.append(result) + return results, headers + + def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, @@ -88,7 +171,7 @@ def __init__(self, format_name=None): 'textile') for tabulate_format in tabulate_formats: self._output_formats[tabulate_format] = (tabulate_wrapper, { - 'preprocessor': (bytes_to_string, ), + 'preprocessor': (bytes_to_string, align_decimals, quote_whitespaces), 'table_format': tabulate_format, 'missing_value': '' }) @@ -118,6 +201,18 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. + >>> print(OutputFormatter().format_output( \ + [["abc", Decimal(1)], ["defg", Decimal('11.1')], ["hi", Decimal('1.1')]], \ + ["text", "numeric"], \ + "psql" \ + )) + +--------+-----------+ + | text | numeric | + |--------+-----------| + | abc | ' 1' | + | defg | '11.1' | + | hi | ' 1.1' | + +--------+-----------+ """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): From 1db7f865b91f57baa19fd27234506d6974895801 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 13:24:47 -0500 Subject: [PATCH 0341/1025] Add basic output_formatter tests. --- tests/test_output_formatter.py | 82 ++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tests/test_output_formatter.py diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py new file mode 100644 index 000000000..f331262e8 --- /dev/null +++ b/tests/test_output_formatter.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +"""Test the generic output formatter interface.""" + +from __future__ import unicode_literals + +from textwrap import dedent + +from mycli.output_formatter import (bytes_to_string, convert_to_string, + csv_wrapper, OutputFormatter, + override_missing_value, tabulate_wrapper, + to_string) + + +def test_to_string(): + """Test the *output_formatter.to_string()* function.""" + assert 'a' == to_string('a') + assert 'a' == to_string(b'a') + assert '1' == to_string(1) + assert '1.23' == to_string(1.23) + + +def test_convert_to_string(): + """Test the *output_formatter.convert_to_string()* function.""" + data = [[1, 'John'], [2, 'Jill']] + headers = [0, 'name'] + expected = ([['1', 'John'], ['2', 'Jill']], ['0', 'name']) + + assert expected == convert_to_string(data, headers) + + +def test_override_missing_values(): + """Test the *output_formatter.override_missing_values()* function.""" + data = [[1, None], [2, 'Jill']] + headers = [0, 'name'] + expected = ([[1, ''], [2, 'Jill']], [0, 'name']) + + assert expected == override_missing_value(data, headers, + missing_value='') + + +def test_bytes_to_string(): + """Test the *output_formatter.bytes_to_string()* function.""" + data = [[1, 'John'], [2, b'Jill']] + headers = [0, 'name'] + expected = ([[1, 'John'], [2, 'Jill']], [0, 'name']) + + assert expected == bytes_to_string(data, headers) + + +def test_tabulate_wrapper(): + """Test the *output_formatter.tabulate_wrapper()* function.""" + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = tabulate_wrapper(data, headers, table_format='psql') + assert output == dedent('''\ + +-----------+----------+ + | letters | number | + |-----------+----------| + | abc | 1 | + | d | 456 | + +-----------+----------+''') + + +def test_csv_wrapper(): + """Test the *output_formatter.csv_wrapper()* function.""" + # Test comma-delimited output. + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = csv_wrapper(data, headers) + assert output == dedent('''\ + letters,number\r\n\ + abc,1\r\n\ + d,456\r\n''') + + # Test tab-delimited output. + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = csv_wrapper(data, headers, delimiter='\t') + assert output == dedent('''\ + letters\tnumber\r\n\ + abc\t1\r\n\ + d\t456\r\n''') From 86846167f1fd159a99b89f75d4e24d8290eff85e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 14:28:30 -0500 Subject: [PATCH 0342/1025] Add terminaltables. --- mycli/output_formatter.py | 33 ++++++++++++++++++++++++++++++--- setup.py | 1 + tests/test_output_formatter.py | 16 +++++++++++++++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index bc275984e..566a337f9 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -5,18 +5,18 @@ import contextlib import csv +from decimal import Decimal try: from cStringIO import StringIO except ImportError: from io import StringIO from tabulate import tabulate +import terminaltables from . import encodingutils from .packages.expanded import expanded_table -from decimal import Decimal - def to_string(value): """Convert *value* to a string.""" @@ -143,6 +143,23 @@ def csv_wrapper(data, headers, delimiter=',', **_): return content.getvalue() +def terminal_tables_wrapper(data, headers, table_format=None, **_): + """Wrap terminaltables inside a standard function for OutputFormatter.""" + if table_format == 'ascii': + table = terminaltables.AsciiTable + elif table_format == 'single': + table = terminaltables.SingleTable + elif table_format == 'double': + table = terminaltables.DoubleTable + elif table_format == 'github': + table = terminaltables.GithubFlavoredMarkdownTable + else: + raise ValueError('unrecognized table format: {}'.format(table_format)) + + t = table([headers] + data) + return t.table + + class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" @@ -171,11 +188,21 @@ def __init__(self, format_name=None): 'textile') for tabulate_format in tabulate_formats: self._output_formats[tabulate_format] = (tabulate_wrapper, { - 'preprocessor': (bytes_to_string, align_decimals, quote_whitespaces), + 'preprocessor': (bytes_to_string, align_decimals), 'table_format': tabulate_format, 'missing_value': '' }) + terminal_tables_formats = ('ascii', 'single', 'double', 'github') + for terminal_tables_format in terminal_tables_formats: + self._output_formats[terminal_tables_format] = ( + terminal_tables_wrapper, { + 'preprocessor': (bytes_to_string, override_missing_value, + align_decimals), + 'table_format': terminal_tables_format, + 'missing_value': '' + }) + if format_name: self.set_format_name(format_name) diff --git a/setup.py b/setup.py index 3ecc21c48..21e555086 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'configobj >= 5.0.5', 'pycryptodome >= 3', 'tabulate >= 0.7.6', + 'terminaltables >= 3.0.0', ] setup( diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index f331262e8..66ac42375 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -8,7 +8,7 @@ from mycli.output_formatter import (bytes_to_string, convert_to_string, csv_wrapper, OutputFormatter, override_missing_value, tabulate_wrapper, - to_string) + terminal_tables_wrapper, to_string) def test_to_string(): @@ -80,3 +80,17 @@ def test_csv_wrapper(): letters\tnumber\r\n\ abc\t1\r\n\ d\t456\r\n''') + + +def test_terminal_tables_wrapper(): + """Test the *output_formatter.terminal_tables_wrapper()* function.""" + data = [['abc', 1], ['d', 456]] + headers = ['letters', 'number'] + output = terminal_tables_wrapper(data, headers, table_format='ascii') + assert output == dedent('''\ + +---------+--------+ + | letters | number | + +---------+--------+ + | abc | 1 | + | d | 456 | + +---------+--------+''') From 4c13163941b64dab71bad9a91906351f606c0e06 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 14:43:49 -0500 Subject: [PATCH 0343/1025] Default myclirc to ascii table. --- mycli/myclirc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mycli/myclirc b/mycli/myclirc index 36e9e409a..2f3599273 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -30,11 +30,9 @@ log_level = INFO # Timing of sql statments and table rendering. timing = True -# Table format. Possible values: psql, plain, simple, grid, fancy_grid, pipe, -# orgtbl, jira, rst, mediawiki, moinmoin, html, latex, latex_booktabs, -# textile, csv, tsv. -# Recommended: psql, fancy_grid and grid. -table_format = psql +# Table format. Possible values: ascii, single, double, or github. +# Recommended: ascii +table_format = ascii # Syntax coloring style. Possible values (many support the "-dark" suffix): # manni, igor, xcode, vim, autumn, vs, rrt, native, perldoc, borland, tango, emacs, From dd41cf394602523331d5c57152a6f223a7c52d06 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 14:43:58 -0500 Subject: [PATCH 0344/1025] Fix table format tests. --- mycli/output_formatter.py | 16 ++++++++-------- tests/test_main.py | 14 +++++++------- tests/test_sqlexecute.py | 32 ++++++++++++++++---------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 566a337f9..5e3305318 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -231,15 +231,15 @@ def format_output(self, data, headers, format_name=None, **kwargs): >>> print(OutputFormatter().format_output( \ [["abc", Decimal(1)], ["defg", Decimal('11.1')], ["hi", Decimal('1.1')]], \ ["text", "numeric"], \ - "psql" \ + "ascii" \ )) - +--------+-----------+ - | text | numeric | - |--------+-----------| - | abc | ' 1' | - | defg | '11.1' | - | hi | ' 1.1' | - +--------+-----------+ + +------+---------+ + | text | numeric | + +------+---------+ + | abc | 1 | + | defg | 11.1 | + | hi | 1.1 | + +------+---------+ """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): diff --git a/tests/test_main.py b/tests/test_main.py index 29501f137..1271f18da 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -48,7 +48,7 @@ def test_execute_arg_with_table(executor): sql = 'select * from test;' runner = CliRunner() result = runner.invoke(cli, args=CLI_ARGS + ['-e', sql] + ['--table']) - expected = '+-----+\n| a |\n|-----|\n| abc |\n+-----+\n' + expected = '+-----+\n| a |\n+-----+\n| abc |\n+-----+\n' assert result.exit_code == 0 assert expected in result.output @@ -98,14 +98,14 @@ def test_batch_mode_table(executor): result = runner.invoke(cli, args=CLI_ARGS + ['-t'], input=sql) expected = (dedent("""\ - +------------+ - | count(*) | - |------------| - | 3 | - +------------+ + +----------+ + | count(*) | + +----------+ + | 3 | + +----------+ +-----+ | a | - |-----| + +-----+ | abc | +-----+""")) diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py index d964f43ba..a9b5fbec2 100644 --- a/tests/test_sqlexecute.py +++ b/tests/test_sqlexecute.py @@ -15,7 +15,7 @@ def test_conn(executor): assert results == dedent("""\ +-----+ | a | - |-----| + +-----+ | abc | +-----+ 1 row in set""") @@ -26,11 +26,11 @@ def test_bools(executor): run(executor, '''insert into test values(True)''') results = run(executor, '''select * from test''', join=True) assert results == dedent("""\ - +-----+ - | a | - |-----| - | 1 | - +-----+ + +---+ + | a | + +---+ + | 1 | + +---+ 1 row in set""") @dbtest @@ -41,7 +41,7 @@ def test_binary(executor): assert results == dedent("""\ +----------------------------------------------------------------------------------------------+ | geom | - |----------------------------------------------------------------------------------------------| + +----------------------------------------------------------------------------------------------+ | 0x00000000010200000002000000397f130a11185d4034f44f70b1de43400000000000185d40423ee8d9acde4340 | +----------------------------------------------------------------------------------------------+ 1 row in set""") @@ -140,7 +140,7 @@ def test_favorite_query(executor): > select * from test where a like 'a%' +-----+ | a | - |-----| + +-----+ | abc | +-----+""") @@ -163,13 +163,13 @@ def test_favorite_query_multiple_statement(executor): > select * from test where a like 'a%' +-----+ | a | - |-----| + +-----+ | abc | +-----+ > select * from test where a like 'd%' +-----+ | a | - |-----| + +-----+ | def | +-----+""") @@ -261,13 +261,13 @@ def test_favorite_query_multiline_statement(executor): > select * from test where a like 'a%' +-----+ | a | - |-----| + +-----+ | abc | +-----+ > select * from test where a like 'd%' +-----+ | a | - |-----| + +-----+ | def | +-----+""") @@ -282,7 +282,7 @@ def test_timestamp_null(executor): assert results == dedent("""\ +---------------------+ | a | - |---------------------| + +---------------------+ | 0000-00-00 00:00:00 | +---------------------+ 1 row in set""") @@ -295,7 +295,7 @@ def test_datetime_null(executor): assert results == dedent("""\ +---------------------+ | a | - |---------------------| + +---------------------+ | 0000-00-00 00:00:00 | +---------------------+ 1 row in set""") @@ -308,7 +308,7 @@ def test_date_null(executor): assert results == dedent("""\ +------------+ | a | - |------------| + +------------+ | 0000-00-00 | +------------+ 1 row in set""") @@ -321,7 +321,7 @@ def test_time_null(executor): assert results == dedent("""\ +----------+ | a | - |----------| + +----------+ | 00:00:00 | +----------+ 1 row in set""") From 145d5d2af0e5bf15fdaa3ffbf443eb2942244691 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 15:17:34 -0500 Subject: [PATCH 0345/1025] Add a multi-column test for align_decimals. --- mycli/output_formatter.py | 28 ++++++++++------------------ tests/test_output_formatter.py | 19 +++++++++++++++---- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 5e3305318..068f0246b 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -67,23 +67,19 @@ def align_decimals(data, headers, **_): """ pointpos = len(data[0]) * [0] for row in data: - i = 0 - for v in row: + for i, v in enumerate(row): if isinstance(v, Decimal): - v = str(v) + v = encodingutils.text_type(v) pointpos[i] = max(intlen(v), pointpos[i]) - i += 1 results = [] for row in data: - i = 0 result = [] - for v in row: + for i, v in enumerate(row): if isinstance(v, Decimal): - v = str(v) - result.append((pointpos[i]-intlen(v))*" "+v) + v = encodingutils.text_type(v) + result.append((pointpos[i] - intlen(v)) * " " + v) else: result.append(v) - i += 1 results.append(result) return results, headers @@ -101,26 +97,22 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): ghi jkl """ - """Convert all *data* and *headers* to strings.""" - quote = len(data[0])*[False] + quote = len(data[0]) * [False] for row in data: - i = 0 - for v in row: + for i, v in enumerate(row): v = encodingutils.text_type(v) if v[0] == ' ' or v[-1] == ' ': quote[i] = True - i += 1 results = [] for row in data: result = [] - i = 0 - for v in row: + for i, v in enumerate(row): if quote[i]: - result.append('{quotestyle}{value}{quotestyle}'.format(quotestyle=quotestyle, value=v)) + result.append('{quotestyle}{value}{quotestyle}'.format( + quotestyle=quotestyle, value=v)) else: result.append(v) - i += 1 results.append(result) return results, headers diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 66ac42375..045bd730e 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -3,12 +3,14 @@ from __future__ import unicode_literals +from decimal import Decimal from textwrap import dedent -from mycli.output_formatter import (bytes_to_string, convert_to_string, - csv_wrapper, OutputFormatter, - override_missing_value, tabulate_wrapper, - terminal_tables_wrapper, to_string) +from mycli.output_formatter import (align_decimals, bytes_to_string, + convert_to_string, csv_wrapper, + OutputFormatter, override_missing_value, + tabulate_wrapper, terminal_tables_wrapper, + to_string) def test_to_string(): @@ -47,6 +49,15 @@ def test_bytes_to_string(): assert expected == bytes_to_string(data, headers) +def test_align_decimals(): + """Test the *output_formatter.align_decimals()* function.""" + data = [[Decimal('200'), Decimal('1')], [Decimal('1.00002'), Decimal('1.0')]] + headers = ['num1', 'num2'] + expected = ([['200', '1'], [' 1.00002', '1.0']], ['num1', 'num2']) + + assert expected == align_decimals(data, headers) + + def test_tabulate_wrapper(): """Test the *output_formatter.tabulate_wrapper()* function.""" data = [['abc', 1], ['d', 456]] From 6ea35bad05cee2df19e37fd19301b1129636e264 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 19:19:30 -0500 Subject: [PATCH 0346/1025] Format/idiomatic changes. --- mycli/output_formatter.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 068f0246b..549ee4e8b 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -101,18 +101,16 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): for row in data: for i, v in enumerate(row): v = encodingutils.text_type(v) - if v[0] == ' ' or v[-1] == ' ': + if v.startswith(' ') or v.endswith(' '): quote[i] = True results = [] for row in data: result = [] for i, v in enumerate(row): - if quote[i]: - result.append('{quotestyle}{value}{quotestyle}'.format( - quotestyle=quotestyle, value=v)) - else: - result.append(v) + quotation = quotestyle if quote[i] else '' + result.append('{quotestyle}{value}{quotestyle}'.format( + quotestyle=quotation, value=v)) results.append(result) return results, headers @@ -192,8 +190,8 @@ def __init__(self, format_name=None): 'preprocessor': (bytes_to_string, override_missing_value, align_decimals), 'table_format': terminal_tables_format, - 'missing_value': '' - }) + 'missing_value': ''} + ) if format_name: self.set_format_name(format_name) @@ -220,10 +218,12 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. - >>> print(OutputFormatter().format_output( \ - [["abc", Decimal(1)], ["defg", Decimal('11.1')], ["hi", Decimal('1.1')]], \ - ["text", "numeric"], \ - "ascii" \ + + >>> print(OutputFormatter().format_output(\ + [["abc", Decimal(1)], ["defg", Decimal('11.1')],\ + ["hi", Decimal('1.1')]],\ + ["text", "numeric"],\ + "ascii"\ )) +------+---------+ | text | numeric | From c51cfac3c1458a07a2f9b8acb59b2ef2f9321fbb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 2 Apr 2017 19:19:53 -0500 Subject: [PATCH 0347/1025] Move format_output tests to tests module. --- mycli/output_formatter.py | 14 -------------- tests/test_output_formatter.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py index 549ee4e8b..6656ec8c1 100644 --- a/mycli/output_formatter.py +++ b/mycli/output_formatter.py @@ -218,20 +218,6 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. - - >>> print(OutputFormatter().format_output(\ - [["abc", Decimal(1)], ["defg", Decimal('11.1')],\ - ["hi", Decimal('1.1')]],\ - ["text", "numeric"],\ - "ascii"\ - )) - +------+---------+ - | text | numeric | - +------+---------+ - | abc | 1 | - | defg | 11.1 | - | hi | 1.1 | - +------+---------+ """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 045bd730e..8ddeb888c 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -105,3 +105,21 @@ def test_terminal_tables_wrapper(): | abc | 1 | | d | 456 | +---------+--------+''') + + +def test_output_formatter(): + """Test the *output_formatter.OutputFormatter* class.""" + data = [['abc', Decimal(1)], ['defg', Decimal('11.1')], + ['hi', Decimal('1.1')]] + headers = ['text', 'numeric'] + expected = dedent('''\ + +------+---------+ + | text | numeric | + +------+---------+ + | abc | 1 | + | defg | 11.1 | + | hi | 1.1 | + +------+---------+''') + + assert expected == OutputFormatter().format_output(data, headers, + format_name='ascii') From a36cf85b53f36221032acb6e2ee704e0900d3a0d Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 17:12:48 -0700 Subject: [PATCH 0348/1025] Refactor OutputFormatter to remove tight coupling. --- mycli/main.py | 7 +- mycli/output_formatter.py | 232 ------------------ mycli/output_formatter/__init__.py | 0 .../delimited_output_adapter.py | 22 ++ mycli/output_formatter/output_formatter.py | 91 +++++++ mycli/output_formatter/preprocessors.py | 98 ++++++++ mycli/output_formatter/tabulate_adapter.py | 14 ++ .../terminaltables_adapter.py | 24 ++ mycli/sqlcompleter.py | 6 +- 9 files changed, 255 insertions(+), 239 deletions(-) delete mode 100644 mycli/output_formatter.py create mode 100644 mycli/output_formatter/__init__.py create mode 100644 mycli/output_formatter/delimited_output_adapter.py create mode 100644 mycli/output_formatter/output_formatter.py create mode 100644 mycli/output_formatter/preprocessors.py create mode 100644 mycli/output_formatter/tabulate_adapter.py create mode 100644 mycli/output_formatter/terminaltables_adapter.py diff --git a/mycli/main.py b/mycli/main.py index faca8b37d..0b948a85e 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -37,7 +37,7 @@ from .config import (write_default_config, get_mylogin_cnf_path, open_mylogin_cnf, read_config_files, str_to_bool) from .key_bindings import mycli_bindings -from .output_formatter import OutputFormatter +from .output_formatter import output_formatter from .encodingutils import utf8tounicode from .lexer import MyCliLexer from .__init__ import __version__ @@ -106,7 +106,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.formatter = OutputFormatter(format_name=c['main']['table_format']) + self.formatter = output_formatter.OutputFormatter(format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] @@ -145,7 +145,8 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. self.smart_completion = c['main'].as_bool('smart_completion') - self.completer = SQLCompleter(self.smart_completion) + self.completer = SQLCompleter(self.smart_completion, + supported_formats=self.formatter.supported_formats) self._completer_lock = threading.Lock() # Register custom special commands. diff --git a/mycli/output_formatter.py b/mycli/output_formatter.py deleted file mode 100644 index 6656ec8c1..000000000 --- a/mycli/output_formatter.py +++ /dev/null @@ -1,232 +0,0 @@ -# -*- coding: utf-8 -*- -"""A generic output formatter interface.""" - -from __future__ import unicode_literals - -import contextlib -import csv -from decimal import Decimal -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -from tabulate import tabulate -import terminaltables - -from . import encodingutils -from .packages.expanded import expanded_table - - -def to_string(value): - """Convert *value* to a string.""" - if isinstance(value, encodingutils.binary_type): - return encodingutils.bytes_to_string(value) - else: - return encodingutils.text_type(value) - - -def convert_to_string(data, headers, **_): - """Convert all *data* and *headers* to strings.""" - return ([[to_string(v) for v in row] for row in data], - [to_string(h) for h in headers]) - - -def override_missing_value(data, headers, missing_value='', **_): - """Override missing values in the data with *missing_value*.""" - return ([[missing_value if v is None else v for v in row] for row in data], - headers) - - -def bytes_to_string(data, headers, **_): - """Convert all *data* and *headers* bytes to strings.""" - return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], - [encodingutils.bytes_to_string(h) for h in headers]) - - -def intlen(value): - """Find (character) length - >>> intlen('11.1') - 2 - >>> intlen('11') - 2 - >>> intlen('1.1') - 1 - """ - pos = value.find('.') - if pos < 0: - pos = len(value) - return pos - -def align_decimals(data, headers, **_): - """Align decimals to decimal point - >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) - 1 - 11.1 - 1.1 - """ - pointpos = len(data[0]) * [0] - for row in data: - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - pointpos[i] = max(intlen(v), pointpos[i]) - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - result.append((pointpos[i] - intlen(v)) * " " + v) - else: - result.append(v) - results.append(result) - return results, headers - - -def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote whitespace - >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) - ' before' - 'after ' - ' both ' - 'none' - >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) - abc - def - ghi - jkl - """ - quote = len(data[0]) * [False] - for row in data: - for i, v in enumerate(row): - v = encodingutils.text_type(v) - if v.startswith(' ') or v.endswith(' '): - quote[i] = True - - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - quotation = quotestyle if quote[i] else '' - result.append('{quotestyle}{value}{quotestyle}'.format( - quotestyle=quotation, value=v)) - results.append(result) - return results, headers - - -def tabulate_wrapper(data, headers, table_format=None, missing_value='', **_): - """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value, disable_numparse=True) - - -def csv_wrapper(data, headers, delimiter=',', **_): - """Wrap CSV formatting inside a standard function for OutputFormatter.""" - with contextlib.closing(StringIO()) as content: - writer = csv.writer(content, delimiter=str(delimiter)) - - writer.writerow(headers) - for row in data: - writer.writerow(row) - - return content.getvalue() - - -def terminal_tables_wrapper(data, headers, table_format=None, **_): - """Wrap terminaltables inside a standard function for OutputFormatter.""" - if table_format == 'ascii': - table = terminaltables.AsciiTable - elif table_format == 'single': - table = terminaltables.SingleTable - elif table_format == 'double': - table = terminaltables.DoubleTable - elif table_format == 'github': - table = terminaltables.GithubFlavoredMarkdownTable - else: - raise ValueError('unrecognized table format: {}'.format(table_format)) - - t = table([headers] + data) - return t.table - - -class OutputFormatter(object): - """A class with a standard interface for various formatting libraries.""" - - def __init__(self, format_name=None): - """Register the supported output formats.""" - self._output_formats = { - 'csv': (csv_wrapper, { - 'preprocessor': (override_missing_value, bytes_to_string), - 'missing_value': '' - }), - 'tsv': (csv_wrapper, { - 'preprocessor': (override_missing_value, bytes_to_string), - 'missing_value': '', - 'delimiter': '\t' - }), - 'expanded': (expanded_table, { - 'preprocessor': (override_missing_value, convert_to_string), - 'missing_value': '' - }) - } - self._format_name = None - - tabulate_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', - 'moinmoin', 'html', 'latex', 'latex_booktabs', - 'textile') - for tabulate_format in tabulate_formats: - self._output_formats[tabulate_format] = (tabulate_wrapper, { - 'preprocessor': (bytes_to_string, align_decimals), - 'table_format': tabulate_format, - 'missing_value': '' - }) - - terminal_tables_formats = ('ascii', 'single', 'double', 'github') - for terminal_tables_format in terminal_tables_formats: - self._output_formats[terminal_tables_format] = ( - terminal_tables_wrapper, { - 'preprocessor': (bytes_to_string, override_missing_value, - align_decimals), - 'table_format': terminal_tables_format, - 'missing_value': ''} - ) - - if format_name: - self.set_format_name(format_name) - - def set_format_name(self, format_name): - """Set the OutputFormatter's default format.""" - if format_name in self.supported_formats(): - self._format_name = format_name - else: - raise ValueError('unrecognized format_name: {}'.format( - format_name)) - - def get_format_name(self): - """Get the OutputFormatter's default format.""" - return self._format_name - - def supported_formats(self): - """Return the supported output format names.""" - return tuple(self._output_formats.keys()) - - def format_output(self, data, headers, format_name=None, **kwargs): - """Format the headers and data using a specific formatter. - - *format_name* must be a formatter available in `supported_formats()`. - - All keyword arguments are passed to the specified formatter. - """ - format_name = format_name or self._format_name - if format_name not in self.supported_formats(): - raise ValueError('unrecognized format: {}'.format(format_name)) - - function, fkwargs = self._output_formats[format_name] - fkwargs.update(kwargs) - preprocessor = fkwargs.get('preprocessor', None) - if preprocessor: - for f in preprocessor: - data, headers = f(data, headers, **fkwargs) - return function(data, headers, **fkwargs) diff --git a/mycli/output_formatter/__init__.py b/mycli/output_formatter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py new file mode 100644 index 000000000..04571e38f --- /dev/null +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -0,0 +1,22 @@ +import contextlib +import csv +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +from .preprocessors import (override_missing_value, bytes_to_string) + +supported_formats = ('csv',) +delimiter_preprocessors = (override_missing_value, bytes_to_string) + +def delimiter_adapter(data, headers, delimiter=',', **_): + """Wrap CSV formatting inside a standard function for OutputFormatter.""" + with contextlib.closing(StringIO()) as content: + writer = csv.writer(content, delimiter=str(delimiter)) + + writer.writerow(headers) + for row in data: + writer.writerow(row) + + return content.getvalue() diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py new file mode 100644 index 000000000..e4b280230 --- /dev/null +++ b/mycli/output_formatter/output_formatter.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from ..packages.expanded import expanded_table +from .preprocessors import (override_missing_value, convert_to_string) +from .delimited_output_adapter import delimiter_adapter, delimiter_preprocessors +from .tabulate_adapter import (tabulate_adapter, + supported_formats as tabulate_formats, + preprocessors as tabulate_preprocessors) +from .terminaltables_adapter import (terminaltables_adapter, + preprocessors as terminaltables_preprocessors, + supported_formats as terminaltables_formats) +from collections import namedtuple + +OutputFormatHandler = namedtuple('OutputFormatHandler', + 'format_name preprocessors formatter formatter_args') + +"""A generic output formatter interface.""" +class OutputFormatter(object): + """A class with a standard interface for various formatting libraries.""" + + _output_formats = {} + + def __init__(self, format_name=None): + """Register the supported output formats.""" + self._format_name = format_name + + def set_format_name(self, format_name): + """Set the OutputFormatter's default format.""" + if format_name in self.supported_formats(): + self._format_name = format_name + else: + raise ValueError('unrecognized format_name: {}'.format( + format_name)) + + def get_format_name(self): + """Get the OutputFormatter's default format.""" + return self._format_name + + def supported_formats(self): + """Return the supported output format names.""" + return tuple(self._output_formats.keys()) + + @classmethod + def register_new_formatter(cls, format_name, handler, preprocessors=None, + kwargs=None): + """Register a new fomatter to format the output""" + cls._output_formats[format_name] = OutputFormatHandler(format_name, + preprocessors, handler, kwargs) + + def format_output(self, data, headers, format_name=None, **kwargs): + """Format the headers and data using a specific formatter. + + *format_name* must be a formatter available in `supported_formats()`. + + All keyword arguments are passed to the specified formatter. + """ + format_name = format_name or self._format_name + if format_name not in self.supported_formats(): + raise ValueError('unrecognized format: {}'.format(format_name)) + + (_, preprocessors, formatter, fkwargs) = self._output_formats[format_name] + fkwargs.update(kwargs) + if preprocessors: + for f in preprocessors: + data, headers = f(data, headers, **fkwargs) + return formatter(data, headers, **fkwargs) + +OutputFormatter.register_new_formatter('csv', + delimiter_adapter, + delimiter_preprocessors, {'missing_value': ''}) +OutputFormatter.register_new_formatter('tsv', + delimiter_adapter, + delimiter_preprocessors, + {'missing_value': '', 'delimiter': '\t'}) +OutputFormatter.register_new_formatter('expanded', + expanded_table, + (override_missing_value, convert_to_string), + {'missing_value': '', 'delimiter': '\t'}) + +for tabulate_format in tabulate_formats: + OutputFormatter.register_new_formatter(tabulate_format, + tabulate_adapter, + tabulate_preprocessors, + {'table_format': tabulate_format, 'missing_value': ''}) + +for terminaltables_format in terminaltables_formats: + OutputFormatter.register_new_formatter(terminaltables_format, + terminaltables_adapter, + terminaltables_preprocessors, + {'table_format': terminaltables_format, 'missing_value': ''}) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py new file mode 100644 index 000000000..b3a9373af --- /dev/null +++ b/mycli/output_formatter/preprocessors.py @@ -0,0 +1,98 @@ +from decimal import Decimal +from .. import encodingutils + +def to_string(value): + """Convert *value* to a string.""" + if isinstance(value, encodingutils.binary_type): + return encodingutils.bytes_to_string(value) + else: + return encodingutils.text_type(value) + + +def convert_to_string(data, headers, **_): + """Convert all *data* and *headers* to strings.""" + return ([[to_string(v) for v in row] for row in data], + [to_string(h) for h in headers]) + + +def override_missing_value(data, headers, missing_value='', **_): + """Override missing values in the data with *missing_value*.""" + return ([[missing_value if v is None else v for v in row] for row in data], + headers) + + +def bytes_to_string(data, headers, **_): + """Convert all *data* and *headers* bytes to strings.""" + return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], + [encodingutils.bytes_to_string(h) for h in headers]) + + +def intlen(value): + """Find (character) length + >>> intlen('11.1') + 2 + >>> intlen('11') + 2 + >>> intlen('1.1') + 1 + """ + pos = value.find('.') + if pos < 0: + pos = len(value) + return pos + +def align_decimals(data, headers, **_): + """Align decimals to decimal point + >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) + 1 + 11.1 + 1.1 + """ + pointpos = len(data[0]) * [0] + for row in data: + for i, v in enumerate(row): + if isinstance(v, Decimal): + v = encodingutils.text_type(v) + pointpos[i] = max(intlen(v), pointpos[i]) + results = [] + for row in data: + result = [] + for i, v in enumerate(row): + if isinstance(v, Decimal): + v = encodingutils.text_type(v) + result.append((pointpos[i] - intlen(v)) * " " + v) + else: + result.append(v) + results.append(result) + return results, headers + + +def quote_whitespaces(data, headers, quotestyle="'", **_): + """Quote whitespace + >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) + ' before' + 'after ' + ' both ' + 'none' + >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) + abc + def + ghi + jkl + """ + quote = len(data[0]) * [False] + for row in data: + for i, v in enumerate(row): + v = encodingutils.text_type(v) + if v.startswith(' ') or v.endswith(' '): + quote[i] = True + + results = [] + for row in data: + result = [] + for i, v in enumerate(row): + quotation = quotestyle if quote[i] else '' + result.append('{quotestyle}{value}{quotestyle}'.format( + quotestyle=quotation, value=v)) + results.append(result) + return results, headers diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py new file mode 100644 index 000000000..8c4599c12 --- /dev/null +++ b/mycli/output_formatter/tabulate_adapter.py @@ -0,0 +1,14 @@ +from tabulate import tabulate +from .preprocessors import (bytes_to_string, align_decimals) + +supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', + 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', + 'moinmoin', 'html', 'latex', 'latex_booktabs', + 'textile') + +preprocessors = (bytes_to_string, align_decimals) + +def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): + """Wrap tabulate inside a standard function for OutputFormatter.""" + return tabulate(data, headers, tablefmt=table_format, + missingval=missing_value, disable_numparse=True) diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py new file mode 100644 index 000000000..159810783 --- /dev/null +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -0,0 +1,24 @@ +import terminaltables +from .preprocessors import (bytes_to_string, align_decimals, + override_missing_value) + +supported_formats = ('ascii', 'single', 'double', 'github') +preprocessors = (bytes_to_string, override_missing_value, align_decimals) + +def terminaltables_adapter(data, headers, table_format=None, **_): + """Wrap terminaltables inside a standard function for OutputFormatter.""" + + table_format_handler = { + 'ascii': terminaltables.AsciiTable, + 'single': terminaltables.SingleTable, + 'double': terminaltables.DoubleTable, + 'github': terminaltables.GithubFlavoredMarkdownTable, + } + + try: + table = table_format_handler[table_format] + except KeyError: + raise ValueError('unrecognized table format: {}'.format(table_format)) + + t = table([headers] + data) + return t.table diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 656e5d531..f01afb1da 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -6,7 +6,6 @@ from prompt_toolkit.completion import Completer, Completion -from .output_formatter import OutputFormatter from .packages.completion_engine import suggest_type from .packages.parseutils import last_word from .packages.special.favoritequeries import favoritequeries @@ -50,7 +49,7 @@ class SQLCompleter(Completer): users = [] - def __init__(self, smart_completion=True): + def __init__(self, smart_completion=True, supported_formats=()): super(self.__class__, self).__init__() self.smart_completion = smart_completion self.reserved_words = set() @@ -59,8 +58,7 @@ def __init__(self, smart_completion=True): self.name_pattern = compile("^[_a-z][_a-z0-9\$]*$") self.special_commands = [] - formatter = OutputFormatter() - self.table_formats = formatter.supported_formats() + self.table_formats = supported_formats self.reset_completions() def escape_name(self, name): From 7a81b9e6bc173fed75fd2653a67f606942bd04d6 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 19:52:35 -0700 Subject: [PATCH 0349/1025] Update the tests. --- tests/test_output_formatter.py | 18 ++++++++++++------ tests/utils.py | 1 - 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 8ddeb888c..484f52d81 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -6,12 +6,18 @@ from decimal import Decimal from textwrap import dedent -from mycli.output_formatter import (align_decimals, bytes_to_string, - convert_to_string, csv_wrapper, - OutputFormatter, override_missing_value, - tabulate_wrapper, terminal_tables_wrapper, - to_string) - +from mycli.output_formatter.preprocessors import (align_decimals, + bytes_to_string, + convert_to_string, + override_missing_value, + to_string) +from mycli.output_formatter.output_formatter import OutputFormatter +from mycli.output_formatter.delimited_output_adapter import (delimiter_adapter + as csv_wrapper) +from mycli.output_formatter.tabulate_adapter import (tabulate_adapter as + tabulate_wrapper) +from mycli.output_formatter.terminaltables_adapter import \ + (terminaltables_adapter as terminal_tables_wrapper) def test_to_string(): """Test the *output_formatter.to_string()* function.""" diff --git a/tests/utils.py b/tests/utils.py index 64006324f..bff31045d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,7 +4,6 @@ import pytest from mycli.main import MyCli, special -from mycli.output_formatter import OutputFormatter PASSWORD = getenv('PYTEST_PASSWORD') USER = getenv('PYTEST_USER', 'root') From a96867c6fa06049d34d309927535de2851ea18da Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 3 Apr 2017 23:01:12 -0500 Subject: [PATCH 0350/1025] PEP8 edits. --- .../delimited_output_adapter.py | 3 +- mycli/output_formatter/output_formatter.py | 74 ++++++++++--------- mycli/output_formatter/preprocessors.py | 5 +- mycli/output_formatter/tabulate_adapter.py | 11 +-- .../terminaltables_adapter.py | 14 ++-- tests/test_output_formatter.py | 22 +++--- 6 files changed, 71 insertions(+), 58 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 04571e38f..04e9f252a 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -5,11 +5,12 @@ except ImportError: from io import StringIO -from .preprocessors import (override_missing_value, bytes_to_string) +from .preprocessors import override_missing_value, bytes_to_string supported_formats = ('csv',) delimiter_preprocessors = (override_missing_value, bytes_to_string) + def delimiter_adapter(data, headers, delimiter=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index e4b280230..8f698fc0b 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -1,21 +1,25 @@ # -*- coding: utf-8 -*- +"""A generic output formatter interface.""" + from __future__ import unicode_literals +from collections import namedtuple -from ..packages.expanded import expanded_table +from mycli.packages.expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) -from .delimited_output_adapter import delimiter_adapter, delimiter_preprocessors +from .delimited_output_adapter import (delimiter_adapter, + delimiter_preprocessors) from .tabulate_adapter import (tabulate_adapter, - supported_formats as tabulate_formats, - preprocessors as tabulate_preprocessors) -from .terminaltables_adapter import (terminaltables_adapter, - preprocessors as terminaltables_preprocessors, - supported_formats as terminaltables_formats) -from collections import namedtuple + supported_formats as tabulate_formats, + preprocessors as tabulate_preprocessors) +from .terminaltables_adapter import ( + terminaltables_adapter, preprocessors as terminaltables_preprocessors, + supported_formats as terminaltables_formats) + +OutputFormatHandler = namedtuple( + 'OutputFormatHandler', + 'format_name preprocessors formatter formatter_args') -OutputFormatHandler = namedtuple('OutputFormatHandler', - 'format_name preprocessors formatter formatter_args') -"""A generic output formatter interface.""" class OutputFormatter(object): """A class with a standard interface for various formatting libraries.""" @@ -43,10 +47,10 @@ def supported_formats(self): @classmethod def register_new_formatter(cls, format_name, handler, preprocessors=None, - kwargs=None): - """Register a new fomatter to format the output""" - cls._output_formats[format_name] = OutputFormatHandler(format_name, - preprocessors, handler, kwargs) + kwargs=None): + """Register a new formatter to format the output.""" + cls._output_formats[format_name] = OutputFormatHandler( + format_name, preprocessors, handler, kwargs) def format_output(self, data, headers, format_name=None, **kwargs): """Format the headers and data using a specific formatter. @@ -66,26 +70,28 @@ def format_output(self, data, headers, format_name=None, **kwargs): data, headers = f(data, headers, **fkwargs) return formatter(data, headers, **fkwargs) -OutputFormatter.register_new_formatter('csv', - delimiter_adapter, - delimiter_preprocessors, {'missing_value': ''}) -OutputFormatter.register_new_formatter('tsv', - delimiter_adapter, - delimiter_preprocessors, - {'missing_value': '', 'delimiter': '\t'}) -OutputFormatter.register_new_formatter('expanded', - expanded_table, - (override_missing_value, convert_to_string), - {'missing_value': '', 'delimiter': '\t'}) + +OutputFormatter.register_new_formatter('csv', delimiter_adapter, + delimiter_preprocessors, + {'missing_value': ''}) +OutputFormatter.register_new_formatter('tsv', delimiter_adapter, + delimiter_preprocessors, + {'missing_value': '', + 'delimiter': '\t'}) +OutputFormatter.register_new_formatter('expanded', expanded_table, + (override_missing_value, + convert_to_string), + {'missing_value': '', + 'delimiter': '\t'}) for tabulate_format in tabulate_formats: - OutputFormatter.register_new_formatter(tabulate_format, - tabulate_adapter, - tabulate_preprocessors, - {'table_format': tabulate_format, 'missing_value': ''}) + OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, + tabulate_preprocessors, + {'table_format': tabulate_format, + 'missing_value': ''}) for terminaltables_format in terminaltables_formats: - OutputFormatter.register_new_formatter(terminaltables_format, - terminaltables_adapter, - terminaltables_preprocessors, - {'table_format': terminaltables_format, 'missing_value': ''}) + OutputFormatter.register_new_formatter( + terminaltables_format, terminaltables_adapter, + terminaltables_preprocessors, + {'table_format': terminaltables_format, 'missing_value': ''}) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index b3a9373af..d6db3fd44 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -1,5 +1,7 @@ from decimal import Decimal -from .. import encodingutils + +from mycli import encodingutils + def to_string(value): """Convert *value* to a string.""" @@ -41,6 +43,7 @@ def intlen(value): pos = len(value) return pos + def align_decimals(data, headers, **_): """Align decimals to decimal point >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 8c4599c12..0db31ff9f 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,13 +1,14 @@ from tabulate import tabulate -from .preprocessors import (bytes_to_string, align_decimals) -supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', - 'moinmoin', 'html', 'latex', 'latex_booktabs', - 'textile') +from .preprocessors import bytes_to_string, align_decimals + +supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', + 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', + 'html', 'latex', 'latex_booktabs', 'textile') preprocessors = (bytes_to_string, align_decimals) + def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" return tabulate(data, headers, tablefmt=table_format, diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py index 159810783..ac580517f 100644 --- a/mycli/output_formatter/terminaltables_adapter.py +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -1,19 +1,21 @@ import terminaltables + from .preprocessors import (bytes_to_string, align_decimals, - override_missing_value) + override_missing_value) supported_formats = ('ascii', 'single', 'double', 'github') preprocessors = (bytes_to_string, override_missing_value, align_decimals) + def terminaltables_adapter(data, headers, table_format=None, **_): """Wrap terminaltables inside a standard function for OutputFormatter.""" table_format_handler = { - 'ascii': terminaltables.AsciiTable, - 'single': terminaltables.SingleTable, - 'double': terminaltables.DoubleTable, - 'github': terminaltables.GithubFlavoredMarkdownTable, - } + 'ascii': terminaltables.AsciiTable, + 'single': terminaltables.SingleTable, + 'double': terminaltables.DoubleTable, + 'github': terminaltables.GithubFlavoredMarkdownTable, + } try: table = table_format_handler[table_format] diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 484f52d81..7c0890fde 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -2,22 +2,22 @@ """Test the generic output formatter interface.""" from __future__ import unicode_literals - from decimal import Decimal from textwrap import dedent from mycli.output_formatter.preprocessors import (align_decimals, - bytes_to_string, - convert_to_string, - override_missing_value, - to_string) + bytes_to_string, + convert_to_string, + override_missing_value, + to_string) from mycli.output_formatter.output_formatter import OutputFormatter -from mycli.output_formatter.delimited_output_adapter import (delimiter_adapter - as csv_wrapper) -from mycli.output_formatter.tabulate_adapter import (tabulate_adapter as - tabulate_wrapper) -from mycli.output_formatter.terminaltables_adapter import \ - (terminaltables_adapter as terminal_tables_wrapper) +from mycli.output_formatter.delimited_output_adapter import ( + delimiter_adapter as csv_wrapper) +from mycli.output_formatter.tabulate_adapter import ( + tabulate_adapter as tabulate_wrapper) +from mycli.output_formatter.terminaltables_adapter import ( + terminaltables_adapter as terminal_tables_wrapper) + def test_to_string(): """Test the *output_formatter.to_string()* function.""" From 3d9feb9241787eeecad974f7a5c1e5f9429ef604 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 21:14:52 -0700 Subject: [PATCH 0351/1025] Move expanded.py into output_formatter package. --- mycli/output_formatter/delimited_output_adapter.py | 2 +- mycli/{packages => output_formatter}/expanded.py | 0 mycli/output_formatter/output_formatter.py | 5 ++--- tests/test_expanded.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) rename mycli/{packages => output_formatter}/expanded.py (100%) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 04e9f252a..7111af6d3 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -7,7 +7,7 @@ from .preprocessors import override_missing_value, bytes_to_string -supported_formats = ('csv',) +supported_formats = ('csv', 'tsv') delimiter_preprocessors = (override_missing_value, bytes_to_string) diff --git a/mycli/packages/expanded.py b/mycli/output_formatter/expanded.py similarity index 100% rename from mycli/packages/expanded.py rename to mycli/output_formatter/expanded.py diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 8f698fc0b..1564026b3 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals from collections import namedtuple -from mycli.packages.expanded import expanded_table +from .expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) from .delimited_output_adapter import (delimiter_adapter, delimiter_preprocessors) @@ -81,8 +81,7 @@ def format_output(self, data, headers, format_name=None, **kwargs): OutputFormatter.register_new_formatter('expanded', expanded_table, (override_missing_value, convert_to_string), - {'missing_value': '', - 'delimiter': '\t'}) + {'missing_value': ''}) for tabulate_format in tabulate_formats: OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, diff --git a/tests/test_expanded.py b/tests/test_expanded.py index db503cba8..7233e91ce 100644 --- a/tests/test_expanded.py +++ b/tests/test_expanded.py @@ -1,7 +1,7 @@ """Test the vertical, expanded table formatter.""" from textwrap import dedent -from mycli.packages.expanded import expanded_table +from mycli.output_formatter.expanded import expanded_table from mycli.encodingutils import text_type From b5bda3f20f2cea84bc72aa6a28cbf1f31cfa67f7 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 21:26:33 -0700 Subject: [PATCH 0352/1025] Refactor the delimiter adapter to include tsv. --- mycli/output_formatter/delimited_output_adapter.py | 9 +++++++-- mycli/output_formatter/output_formatter.py | 14 +++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 7111af6d3..d3b5120b9 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -11,10 +11,15 @@ delimiter_preprocessors = (override_missing_value, bytes_to_string) -def delimiter_adapter(data, headers, delimiter=',', **_): +def delimiter_adapter(data, headers, table_format=',', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: - writer = csv.writer(content, delimiter=str(delimiter)) + if table_format == 'csv': + writer = csv.writer(content, delimiter=',') + elif table_format == 'tsv': + writer = csv.writer(content, delimiter='\t') + else: + raise ValueError('Invalid table_format specified.') writer.writerow(headers) for row in data: diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 1564026b3..a47029cca 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -7,6 +7,7 @@ from .expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) from .delimited_output_adapter import (delimiter_adapter, + supported_formats as delimiter_formats, delimiter_preprocessors) from .tabulate_adapter import (tabulate_adapter, supported_formats as tabulate_formats, @@ -71,18 +72,17 @@ def format_output(self, data, headers, format_name=None, **kwargs): return formatter(data, headers, **fkwargs) -OutputFormatter.register_new_formatter('csv', delimiter_adapter, - delimiter_preprocessors, - {'missing_value': ''}) -OutputFormatter.register_new_formatter('tsv', delimiter_adapter, - delimiter_preprocessors, - {'missing_value': '', - 'delimiter': '\t'}) OutputFormatter.register_new_formatter('expanded', expanded_table, (override_missing_value, convert_to_string), {'missing_value': ''}) +for delimiter_format in delimiter_formats: + OutputFormatter.register_new_formatter(delimiter_format, delimiter_adapter, + delimiter_preprocessors, + {'table_format': delimiter_format, + 'missing_value': ''}) + for tabulate_format in tabulate_formats: OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, tabulate_preprocessors, From 2120b3653e3f8027ebfa16703737b65aa7d31f81 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 3 Apr 2017 21:33:41 -0700 Subject: [PATCH 0353/1025] Fix failing tests. --- mycli/output_formatter/delimited_output_adapter.py | 2 +- tests/test_output_formatter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index d3b5120b9..8036a5ff7 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -11,7 +11,7 @@ delimiter_preprocessors = (override_missing_value, bytes_to_string) -def delimiter_adapter(data, headers, table_format=',', **_): +def delimiter_adapter(data, headers, table_format='csv', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: if table_format == 'csv': diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 7c0890fde..7949032fd 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -92,7 +92,7 @@ def test_csv_wrapper(): # Test tab-delimited output. data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = csv_wrapper(data, headers, delimiter='\t') + output = csv_wrapper(data, headers, table_format='tsv') assert output == dedent('''\ letters\tnumber\r\n\ abc\t1\r\n\ From 98f827a7bba38d7e31692d0f435e158668f7c7b2 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 4 Apr 2017 20:22:21 +0200 Subject: [PATCH 0354/1025] --user-config-file parameter to only read user config file from the given file --- mycli/main.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index ba3443955..ca3cffc6f 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -92,12 +92,12 @@ class MyCli(object): ] default_config_file = os.path.join(PACKAGE_ROOT, 'myclirc') - user_config_file = '~/.myclirc' def __init__(self, sqlexecute=None, prompt=None, logfile=None, defaults_suffix=None, defaults_file=None, - login_path=None, auto_vertical_output=False, warn=None): + login_path=None, auto_vertical_output=False, warn=None, + myclirc="~/.myclirc"): self.sqlexecute = sqlexecute self.logfile = logfile self.defaults_suffix = defaults_suffix @@ -112,7 +112,7 @@ def __init__(self, sqlexecute=None, prompt=None, # Load config. config_files = ([self.default_config_file] + self.system_config_files + - [self.user_config_file]) + [myclirc]) c = self.config = read_config_files(config_files) self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] @@ -132,7 +132,7 @@ def __init__(self, sqlexecute=None, prompt=None, # Write user config if system config wasn't the last config loaded. if c.filename not in self.system_config_files: - write_default_config(self.default_config_file, self.user_config_file) + write_default_config(self.default_config_file, myclirc) # audit log if self.logfile is None and 'audit_log' in c['main']: @@ -759,6 +759,8 @@ def run_query(self, query, table_format=None, new_line=True): help='Read config group with the specified suffix.') @click.option('--defaults-file', type=click.Path(), help='Only read default options from the given file') +@click.option('--myclirc', type=click.Path(), default="~/.myclirc", + help='Location of myclirc file.') @click.option('--auto-vertical-output', is_flag=True, help='Automatically switch to vertical output mode if the result is wider than the terminal width.') @click.option('-t', '--table', is_flag=True, @@ -778,7 +780,7 @@ def cli(database, user, host, port, socket, password, dbname, version, prompt, logfile, defaults_group_suffix, defaults_file, login_path, auto_vertical_output, local_infile, ssl_ca, ssl_capath, ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, csv, - warn, execute): + warn, execute, myclirc): if version: print('Version:', __version__) @@ -787,7 +789,8 @@ def cli(database, user, host, port, socket, password, dbname, mycli = MyCli(prompt=prompt, logfile=logfile, defaults_suffix=defaults_group_suffix, defaults_file=defaults_file, login_path=login_path, - auto_vertical_output=auto_vertical_output, warn=warn) + auto_vertical_output=auto_vertical_output, warn=warn, + myclirc=myclirc) # Choose which ever one has a valid value. database = database or dbname From 252a257ace53f12e7631794e959f3ccac66524c1 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 20 Mar 2017 16:33:59 +0100 Subject: [PATCH 0355/1025] copy from pgcli 56af64585f4f033c25cd574072185c85c26e52a2 --- tests/behave.ini | 5 + tests/features/basic_commands.feature | 19 ++++ tests/features/crud_database.feature | 20 ++++ tests/features/crud_table.feature | 22 ++++ tests/features/db_utils.py | 82 ++++++++++++++ tests/features/environment.py | 96 ++++++++++++++++ tests/features/fixture_data/help.txt | 24 ++++ tests/features/fixture_data/help_commands.txt | 21 ++++ tests/features/fixture_utils.py | 34 ++++++ tests/features/iocommands.feature | 10 ++ tests/features/named_queries.feature | 12 ++ tests/features/specials.feature | 9 ++ tests/features/steps/basic_commands.py | 47 ++++++++ tests/features/steps/crud_database.py | 97 ++++++++++++++++ tests/features/steps/crud_table.py | 107 ++++++++++++++++++ tests/features/steps/iocommands.py | 44 +++++++ tests/features/steps/named_queries.py | 59 ++++++++++ tests/features/steps/specials.py | 26 +++++ tests/features/steps/wrappers.py | 15 +++ 19 files changed, 749 insertions(+) create mode 100644 tests/behave.ini create mode 100644 tests/features/basic_commands.feature create mode 100644 tests/features/crud_database.feature create mode 100644 tests/features/crud_table.feature create mode 100644 tests/features/db_utils.py create mode 100644 tests/features/environment.py create mode 100644 tests/features/fixture_data/help.txt create mode 100644 tests/features/fixture_data/help_commands.txt create mode 100644 tests/features/fixture_utils.py create mode 100644 tests/features/iocommands.feature create mode 100644 tests/features/named_queries.feature create mode 100644 tests/features/specials.feature create mode 100644 tests/features/steps/basic_commands.py create mode 100644 tests/features/steps/crud_database.py create mode 100644 tests/features/steps/crud_table.py create mode 100644 tests/features/steps/iocommands.py create mode 100644 tests/features/steps/named_queries.py create mode 100644 tests/features/steps/specials.py create mode 100644 tests/features/steps/wrappers.py diff --git a/tests/behave.ini b/tests/behave.ini new file mode 100644 index 000000000..c555d3c07 --- /dev/null +++ b/tests/behave.ini @@ -0,0 +1,5 @@ +[behave.userdata] +pg_test_user = postgres +pg_test_pass = +pg_test_host = localhost +pg_test_db = pgcli_behave_tests diff --git a/tests/features/basic_commands.feature b/tests/features/basic_commands.feature new file mode 100644 index 000000000..227fe769b --- /dev/null +++ b/tests/features/basic_commands.feature @@ -0,0 +1,19 @@ +Feature: run the cli, + call the help command, + exit the cli + + Scenario: run the cli + When we run dbcli + then we see dbcli prompt + + Scenario: run "\?" command + When we run dbcli + and we wait for prompt + and we send "\?" command + then we see help output + + Scenario: run the cli and exit + When we run dbcli + and we wait for prompt + and we send "ctrl + d" + then dbcli exits diff --git a/tests/features/crud_database.feature b/tests/features/crud_database.feature new file mode 100644 index 000000000..c72468c30 --- /dev/null +++ b/tests/features/crud_database.feature @@ -0,0 +1,20 @@ +Feature: manipulate databases: + create, drop, connect, disconnect + + Scenario: create and drop temporary database + When we run dbcli + and we wait for prompt + and we create database + then we see database created + when we drop database + then we see database dropped + when we connect to dbserver + then we see database connected + + Scenario: connect and disconnect from test database + When we run dbcli + and we wait for prompt + and we connect to test database + then we see database connected + when we connect to dbserver + then we see database connected diff --git a/tests/features/crud_table.feature b/tests/features/crud_table.feature new file mode 100644 index 000000000..d2209fd0e --- /dev/null +++ b/tests/features/crud_table.feature @@ -0,0 +1,22 @@ +Feature: manipulate tables: + create, insert, update, select, delete from, drop + + Scenario: create, insert, select from, update, drop table + When we run dbcli + and we wait for prompt + and we connect to test database + then we see database connected + when we create table + then we see table created + when we insert into table + then we see record inserted + when we update table + then we see record updated + when we select from table + then we see data selected + when we delete from table + then we see record deleted + when we drop table + then we see table dropped + when we connect to dbserver + then we see database connected diff --git a/tests/features/db_utils.py b/tests/features/db_utils.py new file mode 100644 index 000000000..63c6586b7 --- /dev/null +++ b/tests/features/db_utils.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from __future__ import print_function + +from psycopg2 import connect +from psycopg2.extensions import AsIs + + +def create_db(hostname='localhost', username=None, password=None, + dbname=None): + """ + Create test database. + :param hostname: string + :param username: string + :param password: string + :param dbname: string + :return: + """ + cn = create_cn(hostname, password, username, 'postgres') + + # ISOLATION_LEVEL_AUTOCOMMIT = 0 + # Needed for DB creation. + cn.set_isolation_level(0) + + with cn.cursor() as cr: + cr.execute('drop database if exists %s', (AsIs(dbname),)) + cr.execute('create database %s', (AsIs(dbname),)) + + cn.close() + + cn = create_cn(hostname, password, username, dbname) + return cn + + +def create_cn(hostname, password, username, dbname): + """ + Open connection to database. + :param hostname: + :param password: + :param username: + :param dbname: string + :return: psycopg2.connection + """ + if password: + cn = connect(host=hostname, user=username, database=dbname, + password=password) + else: + cn = connect(user=username, database=dbname) + + print('Created connection: {0}.'.format(cn.dsn)) + return cn + + +def drop_db(hostname='localhost', username=None, password=None, + dbname=None): + """ + Drop database. + :param hostname: string + :param username: string + :param password: string + :param dbname: string + """ + cn = create_cn(hostname, password, username, 'postgres') + + # ISOLATION_LEVEL_AUTOCOMMIT = 0 + # Needed for DB drop. + cn.set_isolation_level(0) + + with cn.cursor() as cr: + cr.execute('drop database if exists %s', (AsIs(dbname),)) + + close_cn(cn) + + +def close_cn(cn=None): + """ + Close connection. + :param connection: psycopg2.connection + """ + if cn: + cn.close() + print('Closed connection: {0}.'.format(cn.dsn)) diff --git a/tests/features/environment.py b/tests/features/environment.py new file mode 100644 index 000000000..51ae86b5a --- /dev/null +++ b/tests/features/environment.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from __future__ import print_function + +import os +import sys +import db_utils as dbutils +import fixture_utils as fixutils + + +def before_all(context): + """ + Set env parameters. + """ + os.environ['LINES'] = "100" + os.environ['COLUMNS'] = "100" + os.environ['PAGER'] = 'cat' + os.environ['EDITOR'] = 'ex' + os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../.coveragerc" + + context.exit_sent = False + + vi = '_'.join([str(x) for x in sys.version_info[:3]]) + db_name = context.config.userdata.get('pg_test_db', None) + db_name_full = '{0}_{1}'.format(db_name, vi) + + # Store get params from config. + context.conf = { + 'host': context.config.userdata.get('pg_test_host', 'localhost'), + 'user': context.config.userdata.get('pg_test_user', 'postgres'), + 'pass': context.config.userdata.get('pg_test_pass', None), + 'dbname': db_name_full, + 'dbname_tmp': db_name_full + '_tmp', + 'vi': vi, + 'cli_command': context.config.userdata.get('pg_cli_command', None) or + sys.executable + + ' -c "import coverage; coverage.process_startup(); import pgcli.main; pgcli.main.cli()"' + } + + # Store old env vars. + context.pgenv = { + 'PGDATABASE': os.environ.get('PGDATABASE', None), + 'PGUSER': os.environ.get('PGUSER', None), + 'PGHOST': os.environ.get('PGHOST', None), + 'PGPASSWORD': os.environ.get('PGPASSWORD', None), + } + + # Set new env vars. + os.environ['PGDATABASE'] = context.conf['dbname'] + os.environ['PGUSER'] = context.conf['user'] + os.environ['PGHOST'] = context.conf['host'] + + if context.conf['pass']: + os.environ['PGPASSWORD'] = context.conf['pass'] + else: + if 'PGPASSWORD' in os.environ: + del os.environ['PGPASSWORD'] + if 'PGHOST' in os.environ: + del os.environ['PGHOST'] + + context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], + context.conf['pass'], + context.conf['dbname']) + + context.fixture_data = fixutils.read_fixture_files() + + +def after_all(context): + """ + Unset env parameters. + """ + dbutils.close_cn(context.cn) + dbutils.drop_db(context.conf['host'], context.conf['user'], + context.conf['pass'], context.conf['dbname']) + + # Restore env vars. + for k, v in context.pgenv.items(): + if k in os.environ and v is None: + del os.environ[k] + elif v: + os.environ[k] = v + + +def after_scenario(context, _): + """ + Cleans up after each test complete. + """ + + if hasattr(context, 'cli') and not context.exit_sent: + # Terminate nicely. + context.cli.terminate() + +# TODO: uncomment to debug a failure +# def after_step(context, step): +# if step.status == "failed": +# import ipdb; ipdb.set_trace() diff --git a/tests/features/fixture_data/help.txt b/tests/features/fixture_data/help.txt new file mode 100644 index 000000000..deb499a4e --- /dev/null +++ b/tests/features/fixture_data/help.txt @@ -0,0 +1,24 @@ ++--------------------------+-----------------------------------------------+ +| Command | Description | +|--------------------------+-----------------------------------------------| +| \# | Refresh auto-completions. | +| \? | Show Help. | +| \c[onnect] database_name | Change to a new database. | +| \d [pattern] | List or describe tables, views and sequences. | +| \dT[S+] [pattern] | List data types | +| \df[+] [pattern] | List functions. | +| \di[+] [pattern] | List indexes. | +| \dn[+] [pattern] | List schemas. | +| \ds[+] [pattern] | List sequences. | +| \dt[+] [pattern] | List tables. | +| \du[+] [pattern] | List roles. | +| \dv[+] [pattern] | List views. | +| \e [file] | Edit the query with external editor. | +| \l | List databases. | +| \n[+] [name] | List or execute named queries. | +| \nd [name [query]] | Delete a named query. | +| \ns name query | Save a named query. | +| \refresh | Refresh auto-completions. | +| \timing | Toggle timing of commands. | +| \x | Toggle expanded output. | ++--------------------------+-----------------------------------------------+ diff --git a/tests/features/fixture_data/help_commands.txt b/tests/features/fixture_data/help_commands.txt new file mode 100644 index 000000000..3f3dd0b95 --- /dev/null +++ b/tests/features/fixture_data/help_commands.txt @@ -0,0 +1,21 @@ +Command +\# +\? +\c[onnect] database_name +\d [pattern] +\dT[S+] [pattern] +\df[+] [pattern] +\di[+] [pattern] +\dn[+] [pattern] +\ds[+] [pattern] +\dt[+] [pattern] +\du[+] [pattern] +\dv[+] [pattern] +\e [file] +\l +\n[+] [name] +\nd [name] +\ns name query +\refresh +\timing +\x \ No newline at end of file diff --git a/tests/features/fixture_utils.py b/tests/features/fixture_utils.py new file mode 100644 index 000000000..917baeb32 --- /dev/null +++ b/tests/features/fixture_utils.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from __future__ import print_function + +import os +import codecs + + +def read_fixture_lines(filename): + """ + Read lines of text from file. + :param filename: string name + :return: list of strings + """ + lines = [] + for line in codecs.open(filename, 'rb', encoding='utf-8'): + lines.append(line.strip()) + return lines + + +def read_fixture_files(): + """ + Read all files inside fixture_data directory. + """ + fixture_dict = {} + + current_dir = os.path.dirname(__file__) + fixture_dir = os.path.join(current_dir, 'fixture_data/') + for filename in os.listdir(fixture_dir): + if filename not in ['.', '..']: + fullname = os.path.join(fixture_dir, filename) + fixture_dict[filename] = read_fixture_lines(fullname) + + return fixture_dict diff --git a/tests/features/iocommands.feature b/tests/features/iocommands.feature new file mode 100644 index 000000000..d043dc2ea --- /dev/null +++ b/tests/features/iocommands.feature @@ -0,0 +1,10 @@ +Feature: I/O commands + + Scenario: edit sql in file with external editor + When we run dbcli + and we wait for prompt + and we start external editor providing a file name + and we type sql in the editor + and we exit the editor + then we see dbcli prompt + and we see the sql in prompt diff --git a/tests/features/named_queries.feature b/tests/features/named_queries.feature new file mode 100644 index 000000000..79f31ac3a --- /dev/null +++ b/tests/features/named_queries.feature @@ -0,0 +1,12 @@ +Feature: named queries: + save, use and delete named queries + + Scenario: save, use and delete named queries + When we run dbcli + and we wait for prompt + and we connect to test database + then we see database connected + when we save a named query + then we see the named query saved + when we delete a named query + then we see the named query deleted diff --git a/tests/features/specials.feature b/tests/features/specials.feature new file mode 100644 index 000000000..9bacec45a --- /dev/null +++ b/tests/features/specials.feature @@ -0,0 +1,9 @@ +Feature: Special commands + + @wip + Scenario: run refresh command + When we run dbcli + and we wait for prompt + and we refresh completions + and we wait for prompt + then we see completions refresh started diff --git a/tests/features/steps/basic_commands.py b/tests/features/steps/basic_commands.py new file mode 100644 index 000000000..568c793e4 --- /dev/null +++ b/tests/features/steps/basic_commands.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import pexpect + +from behave import when +import wrappers + + +@when('we run dbcli') +def step_run_cli(context): + """ + Run the process using pexpect. + """ + cli_cmd = context.conf.get('cli_command') + context.cli = pexpect.spawnu(cli_cmd, cwd='..') + context.exit_sent = False + + +@when('we wait for prompt') +def step_wait_prompt(context): + """ + Make sure prompt is displayed. + """ + wrappers.expect_exact(context, '{0}> '.format(context.conf['dbname']), timeout=5) + + +@when('we send "ctrl + d"') +def step_ctrl_d(context): + """ + Send Ctrl + D to hopefully exit. + """ + context.cli.sendcontrol('d') + context.exit_sent = True + + +@when('we send "\?" command') +def step_send_help(context): + """ + Send \? to see help. + """ + context.cli.sendline('\?') diff --git a/tests/features/steps/crud_database.py b/tests/features/steps/crud_database.py new file mode 100644 index 000000000..97bdfa7e2 --- /dev/null +++ b/tests/features/steps/crud_database.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import pexpect + +import wrappers +from behave import when, then + + +@when('we create database') +def step_db_create(context): + """ + Send create database. + """ + context.cli.sendline('create database {0};'.format( + context.conf['dbname_tmp'])) + + context.response = { + 'database_name': context.conf['dbname_tmp'] + } + + +@when('we drop database') +def step_db_drop(context): + """ + Send drop database. + """ + context.cli.sendline('drop database {0};'.format( + context.conf['dbname_tmp'])) + + +@when('we connect to test database') +def step_db_connect_test(context): + """ + Send connect to database. + """ + db_name = context.conf['dbname'] + context.cli.sendline('\\connect {0}'.format(db_name)) + + +@when('we connect to dbserver') +def step_db_connect_dbserver(context): + """ + Send connect to database. + """ + context.cli.sendline('\\connect postgres') + + +@then('dbcli exits') +def step_wait_exit(context): + """ + Make sure the cli exits. + """ + wrappers.expect_exact(context, pexpect.EOF, timeout=5) + + +@then('we see dbcli prompt') +def step_see_prompt(context): + """ + Wait to see the prompt. + """ + wrappers.expect_exact(context, '{0}> '.format(context.conf['dbname']), timeout=5) + + +@then('we see help output') +def step_see_help(context): + for expected_line in context.fixture_data['help_commands.txt']: + wrappers.expect_exact(context, expected_line, timeout=1) + + +@then('we see database created') +def step_see_db_created(context): + """ + Wait to see create database output. + """ + wrappers.expect_exact(context, 'CREATE DATABASE', timeout=2) + + +@then('we see database dropped') +def step_see_db_dropped(context): + """ + Wait to see drop database output. + """ + wrappers.expect_exact(context, 'DROP DATABASE', timeout=2) + + +@then('we see database connected') +def step_see_db_connected(context): + """ + Wait to see drop database output. + """ + wrappers.expect_exact(context, 'You are now connected to database', timeout=2) diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py new file mode 100644 index 000000000..0863a224f --- /dev/null +++ b/tests/features/steps/crud_table.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import wrappers +from behave import when, then + + +@when('we create table') +def step_create_table(context): + """ + Send create table. + """ + context.cli.sendline('create table a(x text);') + + +@when('we insert into table') +def step_insert_into_table(context): + """ + Send insert into table. + """ + context.cli.sendline('''insert into a(x) values('xxx');''') + + +@when('we update table') +def step_update_table(context): + """ + Send insert into table. + """ + context.cli.sendline('''update a set x = 'yyy' where x = 'xxx';''') + + +@when('we select from table') +def step_select_from_table(context): + """ + Send select from table. + """ + context.cli.sendline('select * from a;') + + +@when('we delete from table') +def step_delete_from_table(context): + """ + Send deete from table. + """ + context.cli.sendline('''delete from a where x = 'yyy';''') + + +@when('we drop table') +def step_drop_table(context): + """ + Send drop table. + """ + context.cli.sendline('drop table a;') + + +@then('we see table created') +def step_see_table_created(context): + """ + Wait to see create table output. + """ + wrappers.expect_exact(context, 'CREATE TABLE', timeout=2) + + +@then('we see record inserted') +def step_see_record_inserted(context): + """ + Wait to see insert output. + """ + wrappers.expect_exact(context, 'INSERT 0 1', timeout=2) + + +@then('we see record updated') +def step_see_record_updated(context): + """ + Wait to see update output. + """ + wrappers.expect_exact(context, 'UPDATE 1', timeout=2) + + +@then('we see data selected') +def step_see_data_selected(context): + """ + Wait to see select output. + """ + wrappers.expect_exact(context, 'yyy', timeout=1) + wrappers.expect_exact(context, 'SELECT 1', timeout=1) + + +@then('we see record deleted') +def step_see_data_deleted(context): + """ + Wait to see delete output. + """ + wrappers.expect_exact(context, 'DELETE 1', timeout=2) + + +@then('we see table dropped') +def step_see_table_dropped(context): + """ + Wait to see drop output. + """ + wrappers.expect_exact(context, 'DROP TABLE', timeout=2) diff --git a/tests/features/steps/iocommands.py b/tests/features/steps/iocommands.py new file mode 100644 index 000000000..885200469 --- /dev/null +++ b/tests/features/steps/iocommands.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 +from __future__ import unicode_literals +import os +import wrappers + +from behave import when, then + + +@when('we start external editor providing a file name') +def step_edit_file(context): + """ + Edit file with external editor. + """ + context.editor_file_name = 'test_file_{0}.sql'.format(context.conf['vi']) + if os.path.exists(context.editor_file_name): + os.remove(context.editor_file_name) + context.cli.sendline('\e {0}'.format(context.editor_file_name)) + wrappers.expect_exact(context, 'Entering Ex mode. Type "visual" to go to Normal mode.', timeout=2) + wrappers.expect_exact(context, '\r\n:', timeout=2) + + +@when('we type sql in the editor') +def step_edit_type_sql(context): + context.cli.sendline('i') + context.cli.sendline('select * from abc') + context.cli.sendline('.') + wrappers.expect_exact(context, ':', timeout=2) + + +@when('we exit the editor') +def step_edit_quit(context): + context.cli.sendline('x') + wrappers.expect_exact(context, "written", timeout=2) + + +@then('we see the sql in prompt') +def step_edit_done_sql(context): + for match in 'select * from abc'.split(' '): + wrappers.expect_exact(context, match, timeout=1) + # Cleanup the command line. + context.cli.sendcontrol('u') + # Cleanup the edited file. + if context.editor_file_name and os.path.exists(context.editor_file_name): + os.remove(context.editor_file_name) diff --git a/tests/features/steps/named_queries.py b/tests/features/steps/named_queries.py new file mode 100644 index 000000000..cd1273c2a --- /dev/null +++ b/tests/features/steps/named_queries.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import wrappers +from behave import when, then + + +@when('we save a named query') +def step_save_named_query(context): + """ + Send \ns command + """ + context.cli.sendline('\\ns foo SELECT 12345') + + +@when('we use a named query') +def step_use_named_query(context): + """ + Send \n command + """ + context.cli.sendline('\\n foo') + + +@when('we delete a named query') +def step_delete_named_query(context): + """ + Send \nd command + """ + context.cli.sendline('\\nd foo') + + +@then('we see the named query saved') +def step_see_named_query_saved(context): + """ + Wait to see query saved. + """ + wrappers.expect_exact(context, 'Saved.', timeout=1) + + +@then('we see the named query executed') +def step_see_named_query_executed(context): + """ + Wait to see select output. + """ + wrappers.expect_exact(context, '12345', timeout=1) + wrappers.expect_exact(context, 'SELECT 1', timeout=1) + + +@then('we see the named query deleted') +def step_see_named_query_deleted(context): + """ + Wait to see query deleted. + """ + wrappers.expect_exact(context, 'foo: Deleted', timeout=1) diff --git a/tests/features/steps/specials.py b/tests/features/steps/specials.py new file mode 100644 index 000000000..a5122c22e --- /dev/null +++ b/tests/features/steps/specials.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 +""" +Steps for behavioral style tests are defined in this module. +Each step is defined by the string decorating it. +This string is used to call the step in "*.feature" file. +""" +from __future__ import unicode_literals + +import wrappers +from behave import when, then + + +@when('we refresh completions') +def step_refresh_completions(context): + """ + Send refresh command. + """ + context.cli.sendline('\\refresh') + + +@then('we see completions refresh started') +def step_see_refresh_started(context): + """ + Wait to see refresh output. + """ + wrappers.expect_exact(context, 'refresh started in the background', timeout=2) diff --git a/tests/features/steps/wrappers.py b/tests/features/steps/wrappers.py new file mode 100644 index 000000000..eac7c8304 --- /dev/null +++ b/tests/features/steps/wrappers.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 +from __future__ import unicode_literals + +import re + + +def expect_exact(context, expected, timeout): + try: + context.cli.expect_exact(expected, timeout=timeout) + except: + # Strip color codes out of the output. + actual = re.sub(r'\x1b\[([0-9A-Za-z;?])+[m|K]?', '', context.cli.before) + raise Exception('Expected:\n---\n{0!r}\n---\n\nActual:\n---\n{1!r}\n---'.format( + expected, + actual)) From 48a9ab73eb2f2099d8b297d39684040173f6f0ed Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sun, 2 Apr 2017 20:40:10 +0200 Subject: [PATCH 0356/1025] Changes for differences between pgcli and mycli --- .coveragerc | 3 + .gitignore | 3 + .travis.yml | 6 +- changelog.md | 1 + requirements-dev.txt | 3 + tests/behave.ini | 5 -- tests/features/db_utils.py | 52 +++++++++-------- tests/features/environment.py | 58 ++++++++----------- tests/features/fixture_data/help_commands.txt | 48 ++++++++------- tests/features/fixture_utils.py | 5 +- tests/features/steps/basic_commands.py | 23 ++++++-- tests/features/steps/crud_database.py | 21 ++++--- tests/features/steps/crud_table.py | 17 +++--- tests/features/steps/named_queries.py | 6 +- tests/features/steps/specials.py | 4 +- 15 files changed, 143 insertions(+), 112 deletions(-) create mode 100644 .coveragerc delete mode 100644 tests/behave.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..ae818eefb --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +parallel=True +source=mycli diff --git a/.gitignore b/.gitignore index 12588dc82..59fa76be2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,12 @@ /dist /mycli.egg-info /src +/tests/behave.ini .vagrant *.pyc *.deb *.swp .cache/ +.coverage +.coverage.* diff --git a/.travis.yml b/.travis.yml index 1a7991836..c0473c48f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,16 @@ python: - "3.6" install: - - pip install PyMySQL . pytest mock codecov + - pip install PyMySQL . pytest mock codecov pexpect behave script: - coverage run --source mycli -m py.test + - cd tests + - behave + - cd .. after_success: + - coverage combine - codecov notifications: diff --git a/changelog.md b/changelog.md index 924d8efa4..7fe1b0243 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,7 @@ Bug Fixes: * Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) * Fix bug where mycli would not start due to the thanks/credit intro text. (Thanks: [Thomas Roten]). +* Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). Internal Changes: ----------------- diff --git a/requirements-dev.txt b/requirements-dev.txt index e54eabb9c..7511ac52e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,6 @@ mock pytest tox twine==1.8.1 +behave +pexpect +coverage==4.3.4 diff --git a/tests/behave.ini b/tests/behave.ini deleted file mode 100644 index c555d3c07..000000000 --- a/tests/behave.ini +++ /dev/null @@ -1,5 +0,0 @@ -[behave.userdata] -pg_test_user = postgres -pg_test_pass = -pg_test_host = localhost -pg_test_db = pgcli_behave_tests diff --git a/tests/features/db_utils.py b/tests/features/db_utils.py index 63c6586b7..f604c608c 100644 --- a/tests/features/db_utils.py +++ b/tests/features/db_utils.py @@ -2,9 +2,7 @@ from __future__ import unicode_literals from __future__ import print_function -from psycopg2 import connect -from psycopg2.extensions import AsIs - +import pymysql def create_db(hostname='localhost', username=None, password=None, dbname=None): @@ -16,15 +14,17 @@ def create_db(hostname='localhost', username=None, password=None, :param dbname: string :return: """ - cn = create_cn(hostname, password, username, 'postgres') - - # ISOLATION_LEVEL_AUTOCOMMIT = 0 - # Needed for DB creation. - cn.set_isolation_level(0) + cn = pymysql.connect( + host=hostname, + user=username, + password=password, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) with cn.cursor() as cr: - cr.execute('drop database if exists %s', (AsIs(dbname),)) - cr.execute('create database %s', (AsIs(dbname),)) + cr.execute('drop database if exists '+dbname) + cr.execute('create database '+dbname) cn.close() @@ -41,13 +41,15 @@ def create_cn(hostname, password, username, dbname): :param dbname: string :return: psycopg2.connection """ - if password: - cn = connect(host=hostname, user=username, database=dbname, - password=password) - else: - cn = connect(user=username, database=dbname) + cn = pymysql.connect( + host=hostname, + user=username, + password=password, + db=dbname, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) - print('Created connection: {0}.'.format(cn.dsn)) return cn @@ -60,14 +62,17 @@ def drop_db(hostname='localhost', username=None, password=None, :param password: string :param dbname: string """ - cn = create_cn(hostname, password, username, 'postgres') - - # ISOLATION_LEVEL_AUTOCOMMIT = 0 - # Needed for DB drop. - cn.set_isolation_level(0) + cn = pymysql.connect( + host=hostname, + user=username, + password=password, + db=dbname, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) with cn.cursor() as cr: - cr.execute('drop database if exists %s', (AsIs(dbname),)) + cr.execute('drop database if exists '+dbname) close_cn(cn) @@ -75,8 +80,7 @@ def drop_db(hostname='localhost', username=None, password=None, def close_cn(cn=None): """ Close connection. - :param connection: psycopg2.connection + :param connection: pymysql.connection """ if cn: cn.close() - print('Closed connection: {0}.'.format(cn.dsn)) diff --git a/tests/features/environment.py b/tests/features/environment.py index 51ae86b5a..3f55757b8 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -21,43 +21,31 @@ def before_all(context): context.exit_sent = False vi = '_'.join([str(x) for x in sys.version_info[:3]]) - db_name = context.config.userdata.get('pg_test_db', None) + db_name = context.config.userdata.get('my_test_db', None) or "mycli_behave_tests" db_name_full = '{0}_{1}'.format(db_name, vi) - # Store get params from config. + # Store get params from config/environment variables context.conf = { - 'host': context.config.userdata.get('pg_test_host', 'localhost'), - 'user': context.config.userdata.get('pg_test_user', 'postgres'), - 'pass': context.config.userdata.get('pg_test_pass', None), - 'dbname': db_name_full, + 'host': context.config.userdata.get( + 'my_test_host', + os.getenv('PYTEST_HOST', 'localhost') + ), + 'user': context.config.userdata.get( + 'my_test_user', + os.getenv('PYTEST_USER', 'root') + ), + 'pass': context.config.userdata.get( + 'my_test_pass', + os.getenv('PYTEST_PASSWORD', None) + ), + 'cli_command': context.config.userdata.get( + 'my_cli_command', None) or + sys.executable+' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"', + 'dbname': db_name, 'dbname_tmp': db_name_full + '_tmp', 'vi': vi, - 'cli_command': context.config.userdata.get('pg_cli_command', None) or - sys.executable + - ' -c "import coverage; coverage.process_startup(); import pgcli.main; pgcli.main.cli()"' } - # Store old env vars. - context.pgenv = { - 'PGDATABASE': os.environ.get('PGDATABASE', None), - 'PGUSER': os.environ.get('PGUSER', None), - 'PGHOST': os.environ.get('PGHOST', None), - 'PGPASSWORD': os.environ.get('PGPASSWORD', None), - } - - # Set new env vars. - os.environ['PGDATABASE'] = context.conf['dbname'] - os.environ['PGUSER'] = context.conf['user'] - os.environ['PGHOST'] = context.conf['host'] - - if context.conf['pass']: - os.environ['PGPASSWORD'] = context.conf['pass'] - else: - if 'PGPASSWORD' in os.environ: - del os.environ['PGPASSWORD'] - if 'PGHOST' in os.environ: - del os.environ['PGHOST'] - context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], context.conf['pass'], context.conf['dbname']) @@ -74,11 +62,11 @@ def after_all(context): context.conf['pass'], context.conf['dbname']) # Restore env vars. - for k, v in context.pgenv.items(): - if k in os.environ and v is None: - del os.environ[k] - elif v: - os.environ[k] = v + #for k, v in context.pgenv.items(): + # if k in os.environ and v is None: + # del os.environ[k] + # elif v: + # os.environ[k] = v def after_scenario(context, _): diff --git a/tests/features/fixture_data/help_commands.txt b/tests/features/fixture_data/help_commands.txt index 3f3dd0b95..1c5a08bf4 100644 --- a/tests/features/fixture_data/help_commands.txt +++ b/tests/features/fixture_data/help_commands.txt @@ -1,21 +1,27 @@ -Command -\# -\? -\c[onnect] database_name -\d [pattern] -\dT[S+] [pattern] -\df[+] [pattern] -\di[+] [pattern] -\dn[+] [pattern] -\ds[+] [pattern] -\dt[+] [pattern] -\du[+] [pattern] -\dv[+] [pattern] -\e [file] -\l -\n[+] [name] -\nd [name] -\ns name query -\refresh -\timing -\x \ No newline at end of file ++-------------+-------------------+---------------------------------------------------------+ +| Command | Shortcut | Description | +|-------------+-------------------+---------------------------------------------------------| +| \G | \G | Display results vertically. | +| \dt | \dt [table] | List or describe tables. | +| \e | \e | Edit command with editor. (uses $EDITOR) | +| \f | \f [name] | List or execute favorite queries. | +| \fd | \fd [name] | Delete a favorite query. | +| \fs | \fs name query | Save a favorite query. | +| \l | \l | List databases. | +| \timing | \t | Toggle timing of commands. | +| connect | \r | Reconnect to the database. Optional database argument. | +| exit | \q | Exit. | +| help | \? | Show this help. | +| nopager | \n | Disable pager, print to stdout. | +| notee | notee | stop writing to an output file | +| pager | \P [command] | Set PAGER. Print the query results via PAGER | +| prompt | \R | Change prompt format. | +| quit | \q | Quit. | +| rehash | \# | Refresh auto-completions. | +| source | \. filename | Execute commands from file. | +| status | \s | Get status information from the server. | +| system | system [command] | Execute a system commmand. | +| tableformat | \T | Change Table Type. | +| tee | tee [-o] filename | write to an output file (optionally overwrite using -o) | +| use | \u | Change to a new database. | ++-------------+-------------------+---------------------------------------------------------+ diff --git a/tests/features/fixture_utils.py b/tests/features/fixture_utils.py index 917baeb32..f3b490c40 100644 --- a/tests/features/fixture_utils.py +++ b/tests/features/fixture_utils.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from __future__ import print_function import os -import codecs +import io def read_fixture_lines(filename): @@ -13,7 +12,7 @@ def read_fixture_lines(filename): :return: list of strings """ lines = [] - for line in codecs.open(filename, 'rb', encoding='utf-8'): + for line in io.open(filename, 'r', encoding='utf8'): lines.append(line.strip()) return lines diff --git a/tests/features/steps/basic_commands.py b/tests/features/steps/basic_commands.py index 568c793e4..7aa476640 100644 --- a/tests/features/steps/basic_commands.py +++ b/tests/features/steps/basic_commands.py @@ -17,8 +17,20 @@ def step_run_cli(context): """ Run the process using pexpect. """ - cli_cmd = context.conf.get('cli_command') - context.cli = pexpect.spawnu(cli_cmd, cwd='..') + run_args = [] + if context.conf.get('host', None): + run_args.extend(('-h', context.conf['host'])) + if context.conf.get('user', None): + run_args.extend(('-u', context.conf['user'])) + if context.conf.get('pass', None): + run_args.extend(('-p', context.conf['pass'])) + if context.conf.get('dbname', None): + run_args.extend(('-D', context.conf['dbname'])) + cli_cmd = context.conf.get('cli_command', None) or sys.executable+' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' + + cmd_parts = [cli_cmd] + run_args + cmd = ' '.join(cmd_parts) + context.cli = pexpect.spawnu(cmd, cwd='..') context.exit_sent = False @@ -27,7 +39,10 @@ def step_wait_prompt(context): """ Make sure prompt is displayed. """ - wrappers.expect_exact(context, '{0}> '.format(context.conf['dbname']), timeout=5) + user = context.conf['user'] + host = context.conf['host'] + dbname = context.conf['dbname'] + wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format(user, host, dbname), timeout=5) @when('we send "ctrl + d"') @@ -44,4 +59,4 @@ def step_send_help(context): """ Send \? to see help. """ - context.cli.sendline('\?') + context.cli.sendline('\\?') diff --git a/tests/features/steps/crud_database.py b/tests/features/steps/crud_database.py index 97bdfa7e2..3eab34d9d 100644 --- a/tests/features/steps/crud_database.py +++ b/tests/features/steps/crud_database.py @@ -33,6 +33,8 @@ def step_db_drop(context): context.cli.sendline('drop database {0};'.format( context.conf['dbname_tmp'])) + wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + context.cli.sendline('y') @when('we connect to test database') def step_db_connect_test(context): @@ -40,7 +42,7 @@ def step_db_connect_test(context): Send connect to database. """ db_name = context.conf['dbname'] - context.cli.sendline('\\connect {0}'.format(db_name)) + context.cli.sendline('use {0}'.format(db_name)) @when('we connect to dbserver') @@ -48,7 +50,7 @@ def step_db_connect_dbserver(context): """ Send connect to database. """ - context.cli.sendline('\\connect postgres') + context.cli.sendline('use mysql') @then('dbcli exits') @@ -64,13 +66,16 @@ def step_see_prompt(context): """ Wait to see the prompt. """ - wrappers.expect_exact(context, '{0}> '.format(context.conf['dbname']), timeout=5) + user = context.conf['user'] + host = context.conf['host'] + dbname = context.conf['dbname'] + wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format(user, host, dbname), timeout=5) @then('we see help output') def step_see_help(context): for expected_line in context.fixture_data['help_commands.txt']: - wrappers.expect_exact(context, expected_line, timeout=1) + wrappers.expect_exact(context, expected_line+'\r\n', timeout=1) @then('we see database created') @@ -78,7 +83,7 @@ def step_see_db_created(context): """ Wait to see create database output. """ - wrappers.expect_exact(context, 'CREATE DATABASE', timeout=2) + wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see database dropped') @@ -86,7 +91,7 @@ def step_see_db_dropped(context): """ Wait to see drop database output. """ - wrappers.expect_exact(context, 'DROP DATABASE', timeout=2) + wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see database connected') @@ -94,4 +99,6 @@ def step_see_db_connected(context): """ Wait to see drop database output. """ - wrappers.expect_exact(context, 'You are now connected to database', timeout=2) + wrappers.expect_exact(context, 'You are now connected to database "', timeout=2) + wrappers.expect_exact(context, '"', timeout=2) + wrappers.expect_exact(context, ' as user "{0}"\r\n'.format(context.conf['user']), timeout=2) diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py index 0863a224f..72b4e0800 100644 --- a/tests/features/steps/crud_table.py +++ b/tests/features/steps/crud_table.py @@ -48,6 +48,8 @@ def step_delete_from_table(context): Send deete from table. """ context.cli.sendline('''delete from a where x = 'yyy';''') + wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + context.cli.sendline('y') @when('we drop table') @@ -56,6 +58,8 @@ def step_drop_table(context): Send drop table. """ context.cli.sendline('drop table a;') + wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + context.cli.sendline('y') @then('we see table created') @@ -63,7 +67,7 @@ def step_see_table_created(context): """ Wait to see create table output. """ - wrappers.expect_exact(context, 'CREATE TABLE', timeout=2) + wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see record inserted') @@ -71,7 +75,7 @@ def step_see_record_inserted(context): """ Wait to see insert output. """ - wrappers.expect_exact(context, 'INSERT 0 1', timeout=2) + wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see record updated') @@ -79,7 +83,7 @@ def step_see_record_updated(context): """ Wait to see update output. """ - wrappers.expect_exact(context, 'UPDATE 1', timeout=2) + wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see data selected') @@ -87,8 +91,7 @@ def step_see_data_selected(context): """ Wait to see select output. """ - wrappers.expect_exact(context, 'yyy', timeout=1) - wrappers.expect_exact(context, 'SELECT 1', timeout=1) + wrappers.expect_exact(context, '+-----+\r\n| x |\r\n|-----|\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') @@ -96,7 +99,7 @@ def step_see_data_deleted(context): """ Wait to see delete output. """ - wrappers.expect_exact(context, 'DELETE 1', timeout=2) + wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see table dropped') @@ -104,4 +107,4 @@ def step_see_table_dropped(context): """ Wait to see drop output. """ - wrappers.expect_exact(context, 'DROP TABLE', timeout=2) + wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) diff --git a/tests/features/steps/named_queries.py b/tests/features/steps/named_queries.py index cd1273c2a..b53ad47db 100644 --- a/tests/features/steps/named_queries.py +++ b/tests/features/steps/named_queries.py @@ -15,7 +15,7 @@ def step_save_named_query(context): """ Send \ns command """ - context.cli.sendline('\\ns foo SELECT 12345') + context.cli.sendline('\\fs foo SELECT 12345') @when('we use a named query') @@ -23,7 +23,7 @@ def step_use_named_query(context): """ Send \n command """ - context.cli.sendline('\\n foo') + context.cli.sendline('\\f foo') @when('we delete a named query') @@ -31,7 +31,7 @@ def step_delete_named_query(context): """ Send \nd command """ - context.cli.sendline('\\nd foo') + context.cli.sendline('\\fd foo') @then('we see the named query saved') diff --git a/tests/features/steps/specials.py b/tests/features/steps/specials.py index a5122c22e..790b2476f 100644 --- a/tests/features/steps/specials.py +++ b/tests/features/steps/specials.py @@ -15,7 +15,7 @@ def step_refresh_completions(context): """ Send refresh command. """ - context.cli.sendline('\\refresh') + context.cli.sendline('rehash') @then('we see completions refresh started') @@ -23,4 +23,4 @@ def step_see_refresh_started(context): """ Wait to see refresh output. """ - wrappers.expect_exact(context, 'refresh started in the background', timeout=2) + wrappers.expect_exact(context, 'Auto-completion refresh started in the background', timeout=2) From d1f624780ab5217dd06f0b872d2bed0d82e54dfa Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 4 Apr 2017 18:48:23 -0500 Subject: [PATCH 0357/1025] Add --myclirc option to changelog. --- changelog.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 7fe1b0243..06cd58880 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Features: +--------- + +* Add ability to specify alternative myclirc file. (Thanks: [Dick Marinus]). + Bug Fixes: ---------- @@ -10,13 +15,13 @@ Bug Fixes: * Fix requirements and remove old compatibility code (Thanks: [Dick Marinus]) * Fix bug where mycli would not start due to the thanks/credit intro text. (Thanks: [Thomas Roten]). -* Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). Internal Changes: ----------------- * Upload mycli distributions in a safer manner (using twine). (Thanks: [Thomas Roten]). +* Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). 1.9.0: ====== From 549d9ad683a67ae16ef08fc92ba62c02252cc737 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 4 Apr 2017 18:57:56 -0500 Subject: [PATCH 0358/1025] Fix behave tests table formatting. --- tests/features/fixture_data/help_commands.txt | 2 +- tests/features/steps/crud_table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/features/fixture_data/help_commands.txt b/tests/features/fixture_data/help_commands.txt index 1c5a08bf4..ee3d4ca47 100644 --- a/tests/features/fixture_data/help_commands.txt +++ b/tests/features/fixture_data/help_commands.txt @@ -1,6 +1,6 @@ +-------------+-------------------+---------------------------------------------------------+ | Command | Shortcut | Description | -|-------------+-------------------+---------------------------------------------------------| ++-------------+-------------------+---------------------------------------------------------+ | \G | \G | Display results vertically. | | \dt | \dt [table] | List or describe tables. | | \e | \e | Edit command with editor. (uses $EDITOR) | diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py index 72b4e0800..a4ed0fe53 100644 --- a/tests/features/steps/crud_table.py +++ b/tests/features/steps/crud_table.py @@ -91,7 +91,7 @@ def step_see_data_selected(context): """ Wait to see select output. """ - wrappers.expect_exact(context, '+-----+\r\n| x |\r\n|-----|\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) + wrappers.expect_exact(context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') From 7b72b05dcafc1567a473fc5ea542b33dcdfe8e23 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 4 Apr 2017 19:08:55 -0500 Subject: [PATCH 0359/1025] Update README app usage. --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6ecca526c..c12a8eb5e 100644 --- a/README.md +++ b/README.md @@ -43,11 +43,11 @@ $ sudo apt-get install mycli # Only on debian or ubuntu Options: -h, --host TEXT Host address of the database. - -P, --port TEXT Port number to use for connection. Honors + -P, --port INTEGER Port number to use for connection. Honors $MYSQL_TCP_PORT -u, --user TEXT User name to connect to the database. -S, --socket TEXT The socket file to use for connection. - -p, --password Force password prompt. + -p, --password TEXT Password to connect to the database --pass TEXT Password to connect to the database --ssl-ca PATH CA file in PEM format --ssl-capath TEXT CA directory @@ -58,18 +58,21 @@ $ sudo apt-get install mycli # Only on debian or ubuntu against hostname used when connecting. This option is disabled by default -v, --version Version of mycli. - -D, --database TEXT Database to use. + -D, --database TEXT Database to use. -R, --prompt TEXT Prompt format (Default: "\t \u@\h:\d> ") -l, --logfile FILENAME Log every query and its results to a file. --defaults-group-suffix TEXT Read config group with the specified suffix. --defaults-file PATH Only read default options from the given file + --myclirc PATH Location of myclirc file. --auto-vertical-output Automatically switch to vertical output mode if the result is wider than the terminal width. -t, --table Display batch output in table format. + --csv Display batch output in CSV format. --warn / --no-warn Warn before running a destructive query. --local-infile BOOLEAN Enable/disable LOAD DATA LOCAL INFILE. --login-path TEXT Read this path from the login file. + -e, --execute TEXT Execute query to the database. --help Show this message and exit. ### Examples From 26400cb895bfaee690d12815381a72a578e087f6 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 4 Apr 2017 19:12:24 -0500 Subject: [PATCH 0360/1025] Add README check to release script. --- release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/release.py b/release.py index 9a5d399f9..817695838 100755 --- a/release.py +++ b/release.py @@ -101,6 +101,7 @@ def checklist(questions): checks = ['Have you created the debian package?', 'Have you updated the AUTHORS file?', + 'Have you updated the `Usage` section of the README?', ] checklist(checks) From 03e4050128f9948c7ebf2862d3b854a1b9c91344 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Wed, 5 Apr 2017 18:20:39 -0700 Subject: [PATCH 0361/1025] Added pep8radius. --- .travis.yml | 3 +++ requirements-dev.txt | 1 + 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index c0473c48f..f78d94b00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,12 +8,15 @@ python: install: - pip install PyMySQL . pytest mock codecov pexpect behave + - pip install git+https://github.com/hayd/pep8radius.git script: - coverage run --source mycli -m py.test - cd tests - behave - cd .. + # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. + - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) after_success: - coverage combine diff --git a/requirements-dev.txt b/requirements-dev.txt index 7511ac52e..b7e6e2dca 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,3 +5,4 @@ twine==1.8.1 behave pexpect coverage==4.3.4 +pep8radius From 74a739fe9e35af3421e3ffc59620e7af031a0e81 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Wed, 5 Apr 2017 18:38:26 -0700 Subject: [PATCH 0362/1025] Added change to changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 06cd58880..fcf9c198d 100644 --- a/changelog.md +++ b/changelog.md @@ -22,6 +22,7 @@ Internal Changes: * Upload mycli distributions in a safer manner (using twine). (Thanks: [Thomas Roten]). * Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). +* Run pep8 checks in travis (Thanks: [Irina Truong]). 1.9.0: ====== From 079d9f870debfed03a8ec0351859ddd0a0670df0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 5 Apr 2017 22:04:04 -0500 Subject: [PATCH 0363/1025] PEP8 changes per pep8radius request. --- mycli/encodingutils.py | 15 +++++++++------ mycli/main.py | 15 +++++++++------ mycli/output_formatter/expanded.py | 1 + mycli/output_formatter/output_formatter.py | 4 +++- mycli/output_formatter/preprocessors.py | 10 ++++++++-- mycli/sqlexecute.py | 12 ++++++------ tests/features/steps/crud_table.py | 3 ++- tests/test_output_formatter.py | 3 ++- tests/utils.py | 3 ++- 9 files changed, 42 insertions(+), 24 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index aac4aa07d..1a8b5bbb1 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -16,9 +16,10 @@ def unicode2utf8(arg): - """ - Only in Python 2. Psycopg2 expects the args as bytes not unicode. - In Python 3 the args are expected as unicode. + """Convert strings to UTF8-encoded bytes. + + Only in Python 2. In Python 3 the args are expected as unicode. + """ if PY2 and isinstance(arg, text_type): @@ -27,9 +28,10 @@ def unicode2utf8(arg): def utf8tounicode(arg): - """ - Only in Python 2. Psycopg2 returns the error message as utf-8. - In Python 3 the errors are returned as unicode. + """Convert UTF8-encoded bytes to strings. + + Only in Python 2. In Python 3 the errors are returned as strings. + """ if PY2 and isinstance(arg, binary_type): @@ -46,6 +48,7 @@ def bytes_to_string(b): abc >>> print(bytes_to_string('✌')) ✌ + """ if isinstance(b, binary_type): try: diff --git a/mycli/main.py b/mycli/main.py index aea9c7301..64c587057 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -106,7 +106,8 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.formatter = output_formatter.OutputFormatter(format_name=c['main']['table_format']) + self.formatter = output_formatter.OutputFormatter( + format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] @@ -146,7 +147,7 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. self.smart_completion = c['main'].as_bool('smart_completion') self.completer = SQLCompleter(self.smart_completion, - supported_formats=self.formatter.supported_formats) + supported_formats=self.formatter.supported_formats) self._completer_lock = threading.Lock() # Register custom special commands. @@ -186,7 +187,8 @@ def change_table_format(self, arg, **_): yield (None, None, None, 'Changed table type to {}'.format(arg)) except ValueError: - msg = 'Table type {} not yet implemented. Allowed types:'.format(arg) + msg = 'Table type {} not yet implemented. Allowed types:'.format( + arg) for table_type in self.formatter.supported_formats(): msg += "\n\t{}".format(table_type) yield (None, None, None, msg) @@ -524,8 +526,9 @@ def one_iteration(document=None): else: max_width = None - formatted = self.format_output(title, cur, headers, - status, special.is_expanded_output(), max_width) + formatted = self.format_output(title, cur, headers, status, + special.is_expanded_output(), + max_width) output.extend(formatted) end = time() @@ -706,7 +709,7 @@ def get_prompt(self, string): return string def run_query(self, query, new_line=True): - """Runs query""" + """Runs *query*.""" results = self.sqlexecute.run(query) for result in results: title, cur, headers, status = result diff --git a/mycli/output_formatter/expanded.py b/mycli/output_formatter/expanded.py index 2ab579944..f77c1ee38 100644 --- a/mycli/output_formatter/expanded.py +++ b/mycli/output_formatter/expanded.py @@ -19,6 +19,7 @@ def expanded_table(rows, headers, **_): """Format *rows* and *headers* as an expanded table. The values in *rows* and *headers* must be strings. + """ header_len = max([len(x) for x in headers]) padded_headers = [x.ljust(header_len) for x in headers] diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index a47029cca..759fb879a 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -59,12 +59,14 @@ def format_output(self, data, headers, format_name=None, **kwargs): *format_name* must be a formatter available in `supported_formats()`. All keyword arguments are passed to the specified formatter. + """ format_name = format_name or self._format_name if format_name not in self.supported_formats(): raise ValueError('unrecognized format: {}'.format(format_name)) - (_, preprocessors, formatter, fkwargs) = self._output_formats[format_name] + (_, preprocessors, formatter, + fkwargs) = self._output_formats[format_name] fkwargs.update(kwargs) if preprocessors: for f in preprocessors: diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index d6db3fd44..010c1b14c 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -30,13 +30,15 @@ def bytes_to_string(data, headers, **_): def intlen(value): - """Find (character) length + """Find (character) length. + >>> intlen('11.1') 2 >>> intlen('11') 2 >>> intlen('1.1') 1 + """ pos = value.find('.') if pos < 0: @@ -45,11 +47,13 @@ def intlen(value): def align_decimals(data, headers, **_): - """Align decimals to decimal point + """Align decimals to decimal point. + >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) 1 11.1 1.1 + """ pointpos = len(data[0]) * [0] for row in data: @@ -72,6 +76,7 @@ def align_decimals(data, headers, **_): def quote_whitespaces(data, headers, quotestyle="'", **_): """Quote whitespace + >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) ' before' 'after ' @@ -82,6 +87,7 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): def ghi jkl + """ quote = len(data[0]) * [False] for row in data: diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 08030af47..4d5c0a091 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -4,7 +4,7 @@ from .packages import special from pymysql.constants import FIELD_TYPE from pymysql.converters import (convert_mysql_timestamp, convert_datetime, - convert_timedelta, convert_date, conversions) + convert_timedelta, convert_date, conversions) _logger = logging.getLogger(__name__) @@ -67,11 +67,11 @@ def connect(self, database=None, user=None, password=None, host=None, database, user, host, port, socket, charset, local_infile, ssl) conv = conversions.copy() conv.update({ - FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), - FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), - FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), - FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), - }) + FIELD_TYPE.TIMESTAMP: lambda obj: (convert_mysql_timestamp(obj) or obj), + FIELD_TYPE.DATETIME: lambda obj: (convert_datetime(obj) or obj), + FIELD_TYPE.TIME: lambda obj: (convert_timedelta(obj) or obj), + FIELD_TYPE.DATE: lambda obj: (convert_date(obj) or obj), + }) conn = pymysql.connect(database=db, user=user, password=password, host=host, port=port, unix_socket=socket, diff --git a/tests/features/steps/crud_table.py b/tests/features/steps/crud_table.py index a4ed0fe53..b73ff0d6f 100644 --- a/tests/features/steps/crud_table.py +++ b/tests/features/steps/crud_table.py @@ -91,7 +91,8 @@ def step_see_data_selected(context): """ Wait to see select output. """ - wrappers.expect_exact(context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) + wrappers.expect_exact( + context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index 7949032fd..ac5cfc073 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -57,7 +57,8 @@ def test_bytes_to_string(): def test_align_decimals(): """Test the *output_formatter.align_decimals()* function.""" - data = [[Decimal('200'), Decimal('1')], [Decimal('1.00002'), Decimal('1.0')]] + data = [[Decimal('200'), Decimal('1')], [ + Decimal('1.00002'), Decimal('1.0')]] headers = ['num1', 'num2'] expected = ([['200', '1'], [' 1.00002', '1.0']], ['num1', 'num2']) diff --git a/tests/utils.py b/tests/utils.py index bff31045d..b29e5e072 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,7 +44,8 @@ def run(executor, sql, join=False): # It should test raw results. mycli = MyCli() for title, rows, headers, status in executor.run(sql): - result.extend(mycli.format_output(title, rows, headers, status, special.is_expanded_output())) + result.extend(mycli.format_output(title, rows, headers, status, + special.is_expanded_output())) if join: result = '\n'.join(result) From 8bc0fc41db40bb86c35d159fbbab00bc34d5b092 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 5 Apr 2017 22:05:59 -0500 Subject: [PATCH 0364/1025] Add period for pep8radius. --- mycli/output_formatter/preprocessors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index 010c1b14c..c2e221f8f 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -75,7 +75,7 @@ def align_decimals(data, headers, **_): def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote whitespace + """Quote whitespace. >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) ' before' From 5c4c675d8019d36f59f2aedb55f02adb50b151a1 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Wed, 5 Apr 2017 21:09:42 -0700 Subject: [PATCH 0365/1025] Documented pep8radius usage. --- DEVELOP.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/DEVELOP.rst b/DEVELOP.rst index 5169abeb8..0d53e6ad7 100644 --- a/DEVELOP.rst +++ b/DEVELOP.rst @@ -70,3 +70,17 @@ after asking a few questions like maintainer name, email etc. $ vagrant up +PEP8 checks +----------- + +When you submit a PR, the changeset is checked for pep8 compliance using +`pep8radius `_. If you see a build failing because +of these checks, install pep8radius and apply style fixes: + +:: + + $ pip install pep8radius + $ pep8radius --docformatter --diff # view a diff of proposed fixes + $ pep8radius --docformatter --in-place # apply the fixes + +Then commit and push the fixes. From 935fc568f2b33f641c37cbd743ec2a213b82dc81 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 10 Apr 2017 08:06:18 -0500 Subject: [PATCH 0366/1025] Do not reference results rows by index. --- mycli/output_formatter/preprocessors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index c2e221f8f..8fe73bb73 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -55,7 +55,7 @@ def align_decimals(data, headers, **_): 1.1 """ - pointpos = len(data[0]) * [0] + pointpos = len(headers) * [0] for row in data: for i, v in enumerate(row): if isinstance(v, Decimal): @@ -89,7 +89,7 @@ def quote_whitespaces(data, headers, quotestyle="'", **_): jkl """ - quote = len(data[0]) * [False] + quote = len(headers) * [False] for row in data: for i, v in enumerate(row): v = encodingutils.text_type(v) From 5849ac9261a9c5c68a10bd5b3fceee88c2daf4b2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 10 Apr 2017 08:15:21 -0500 Subject: [PATCH 0367/1025] Add empty result tests. --- mycli/output_formatter/preprocessors.py | 24 ++----------------- tests/test_output_formatter.py | 31 ++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py index 8fe73bb73..6f2e459c0 100644 --- a/mycli/output_formatter/preprocessors.py +++ b/mycli/output_formatter/preprocessors.py @@ -47,14 +47,7 @@ def intlen(value): def align_decimals(data, headers, **_): - """Align decimals to decimal point. - - >>> for i in align_decimals([[Decimal(1)], [Decimal('11.1')], [Decimal('1.1')]], [])[0]: print(i[0]) - 1 - 11.1 - 1.1 - - """ + """Align decimals to decimal point.""" pointpos = len(headers) * [0] for row in data: for i, v in enumerate(row): @@ -75,20 +68,7 @@ def align_decimals(data, headers, **_): def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote whitespace. - - >>> for i in quote_whitespaces([[" before"], ["after "], [" both "], ["none"]], [])[0]: print(i[0]) - ' before' - 'after ' - ' both ' - 'none' - >>> for i in quote_whitespaces([["abc"], ["def"], ["ghi"], ["jkl"]], [])[0]: print(i[0]) - abc - def - ghi - jkl - - """ + """Quote leading/trailing whitespace.""" quote = len(headers) * [False] for row in data: for i, v in enumerate(row): diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index ac5cfc073..dcc26e62c 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -8,6 +8,7 @@ from mycli.output_formatter.preprocessors import (align_decimals, bytes_to_string, convert_to_string, + quote_whitespaces, override_missing_value, to_string) from mycli.output_formatter.output_formatter import OutputFormatter @@ -56,7 +57,7 @@ def test_bytes_to_string(): def test_align_decimals(): - """Test the *output_formatter.align_decimals()* function.""" + """Test the *align_decimals()* function.""" data = [[Decimal('200'), Decimal('1')], [ Decimal('1.00002'), Decimal('1.0')]] headers = ['num1', 'num2'] @@ -65,6 +66,34 @@ def test_align_decimals(): assert expected == align_decimals(data, headers) +def test_align_decimals_empty_result(): + """Test *align_decimals()* with no results.""" + data = [] + headers = ['num1', 'num2'] + expected = ([], ['num1', 'num2']) + + assert expected == align_decimals(data, headers) + + +def test_quote_whitespaces(): + """Test the *quote_whitespaces()* function.""" + data = [[" before", "after "], [" both ", "none"]] + headers = ['h1', 'h2'] + expected = ([["' before'", "'after '"], ["' both '", "'none'"]], + ['h1', 'h2']) + + assert expected == quote_whitespaces(data, headers) + + +def test_quote_whitespaces_empty_result(): + """Test the *quote_whitespaces()* function with no results.""" + data = [] + headers = ['h1', 'h2'] + expected = ([], ['h1', 'h2']) + + assert expected == quote_whitespaces(data, headers) + + def test_tabulate_wrapper(): """Test the *output_formatter.tabulate_wrapper()* function.""" data = [['abc', 1], ['d', 456]] From 385f47892fbf2e8c8b0fdf3bde174a2128a3b255 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 14:49:03 -0500 Subject: [PATCH 0368/1025] Revert "Remove tabulate license note." This reverts commit d0f1dbf441ae823d56cadca456b1ff35a35abd88. --- LICENSE.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/LICENSE.txt b/LICENSE.txt index 8afaa2657..9a41a67d4 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -27,3 +27,9 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------- + +This program also bundles with it python-tabulate +(https://pypi.python.org/pypi/tabulate) library. This library is licensed under +MIT License. + +------------------------------------------------------------------------------- From 16738b1dbcb6b146175e3308352c113006b5b3ca Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:13:03 -0500 Subject: [PATCH 0369/1025] Add vendored tabulate version 0.8.0. --- mycli/output_formatter/tabulate_adapter.py | 11 +- mycli/packages/tabulate.py | 1423 ++++++++++++++++++++ setup.py | 1 - 3 files changed, 1428 insertions(+), 7 deletions(-) create mode 100644 mycli/packages/tabulate.py diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 0db31ff9f..14e6c770f 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,15 +1,14 @@ -from tabulate import tabulate - -from .preprocessors import bytes_to_string, align_decimals +from mycli.packages import tabulate +from .preprocessors import bytes_to_string, align_decimals, quote_whitespaces supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', 'html', 'latex', 'latex_booktabs', 'textile') -preprocessors = (bytes_to_string, align_decimals) +preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate(data, headers, tablefmt=table_format, - missingval=missing_value, disable_numparse=True) + return tabulate.tabulate(data, headers, tablefmt=table_format, + missingval=missing_value, disable_numparse=True) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py new file mode 100644 index 000000000..a25032995 --- /dev/null +++ b/mycli/packages/tabulate.py @@ -0,0 +1,1423 @@ +# -*- coding: utf-8 -*- + +"""Pretty-print tabular data.""" + +from __future__ import print_function +from __future__ import unicode_literals +from collections import namedtuple, Iterable +from platform import python_version_tuple +import re + + +if python_version_tuple()[0] < "3": + from itertools import izip_longest + from functools import partial + _none_type = type(None) + _bool_type = bool + _int_type = int + _long_type = long + _float_type = float + _text_type = unicode + _binary_type = str + + def _is_file(f): + return isinstance(f, file) + +else: + from itertools import zip_longest as izip_longest + from functools import reduce, partial + _none_type = type(None) + _bool_type = bool + _int_type = int + _long_type = int + _float_type = float + _text_type = str + _binary_type = bytes + basestring = str + + import io + + def _is_file(f): + return isinstance(f, io.IOBase) + +try: + import wcwidth # optional wide-character (CJK) support +except ImportError: + wcwidth = None + + +__all__ = ["tabulate", "tabulate_formats"] +__version__ = "0.8.0" + + +# minimum extra space in headers +MIN_PADDING = 2 + + +_DEFAULT_FLOATFMT = "g" +_DEFAULT_MISSINGVAL = "" + + +# if True, enable wide-character (CJK) support +WIDE_CHARS_MODE = wcwidth is not None + + +Line = namedtuple("Line", ["begin", "hline", "sep", "end"]) + + +DataRow = namedtuple("DataRow", ["begin", "sep", "end"]) + + +# A table structure is suppposed to be: +# +# --- lineabove --------- +# headerrow +# --- linebelowheader --- +# datarow +# --- linebewteenrows --- +# ... (more datarows) ... +# --- linebewteenrows --- +# last datarow +# --- linebelow --------- +# +# TableFormat's line* elements can be +# +# - either None, if the element is not used, +# - or a Line tuple, +# - or a function: [col_widths], [col_alignments] -> string. +# +# TableFormat's *row elements can be +# +# - either None, if the element is not used, +# - or a DataRow tuple, +# - or a function: [cell_values], [col_widths], [col_alignments] -> string. +# +# padding (an integer) is the amount of white space around data values. +# +# with_header_hide: +# +# - either None, to display all table elements unconditionally, +# - or a list of elements not to be displayed if the table has column +# headers. +# +TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", + "linebetweenrows", "linebelow", + "headerrow", "datarow", + "padding", "with_header_hide"]) + + +def _pipe_segment_with_colons(align, colwidth): + """Return a segment of a horizontal line with optional colons which + indicate column's alignment (as in `pipe` output format).""" + w = colwidth + if align in ["right", "decimal"]: + return ('-' * (w - 1)) + ":" + elif align == "center": + return ":" + ('-' * (w - 2)) + ":" + elif align == "left": + return ":" + ('-' * (w - 1)) + else: + return '-' * w + + +def _pipe_line_with_colons(colwidths, colaligns): + """Return a horizontal line with optional colons to indicate column's + alignment (as in `pipe` output format).""" + segments = [_pipe_segment_with_colons(a, w) for a, w in + zip(colaligns, colwidths)] + return "|" + "|".join(segments) + "|" + + +def _mediawiki_row_with_attrs(separator, cell_values, colwidths, colaligns): + alignment = {"left": '', + "right": 'align="right"| ', + "center": 'align="center"| ', + "decimal": 'align="right"| '} + # hard-coded padding _around_ align attribute and value together + # rather than padding parameter which affects only the value + values_with_attrs = [' ' + alignment.get(a, '') + c + ' ' + for c, a in zip(cell_values, colaligns)] + colsep = separator*2 + return (separator + colsep.join(values_with_attrs)).rstrip() + + +def _textile_row_with_attrs(cell_values, colwidths, colaligns): + cell_values[0] += ' ' + alignment = {"left": "<.", "right": ">.", "center": "=.", "decimal": ">."} + values = (alignment.get(a, '') + v for a, v in zip(colaligns, cell_values)) + return '|' + '|'.join(values) + '|' + + +def _html_begin_table_without_header(colwidths_ignore, colaligns_ignore): + # this table header will be suppressed if there is a header row + return "\n".join(["", ""]) + + +def _html_row_with_attrs(celltag, cell_values, colwidths, colaligns): + alignment = {"left": '', + "right": ' style="text-align: right;"', + "center": ' style="text-align: center;"', + "decimal": ' style="text-align: right;"'} + values_with_attrs = ["<{0}{1}>{2}".format( + celltag, alignment.get(a, ''), c) for c, a in + zip(cell_values, colaligns)] + rowhtml = "" + "".join(values_with_attrs).rstrip() + "" + if celltag == "th": # it's a header row, create a new table header + rowhtml = "\n".join(["
", + "", + rowhtml, + "", + ""]) + return rowhtml + + +def _moin_row_with_attrs(celltag, cell_values, colwidths, colaligns, + header=''): + alignment = {"left": '', + "right": '', + "center": '', + "decimal": ''} + values_with_attrs = ["{0}{1} {2} ".format(celltag, + alignment.get(a, ''), + header+c+header) + for c, a in zip(cell_values, colaligns)] + return "".join(values_with_attrs)+"||" + + +def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): + alignment = {"left": "l", "right": "r", "center": "c", "decimal": "r"} + tabular_columns_fmt = "".join([alignment.get(a, "l") for a in colaligns]) + return "\n".join(["\\begin{tabular}{" + tabular_columns_fmt + "}", + "\\toprule" if booktabs else "\hline"]) + + +LATEX_ESCAPE_RULES = {r"&": r"\&", r"%": r"\%", r"$": r"\$", r"#": r"\#", + r"_": r"\_", r"^": r"\^{}", r"{": r"\{", r"}": r"\}", + r"~": r"\textasciitilde{}", "\\": r"\textbackslash{}", + r"<": r"\ensuremath{<}", r">": r"\ensuremath{>}"} + + +def _latex_row(cell_values, colwidths, colaligns, escrules=LATEX_ESCAPE_RULES): + def escape_char(c): + return escrules.get(c, c) + escaped_values = ["".join(map(escape_char, cell)) for cell in cell_values] + rowfmt = DataRow("", "&", "\\\\") + return _build_simple_row(escaped_values, rowfmt) + + +def _rst_escape_first_column(rows, headers): + def escape_empty(val): + if isinstance(val, (_text_type, _binary_type)) and val.strip() is "": + return ".." + else: + return val + new_headers = list(headers) + new_rows = [] + if headers: + new_headers[0] = escape_empty(headers[0]) + for row in rows: + new_row = list(row) + if new_row: + new_row[0] = escape_empty(row[0]) + new_rows.append(new_row) + return new_rows, new_headers + + +_table_formats = {"simple": + TableFormat( + lineabove=Line("", "-", " ", ""), + linebelowheader=Line("", "-", " ", ""), + linebetweenrows=None, + linebelow=Line("", "-", " ", ""), + headerrow=DataRow("", " ", ""), + datarow=DataRow("", " ", ""), + padding=0, + with_header_hide=["lineabove", "linebelow"]), + "plain": + TableFormat( + lineabove=None, linebelowheader=None, + linebetweenrows=None, linebelow=None, + headerrow=DataRow("", " ", ""), + datarow=DataRow("", " ", ""), + padding=0, with_header_hide=None), + "grid": + TableFormat( + lineabove=Line("+", "-", "+", "+"), + linebelowheader=Line("+", "=", "+", "+"), + linebetweenrows=Line("+", "-", "+", "+"), + linebelow=Line("+", "-", "+", "+"), + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "fancy_grid": + TableFormat( + lineabove=Line("╒", "═", "╤", "╕"), + linebelowheader=Line("╞", "═", "╪", "╡"), + linebetweenrows=Line("├", "─", "┼", "┤"), + linebelow=Line("╘", "═", "╧", "╛"), + headerrow=DataRow("│", "│", "│"), + datarow=DataRow("│", "│", "│"), + padding=1, with_header_hide=None), + "pipe": + TableFormat( + lineabove=_pipe_line_with_colons, + linebelowheader=_pipe_line_with_colons, + linebetweenrows=None, + linebelow=None, + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, + with_header_hide=["lineabove"]), + "orgtbl": + TableFormat( + lineabove=None, + linebelowheader=Line("|", "-", "+", "|"), + linebetweenrows=None, + linebelow=None, + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "jira": + TableFormat( + lineabove=None, + linebelowheader=None, + linebetweenrows=None, + linebelow=None, + headerrow=DataRow("||", "||", "||"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "psql": + TableFormat( + lineabove=Line("+", "-", "+", "+"), + linebelowheader=Line("|", "-", "+", "|"), + linebetweenrows=None, + linebelow=Line("+", "-", "+", "+"), + headerrow=DataRow("|", "|", "|"), + datarow=DataRow("|", "|", "|"), + padding=1, with_header_hide=None), + "rst": + TableFormat( + lineabove=Line("", "=", " ", ""), + linebelowheader=Line("", "=", " ", ""), + linebetweenrows=None, + linebelow=Line("", "=", " ", ""), + headerrow=DataRow("", " ", ""), + datarow=DataRow("", " ", ""), + padding=0, with_header_hide=None), + "mediawiki": + TableFormat(lineabove=Line( + "{| class=\"wikitable\" style=\"text-align: left;\"", + "", "", "\n|+ \n|-"), + linebelowheader=Line("|-", "", "", ""), + linebetweenrows=Line("|-", "", "", ""), + linebelow=Line("|}", "", "", ""), + headerrow=partial(_mediawiki_row_with_attrs, "!"), + datarow=partial(_mediawiki_row_with_attrs, "|"), + padding=0, with_header_hide=None), + "moinmoin": + TableFormat( + lineabove=None, + linebelowheader=None, + linebetweenrows=None, + linebelow=None, + headerrow=partial(_moin_row_with_attrs, "||", + header="'''"), + datarow=partial(_moin_row_with_attrs, "||"), + padding=1, with_header_hide=None), + "html": + TableFormat( + lineabove=_html_begin_table_without_header, + linebelowheader="", + linebetweenrows=None, + linebelow=Line("\n
", "", "", ""), + headerrow=partial(_html_row_with_attrs, "th"), + datarow=partial(_html_row_with_attrs, "td"), + padding=0, with_header_hide=["lineabove"]), + "latex": + TableFormat( + lineabove=_latex_line_begin_tabular, + linebelowheader=Line("\\hline", "", "", ""), + linebetweenrows=None, + linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), + headerrow=_latex_row, + datarow=_latex_row, + padding=1, with_header_hide=None), + "latex_raw": + TableFormat( + lineabove=_latex_line_begin_tabular, + linebelowheader=Line("\\hline", "", "", ""), + linebetweenrows=None, + linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), + headerrow=partial(_latex_row, escrules={}), + datarow=partial(_latex_row, escrules={}), + padding=1, with_header_hide=None), + "latex_booktabs": + TableFormat( + lineabove=partial(_latex_line_begin_tabular, + booktabs=True), + linebelowheader=Line("\\midrule", "", "", ""), + linebetweenrows=None, + linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", + ""), + headerrow=_latex_row, + datarow=_latex_row, + padding=1, with_header_hide=None), + "textile": + TableFormat( + lineabove=None, linebelowheader=None, + linebetweenrows=None, linebelow=None, + headerrow=DataRow("|_. ", "|_.", "|"), + datarow=_textile_row_with_attrs, + padding=1, with_header_hide=None)} + + +tabulate_formats = list(sorted(_table_formats.keys())) + + +# ANSI color codes +_invisible_codes = re.compile(r"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") +_invisible_codes_bytes = re.compile(b"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") + + +def _isconvertible(conv, string): + try: + n = conv(string) + return True + except (ValueError, TypeError): + return False + + +def _isnumber(string): + """ + >>> _isnumber("123.45") + True + >>> _isnumber("123") + True + >>> _isnumber("spam") + False + """ + return _isconvertible(float, string) + + +def _isint(string, inttype=int): + """ + >>> _isint("123") + True + >>> _isint("123.45") + False + """ + return type(string) is inttype or\ + (isinstance(string, _binary_type) or isinstance(string, _text_type))\ + and\ + _isconvertible(inttype, string) + + +def _isbool(string): + """ + >>> _isbool(True) + True + >>> _isbool("False") + True + >>> _isbool(1) + False + """ + return type(string) is _bool_type or\ + (isinstance(string, (_binary_type, _text_type)) and + string in ("True", "False")) + + +def _type(string, has_invisible=True, numparse=True): + """The least generic type (type(None), int, float, str, unicode). + + >>> _type(None) is type(None) + True + >>> _type("foo") is type("") + True + >>> _type("1") is type(1) + True + >>> _type('\x1b[31m42\x1b[0m') is type(42) + True + >>> _type('\x1b[31m42\x1b[0m') is type(42) + True + + """ + + if has_invisible and \ + (isinstance(string, _text_type) or isinstance(string, _binary_type)): + string = _strip_invisible(string) + + if string is None: + return _none_type + elif hasattr(string, "isoformat"): # datetime.datetime, date, and time + return _text_type + elif _isbool(string): + return _bool_type + elif _isint(string) and numparse: + return int + elif _isint(string, _long_type) and numparse: + return int + elif _isnumber(string) and numparse: + return float + elif isinstance(string, _binary_type): + return _binary_type + else: + return _text_type + + +def _afterpoint(string): + """Symbols after a decimal point, -1 if the string lacks the decimal point. + + >>> _afterpoint("123.45") + 2 + >>> _afterpoint("1001") + -1 + >>> _afterpoint("eggs") + -1 + >>> _afterpoint("123e45") + 2 + + """ + if _isnumber(string): + if _isint(string): + return -1 + else: + pos = string.rfind(".") + pos = string.lower().rfind("e") if pos < 0 else pos + if pos >= 0: + return len(string) - pos - 1 + else: + return -1 # no point + else: + return -1 # not a number + + +def _padleft(width, s): + """Flush right. + + >>> _padleft(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430' + True + + """ + fmt = "{0:>%ds}" % width + return fmt.format(s) + + +def _padright(width, s): + """Flush left. + + >>> _padright(6, '\u044f\u0439\u0446\u0430') == '\u044f\u0439\u0446\u0430 ' + True + + """ + fmt = "{0:<%ds}" % width + return fmt.format(s) + + +def _padboth(width, s): + """Center string. + + >>> _padboth(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430 ' + True + + """ + fmt = "{0:^%ds}" % width + return fmt.format(s) + + +def _strip_invisible(s): + "Remove invisible ANSI color codes." + if isinstance(s, _text_type): + return re.sub(_invisible_codes, "", s) + else: # a bytestring + return re.sub(_invisible_codes_bytes, "", s) + + +def _visible_width(s): + """Visible width of a printed string. ANSI color codes are removed. + + >>> _visible_width('\x1b[31mhello\x1b[0m'), _visible_width("world") + (5, 5) + + """ + # optional wide-character support + if wcwidth is not None and WIDE_CHARS_MODE: + len_fn = wcwidth.wcswidth + else: + len_fn = len + if isinstance(s, _text_type) or isinstance(s, _binary_type): + return len_fn(_strip_invisible(s)) + else: + return len_fn(_text_type(s)) + + +def _align_column(strings, alignment, minwidth=0, has_invisible=True): + """[string] -> [padded_string] + + >>> list(map(str,_align_column( + ... ["12.345", "-1234.5", "1.23", "1234.5", "1e+234", "1.0e234"], + ... "decimal"))) + [' 12.345 ', '-1234.5 ', ' 1.23 ', ' 1234.5 ', ' 1e+234 ', ' 1.0e234'] + + >>> list(map(str,_align_column(['123.4', '56.7890'], None))) + ['123.4', '56.7890'] + + """ + if alignment == "right": + strings = [s.strip() for s in strings] + padfn = _padleft + elif alignment == "center": + strings = [s.strip() for s in strings] + padfn = _padboth + elif alignment == "decimal": + if has_invisible: + decimals = [_afterpoint(_strip_invisible(s)) for s in strings] + else: + decimals = [_afterpoint(s) for s in strings] + maxdecimals = max(decimals) + strings = [s + (maxdecimals - decs) * " " + for s, decs in zip(strings, decimals)] + padfn = _padleft + elif not alignment: + return strings + else: + strings = [s.strip() for s in strings] + padfn = _padright + + enable_widechars = wcwidth is not None and WIDE_CHARS_MODE + if has_invisible: + width_fn = _visible_width + elif enable_widechars: # optional wide-character support if available + width_fn = wcwidth.wcswidth + else: + width_fn = len + + s_lens = list(map(len, strings)) + s_widths = list(map(width_fn, strings)) + maxwidth = max(max(s_widths), minwidth) + if not enable_widechars and not has_invisible: + padded_strings = [padfn(maxwidth, s) for s in strings] + else: + # enable wide-character width corrections + visible_widths = [maxwidth - (w - l) for w, l in zip(s_widths, s_lens)] + # wcswidth and _visible_width don't count invisible characters; + # padfn doesn't need to apply another correction + padded_strings = [padfn(w, s) for s, w in zip(strings, visible_widths)] + return padded_strings + + +def _more_generic(type1, type2): + types = {_none_type: 0, _bool_type: 1, int: 2, float: 3, _binary_type: 4, + _text_type: 5} + invtypes = {5: _text_type, 4: _binary_type, 3: float, 2: int, + 1: _bool_type, 0: _none_type} + moregeneric = max(types.get(type1, 5), types.get(type2, 5)) + return invtypes[moregeneric] + + +def _column_type(strings, has_invisible=True, numparse=True): + """The least generic type all column values are convertible to. + + >>> _column_type([True, False]) is _bool_type + True + >>> _column_type(["1", "2"]) is _int_type + True + >>> _column_type(["1", "2.3"]) is _float_type + True + >>> _column_type(["1", "2.3", "four"]) is _text_type + True + >>> _column_type(["four", '\u043f\u044f\u0442\u044c']) is _text_type + True + >>> _column_type([None, "brux"]) is _text_type + True + >>> _column_type([1, 2, None]) is _int_type + True + >>> import datetime as dt + >>> _column_type([dt.datetime(1991,2,19), dt.time(17,35)]) is _text_type + True + + """ + types = [_type(s, has_invisible, numparse) for s in strings] + return reduce(_more_generic, types, _bool_type) + + +def _format(val, valtype, floatfmt, missingval="", has_invisible=True): + """Format a value accoding to its type. + + Unicode is supported: + + >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', + ... '\u0446\u0438\u0444\u0440\u0430'] + >>> tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] + >>> good_result = ('\\u0431\\u0443\\u043a\\u0432\\u0430 ' + ... '\\u0446\\u0438\\u0444\\u0440\\u0430\\n------- ' + ... '-------\\n\\u0430\\u0437 ' + ... '2\\n\\u0431\\u0443\\u043a\\u0438 4') + >>> tabulate(tbl, headers=hrow) == good_result + True + + """ + if val is None: + return missingval + + if valtype in [int, _text_type]: + return "{0}".format(val) + elif valtype is _binary_type: + try: + return _text_type(val, "ascii") + except TypeError: + return _text_type(val) + elif valtype is float: + is_a_colored_number = (has_invisible and + isinstance(val, (_text_type, _binary_type))) + if is_a_colored_number: + raw_val = _strip_invisible(val) + formatted_val = format(float(raw_val), floatfmt) + return val.replace(raw_val, formatted_val) + else: + return format(float(val), floatfmt) + else: + return "{0}".format(val) + + +def _align_header(header, alignment, width, visible_width): + "Pad string header to width chars given known visible_width of the header." + width += len(header) - visible_width + if alignment == "left": + return _padright(width, header) + elif alignment == "center": + return _padboth(width, header) + elif not alignment: + return "{0}".format(header) + else: + return _padleft(width, header) + + +def _prepend_row_index(rows, index): + """Add a left-most index column.""" + if index is None or index is False: + return rows + if len(index) != len(rows): + print('index=', index) + print('rows=', rows) + raise ValueError('index must be as long as the number of data rows') + rows = [[v]+list(row) for v, row in zip(index, rows)] + return rows + + +def _bool(val): + "A wrapper around standard bool() which doesn't throw on NumPy arrays" + try: + return bool(val) + except ValueError: # val is likely to be a numpy array with many elements + return False + + +def _normalize_tabular_data(tabular_data, headers, showindex="default"): + """Transform a supported data type to a list of lists, and a list of headers. + + Supported tabular data types: + + * list-of-lists or another iterable of iterables + + * list of named tuples (usually used with headers="keys") + + * list of dicts (usually used with headers="keys") + + * list of OrderedDicts (usually used with headers="keys") + + * 2D NumPy arrays + + * NumPy record arrays (usually used with headers="keys") + + * dict of iterables (usually used with headers="keys") + + * pandas.DataFrame (usually used with headers="keys") + + The first row can be used as headers if headers="firstrow", + column indices can be used as headers if headers="keys". + + If showindex="default", show row indices of the pandas.DataFrame. + If showindex="always", show row indices for all types of data. + If showindex="never", don't show row indices for all types of data. + If showindex is an iterable, show its values as row indices. + + """ + + try: + bool(headers) + is_headers2bool_broken = False + except ValueError: # numpy.ndarray, pandas.core.index.Index, ... + is_headers2bool_broken = True + headers = list(headers) + + index = None + if hasattr(tabular_data, "keys") and hasattr(tabular_data, "values"): + # dict-like and pandas.DataFrame? + if hasattr(tabular_data.values, "__call__"): + # likely a conventional dict + keys = tabular_data.keys() + # columns have to be transposed + rows = list(izip_longest(*tabular_data.values())) + elif hasattr(tabular_data, "index"): + # values is a property, has .index => it's likely a + # pandas.DataFrame (pandas 0.11.0) + keys = list(tabular_data) + if tabular_data.index.name is not None: + if isinstance(tabular_data.index.name, list): + keys[:0] = tabular_data.index.name + else: + keys[:0] = [tabular_data.index.name] + # values matrix doesn't need to be transposed + vals = tabular_data.values + # for DataFrames add an index per default + index = list(tabular_data.index) + rows = [list(row) for row in vals] + else: + raise ValueError( + "tabular data doesn't appear to be a dict or a DataFrame") + + if headers == "keys": + headers = list(map(_text_type, keys)) # headers should be strings + + else: # it's a usual an iterable of iterables, or a NumPy array + rows = list(tabular_data) + + if (headers == "keys" and not rows): + # an empty table (issue #81) + headers = [] + elif (headers == "keys" and + hasattr(tabular_data, "dtype") and + getattr(tabular_data.dtype, "names")): + # numpy record array + headers = tabular_data.dtype.names + elif (headers == "keys" + and len(rows) > 0 + and isinstance(rows[0], tuple) + and hasattr(rows[0], "_fields")): + # namedtuple + headers = list(map(_text_type, rows[0]._fields)) + elif (len(rows) > 0 + and isinstance(rows[0], dict)): + # dict or OrderedDict + uniq_keys = set() # implements hashed lookup + keys = [] # storage for set + if headers == "firstrow": + firstdict = rows[0] if len(rows) > 0 else {} + keys.extend(firstdict.keys()) + uniq_keys.update(keys) + rows = rows[1:] + for row in rows: + for k in row.keys(): + # Save unique items in input order + if k not in uniq_keys: + keys.append(k) + uniq_keys.add(k) + if headers == 'keys': + headers = keys + elif isinstance(headers, dict): + # a dict of headers for a list of dicts + headers = [headers.get(k, k) for k in keys] + headers = list(map(_text_type, headers)) + elif headers == "firstrow": + if len(rows) > 0: + headers = [firstdict.get(k, k) for k in keys] + headers = list(map(_text_type, headers)) + else: + headers = [] + elif headers: + raise ValueError( + 'headers for a list of dicts is not a dict or a keyword') + rows = [[row.get(k) for k in keys] for row in rows] + + elif (headers == "keys" + and hasattr(tabular_data, "description") + and hasattr(tabular_data, "fetchone") + and hasattr(tabular_data, "rowcount")): + # Python Database API cursor object (PEP 0249) + # print tabulate(cursor, headers='keys') + headers = [column[0] for column in tabular_data.description] + + elif headers == "keys" and len(rows) > 0: + # keys are column indices + headers = list(map(_text_type, range(len(rows[0])))) + + # take headers from the first row if necessary + if headers == "firstrow" and len(rows) > 0: + if index is not None: + headers = [index[0]] + list(rows[0]) + index = index[1:] + else: + headers = rows[0] + headers = list(map(_text_type, headers)) # headers should be strings + rows = rows[1:] + + headers = list(map(_text_type, headers)) + rows = list(map(list, rows)) + + # add or remove an index column + showindex_is_a_str = type(showindex) in [_text_type, _binary_type] + if showindex == "default" and index is not None: + rows = _prepend_row_index(rows, index) + elif isinstance(showindex, Iterable) and not showindex_is_a_str: + rows = _prepend_row_index(rows, list(showindex)) + elif (showindex == "always" or + (_bool(showindex) and not showindex_is_a_str)): + if index is None: + index = list(range(len(rows))) + rows = _prepend_row_index(rows, index) + elif (showindex == "never" or + (not _bool(showindex) and not showindex_is_a_str)): + pass + + # pad with empty headers for initial columns if necessary + if headers and len(rows) > 0: + nhs = len(headers) + ncols = len(rows[0]) + if nhs < ncols: + headers = [""]*(ncols - nhs) + headers + + return rows, headers + + +def tabulate(tabular_data, headers=(), tablefmt="simple", + floatfmt=_DEFAULT_FLOATFMT, numalign="decimal", stralign="left", + missingval=_DEFAULT_MISSINGVAL, showindex="default", + disable_numparse=False): + """Format a fixed width table for pretty printing. + + >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])) + --- --------- + 1 2.34 + -56 8.999 + 2 10001 + --- --------- + + The first required argument (`tabular_data`) can be a + list-of-lists (or another iterable of iterables), a list of named + tuples, a dictionary of iterables, an iterable of dictionaries, + a two-dimensional NumPy array, NumPy record array, or a Pandas' + dataframe. + + + Table headers + ------------- + + To print nice column headers, supply the second argument (`headers`): + + - `headers` can be an explicit list of column headers + - if `headers="firstrow"`, then the first row of data is used + - if `headers="keys"`, then dictionary keys or column indices are used + + Otherwise a headerless table is produced. + + If the number of headers is less than the number of columns, they + are supposed to be names of the last columns. This is consistent + with the plain-text format of R and Pandas' dataframes. + + >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], + ... headers="firstrow")) + sex age + ----- ----- ----- + Alice F 24 + Bob M 19 + + By default, pandas.DataFrame data have an additional column called + row index. To add a similar column to all other types of data, + use `showindex="always"` or `showindex=True`. To suppress row indices + for all types of data, pass `showindex="never" or `showindex=False`. + To add a custom row index column, pass `showindex=some_iterable`. + + >>> print(tabulate([["F",24],["M",19]], showindex="always")) + - - -- + 0 F 24 + 1 M 19 + - - -- + + + Column alignment + ---------------- + + `tabulate` tries to detect column types automatically, and aligns + the values properly. By default it aligns decimal points of the + numbers (or flushes integer numbers to the right), and flushes + everything else to the left. Possible column alignments + (`numalign`, `stralign`) are: "right", "center", "left", "decimal" + (only for `numalign`), and None (to disable alignment). + + + Table formats + ------------- + + `floatfmt` is a format specification used for columns which + contain numeric data with a decimal point. This can also be + a list or tuple of format strings, one per column. + + `None` values are replaced with a `missingval` string (like + `floatfmt`, this can also be a list of values for different + columns): + + >>> print(tabulate([["spam", 1, None], + ... ["eggs", 42, 3.14], + ... ["other", None, 2.7]], missingval="?")) + ----- -- ---- + spam 1 ? + eggs 42 3.14 + other ? 2.7 + ----- -- ---- + + Various plain-text table formats (`tablefmt`) are supported: + 'plain', 'simple', 'grid', 'pipe', 'orgtbl', 'rst', 'mediawiki', + 'latex', 'latex_raw' and 'latex_booktabs'. Variable `tabulate_formats` + contains the list of currently supported formats. + + "plain" format doesn't use any pseudographics to draw tables, + it separates columns with a double space: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "plain")) + strings numbers + spam 41.9999 + eggs 451 + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="plain")) + spam 41.9999 + eggs 451 + + "simple" format is like Pandoc simple_tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "simple")) + strings numbers + --------- --------- + spam 41.9999 + eggs 451 + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="simple")) + ---- -------- + spam 41.9999 + eggs 451 + ---- -------- + + "grid" is similar to tables produced by Emacs table.el package or + Pandoc grid_tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "grid")) + +-----------+-----------+ + | strings | numbers | + +===========+===========+ + | spam | 41.9999 | + +-----------+-----------+ + | eggs | 451 | + +-----------+-----------+ + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="grid")) + +------+----------+ + | spam | 41.9999 | + +------+----------+ + | eggs | 451 | + +------+----------+ + + "fancy_grid" draws a grid using box-drawing characters: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "fancy_grid")) + ╒═══════════╤═══════════╕ + │ strings │ numbers │ + ╞═══════════╪═══════════╡ + │ spam │ 41.9999 │ + ├───────────┼───────────┤ + │ eggs │ 451 │ + ╘═══════════╧═══════════╛ + + "pipe" is like tables in PHP Markdown Extra extension or Pandoc + pipe_tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "pipe")) + | strings | numbers | + |:----------|----------:| + | spam | 41.9999 | + | eggs | 451 | + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="pipe")) + |:-----|---------:| + | spam | 41.9999 | + | eggs | 451 | + + "orgtbl" is like tables in Emacs org-mode and orgtbl-mode. They + are slightly different from "pipe" format by not using colons to + define column alignment, and using a "+" sign to indicate line + intersections: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "orgtbl")) + | strings | numbers | + |-----------+-----------| + | spam | 41.9999 | + | eggs | 451 | + + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="orgtbl")) + | spam | 41.9999 | + | eggs | 451 | + + "rst" is like a simple table format from reStructuredText; please + note that reStructuredText accepts also "grid" tables: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... ["strings", "numbers"], "rst")) + ========= ========= + strings numbers + ========= ========= + spam 41.9999 + eggs 451 + ========= ========= + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")) + ==== ======== + spam 41.9999 + eggs 451 + ==== ======== + + "mediawiki" produces a table markup used in Wikipedia and on other + MediaWiki-based sites: + + >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], + ... ["eggs", "451.0"]], headers="firstrow", + ... tablefmt="mediawiki")) + {| class="wikitable" style="text-align: left;" + |+ + |- + ! strings !! align="right"| numbers + |- + | spam || align="right"| 41.9999 + |- + | eggs || align="right"| 451 + |} + + "html" produces HTML markup: + + >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], + ... ["eggs", "451.0"]], headers="firstrow", + ... tablefmt="html")) + + + + + + + + +
strings numbers
spam 41.9999
eggs 451
+ + "latex" produces a tabular environment of LaTeX document markup: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="latex")) + \\begin{tabular}{lr} + \\hline + spam & 41.9999 \\\\ + eggs & 451 \\\\ + \\hline + \\end{tabular} + + "latex_raw" is similar to "latex", but doesn't escape special characters, + such as backslash and underscore, so LaTeX commands may embedded into + cells' values: + + >>> print(tabulate([["spam$_9$", 41.9999], ["\\\\emph{eggs}", "451.0"]], + ... tablefmt="latex_raw")) + \\begin{tabular}{lr} + \\hline + spam$_9$ & 41.9999 \\\\ + \\emph{eggs} & 451 \\\\ + \\hline + \\end{tabular} + + "latex_booktabs" produces a tabular environment of LaTeX document markup + using the booktabs.sty package: + + >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], + ... tablefmt="latex_booktabs")) + \\begin{tabular}{lr} + \\toprule + spam & 41.9999 \\\\ + eggs & 451 \\\\ + \\bottomrule + \end{tabular} + + Number parsing + -------------- + By default, anything which can be parsed as a number is a number. + This ensures numbers represented as strings are aligned properly. + This can lead to weird results for particular strings such as + specific git SHAs e.g. "42992e1" will be parsed into the number + 429920 and aligned as such. + + To completely disable number parsing (and alignment), use + `disable_numparse=True`. For more fine grained control, a list column + indices is used to disable number parsing only on those columns + e.g. `disable_numparse=[0, 2]` would disable number parsing only on the + first and third columns. + """ + if tabular_data is None: + tabular_data = [] + list_of_lists, headers = _normalize_tabular_data( + tabular_data, headers, showindex=showindex) + + # empty values in the first column of RST tables should be escaped + # (issue #82). "" should be escaped as "\\ " or ".." + if tablefmt == 'rst': + list_of_lists, headers = _rst_escape_first_column(list_of_lists, + headers) + + # optimization: look for ANSI control codes once, + # enable smart width functions only if a control code is found + plain_text = '\n'.join(['\t'.join(map(_text_type, headers))] + + ['\t'.join(map(_text_type, row)) + for row in list_of_lists]) + + has_invisible = re.search(_invisible_codes, plain_text) + enable_widechars = wcwidth is not None and WIDE_CHARS_MODE + if has_invisible: + width_fn = _visible_width + elif enable_widechars: # optional wide-character support if available + width_fn = wcwidth.wcswidth + else: + width_fn = len + + # format rows and columns, convert numeric values to strings + cols = list(izip_longest(*list_of_lists)) + numparses = _expand_numparse(disable_numparse, len(cols)) + coltypes = [_column_type(col, numparse=np) for col, np in + zip(cols, numparses)] + if isinstance(floatfmt, basestring): # old version + # just duplicate the string to use in each column + float_formats = len(cols) * [floatfmt] + else: # if floatfmt is list, tuple etc we have one per column + float_formats = list(floatfmt) + if len(float_formats) < len(cols): + float_formats.extend((len(cols)-len(float_formats)) * + [_DEFAULT_FLOATFMT]) + if isinstance(missingval, basestring): + missing_vals = len(cols) * [missingval] + else: + missing_vals = list(missingval) + if len(missing_vals) < len(cols): + missing_vals.extend((len(cols)-len(missing_vals)) * + [_DEFAULT_MISSINGVAL]) + cols = [[_format(v, ct, fl_fmt, miss_v, has_invisible) for v in c] + for c, ct, fl_fmt, miss_v in zip(cols, coltypes, float_formats, + missing_vals)] + + # align columns + aligns = [numalign if ct in [int, float] else stralign for ct in coltypes] + minwidths = [width_fn(h) + MIN_PADDING + for h in headers] if headers else [0]*len(cols) + cols = [_align_column(c, a, minw, has_invisible) + for c, a, minw in zip(cols, aligns, minwidths)] + + if headers: + # align headers and add headers + t_cols = cols or [['']] * len(headers) + t_aligns = aligns or [stralign] * len(headers) + minwidths = [max(minw, width_fn(c[0])) + for minw, c in zip(minwidths, t_cols)] + headers = [_align_header(h, a, minw, width_fn(h)) + for h, a, minw in zip(headers, t_aligns, minwidths)] + rows = list(zip(*cols)) + else: + minwidths = [width_fn(c[0]) for c in cols] + rows = list(zip(*cols)) + + if not isinstance(tablefmt, TableFormat): + tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) + + return _format_table(tablefmt, headers, rows, minwidths, aligns) + + +def _expand_numparse(disable_numparse, column_count): + """ + Return a list of bools of length `column_count` which indicates whether + number parsing should be used on each column. + If `disable_numparse` is a list of indices, each of those indices are + False, and everything else is True. + If `disable_numparse` is a bool, then the returned list is all the same. + """ + if isinstance(disable_numparse, Iterable): + numparses = [True] * column_count + for index in disable_numparse: + numparses[index] = False + return numparses + else: + return [not disable_numparse] * column_count + + +def _build_simple_row(padded_cells, rowfmt): + "Format row according to DataRow format without padding." + begin, sep, end = rowfmt + return (begin + sep.join(padded_cells) + end).rstrip() + + +def _build_row(padded_cells, colwidths, colaligns, rowfmt): + "Return a string which represents a row of data cells." + if not rowfmt: + return None + if hasattr(rowfmt, "__call__"): + return rowfmt(padded_cells, colwidths, colaligns) + else: + return _build_simple_row(padded_cells, rowfmt) + + +def _build_line(colwidths, colaligns, linefmt): + "Return a string which represents a horizontal line." + if not linefmt: + return None + if hasattr(linefmt, "__call__"): + return linefmt(colwidths, colaligns) + else: + begin, fill, sep, end = linefmt + cells = [fill*w for w in colwidths] + return _build_simple_row(cells, (begin, sep, end)) + + +def _pad_row(cells, padding): + if cells: + pad = " "*padding + padded_cells = [pad + cell + pad for cell in cells] + return padded_cells + else: + return cells + + +def _format_table(fmt, headers, rows, colwidths, colaligns): + """Produce a plain-text representation of the table.""" + lines = [] + hidden = fmt.with_header_hide if (headers and fmt.with_header_hide) else [] + pad = fmt.padding + headerrow = fmt.headerrow + + padded_widths = [(w + 2*pad) for w in colwidths] + padded_headers = _pad_row(headers, pad) + padded_rows = [_pad_row(row, pad) for row in rows] + + if fmt.lineabove and "lineabove" not in hidden: + lines.append(_build_line(padded_widths, colaligns, fmt.lineabove)) + + if padded_headers: + lines.append(_build_row(padded_headers, padded_widths, colaligns, + headerrow)) + if fmt.linebelowheader and "linebelowheader" not in hidden: + lines.append(_build_line(padded_widths, colaligns, + fmt.linebelowheader)) + + if padded_rows and fmt.linebetweenrows and "linebetweenrows" not in hidden: + # initial rows with a line below + for row in padded_rows[:-1]: + lines.append(_build_row(row, padded_widths, colaligns, + fmt.datarow)) + lines.append(_build_line(padded_widths, colaligns, + fmt.linebetweenrows)) + # the last row without a line below + lines.append(_build_row(padded_rows[-1], padded_widths, colaligns, + fmt.datarow)) + else: + for row in padded_rows: + lines.append(_build_row(row, padded_widths, colaligns, + fmt.datarow)) + + if fmt.linebelow and "linebelow" not in hidden: + lines.append(_build_line(padded_widths, colaligns, fmt.linebelow)) + + if headers or rows: + return "\n".join(lines) + else: # a completely empty table + return "" + + +def _main(): + """\ + Usage: tabulate [options] [FILE ...] + + Pretty-print tabular data. + See also https://bitbucket.org/astanin/python-tabulate + + FILE a filename of the file with tabular data; + if "-" or missing, read data from stdin. + + Options: + + -h, --help show this message + -1, --header use the first row of data as a table header + -o FILE, --output FILE print table to FILE (default: stdout) + -s REGEXP, --sep REGEXP use a custom column separator (default: whitespace) + -F FPFMT, --float FPFMT floating point number format (default: g) + -f FMT, --format FMT set output table format; supported formats: + plain, simple, grid, fancy_grid, pipe, orgtbl, + rst, mediawiki, html, latex, latex_raw, + latex_booktabs, tsv + (default: simple) + """ + import getopt + import sys + import textwrap + usage = textwrap.dedent(_main.__doc__) + try: + opts, args = getopt.getopt( + sys.argv[1:], "h1o:s:F:f:", + ["help", "header", "output", "sep=", "float=", "format="]) + except getopt.GetoptError as e: + print(e) + print(usage) + sys.exit(2) + headers = [] + floatfmt = _DEFAULT_FLOATFMT + tablefmt = "simple" + sep = r"\s+" + outfile = "-" + for opt, value in opts: + if opt in ["-1", "--header"]: + headers = "firstrow" + elif opt in ["-o", "--output"]: + outfile = value + elif opt in ["-F", "--float"]: + floatfmt = value + elif opt in ["-f", "--format"]: + if value not in tabulate_formats: + print("%s is not a supported table format" % value) + print(usage) + sys.exit(3) + tablefmt = value + elif opt in ["-s", "--sep"]: + sep = value + elif opt in ["-h", "--help"]: + print(usage) + sys.exit(0) + files = [sys.stdin] if not args else args + with (sys.stdout if outfile == "-" else open(outfile, "w")) as out: + for f in files: + if f == "-": + f = sys.stdin + if _is_file(f): + _pprint_file(f, headers=headers, tablefmt=tablefmt, + sep=sep, floatfmt=floatfmt, file=out) + else: + with open(f) as fobj: + _pprint_file(fobj, headers=headers, tablefmt=tablefmt, + sep=sep, floatfmt=floatfmt, file=out) + + +def _pprint_file(fobject, headers, tablefmt, sep, floatfmt, file): + rows = fobject.readlines() + table = [re.split(sep, r.rstrip()) for r in rows if r.strip()] + print(tabulate(table, headers, tablefmt, floatfmt=floatfmt), file=file) + + +if __name__ == "__main__": + _main() diff --git a/setup.py b/setup.py index 21e555086..417929c63 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', 'pycryptodome >= 3', - 'tabulate >= 0.7.6', 'terminaltables >= 3.0.0', ] From 4c9060569d23fe177324387733a2a1c17a9532e2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:13:30 -0500 Subject: [PATCH 0370/1025] Add preserve whitespace option to tabulate. --- mycli/output_formatter/tabulate_adapter.py | 2 ++ mycli/packages/tabulate.py | 10 +++++++--- tests/test_tabulate.py | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 tests/test_tabulate.py diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 14e6c770f..c9b1acd2a 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,6 +1,8 @@ from mycli.packages import tabulate from .preprocessors import bytes_to_string, align_decimals, quote_whitespaces +tabulate.PRESERVE_WHITESPACE = True + supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', 'html', 'latex', 'latex_booktabs', 'textile') diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index a25032995..292f40bb0 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -53,6 +53,7 @@ def _is_file(f): # minimum extra space in headers MIN_PADDING = 2 +PRESERVE_WHITESPACE = False _DEFAULT_FLOATFMT = "g" _DEFAULT_MISSINGVAL = "" @@ -563,10 +564,12 @@ def _align_column(strings, alignment, minwidth=0, has_invisible=True): """ if alignment == "right": - strings = [s.strip() for s in strings] + if not PRESERVE_WHITESPACE: + strings = [s.strip() for s in strings] padfn = _padleft elif alignment == "center": - strings = [s.strip() for s in strings] + if not PRESERVE_WHITESPACE: + strings = [s.strip() for s in strings] padfn = _padboth elif alignment == "decimal": if has_invisible: @@ -580,7 +583,8 @@ def _align_column(strings, alignment, minwidth=0, has_invisible=True): elif not alignment: return strings else: - strings = [s.strip() for s in strings] + if not PRESERVE_WHITESPACE: + strings = [s.strip() for s in strings] padfn = _padright enable_widechars = wcwidth is not None and WIDE_CHARS_MODE diff --git a/tests/test_tabulate.py b/tests/test_tabulate.py new file mode 100644 index 000000000..ae7c25ce1 --- /dev/null +++ b/tests/test_tabulate.py @@ -0,0 +1,17 @@ +from textwrap import dedent + +from mycli.packages import tabulate + +tabulate.PRESERVE_WHITESPACE = True + + +def test_dont_strip_leading_whitespace(): + data = [[' abc']] + headers = ['xyz'] + tbl = tabulate.tabulate(data, headers, tablefmt='psql') + assert tbl == dedent(''' + +---------+ + | xyz | + |---------| + | abc | + +---------+ ''').strip() From 694919eae6b1a4489e61c8f8adfc390b05ec6f55 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:17:12 -0500 Subject: [PATCH 0371/1025] Remove extra html format. --- mycli/output_formatter/tabulate_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index c9b1acd2a..fc6a7d8d5 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -5,7 +5,7 @@ supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', - 'html', 'latex', 'latex_booktabs', 'textile') + 'latex', 'latex_booktabs', 'textile') preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) From 85d0e026278b7f0f2b044b483412832beae205d8 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:29:49 -0500 Subject: [PATCH 0372/1025] Do not align columns for markup tables. --- mycli/output_formatter/tabulate_adapter.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index fc6a7d8d5..7b4dfac3f 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -3,14 +3,20 @@ tabulate.PRESERVE_WHITESPACE = True -supported_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', 'orgtbl', - 'jira', 'psql', 'rst', 'mediawiki', 'moinmoin', 'html', - 'latex', 'latex_booktabs', 'textile') +supported_markup_formats = ('mediawiki', 'html', 'latex', 'latex_booktabs', + 'textile', 'moinmoin', 'jira') +supported_table_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', + 'orgtbl', 'psql', 'rst') +supported_formats = supported_markup_formats + supported_table_formats preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" - return tabulate.tabulate(data, headers, tablefmt=table_format, - missingval=missing_value, disable_numparse=True) + kwargs = {'tablefmt': table_format, 'missingval': missing_value, + 'disable_numparse': True} + if table_format in supported_markup_formats: + kwargs.update(numalign=None, stralign=None) + + return tabulate.tabulate(data, headers, **kwargs) From ae8311c5424a9dc4632364d671e8615d38d44d67 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:30:51 -0500 Subject: [PATCH 0373/1025] Do not quote whitespaces for tabulate formats. --- mycli/output_formatter/tabulate_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 7b4dfac3f..26ef40aa2 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -9,7 +9,7 @@ 'orgtbl', 'psql', 'rst') supported_formats = supported_markup_formats + supported_table_formats -preprocessors = (bytes_to_string, align_decimals, quote_whitespaces) +preprocessors = (bytes_to_string, align_decimals) def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): From 359044c8e87ed0e8394eac2dacb3e0d53cc498c5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 15:47:42 -0500 Subject: [PATCH 0374/1025] Pep8radius changes. --- mycli/packages/tabulate.py | 41 +++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index 292f40bb0..ab3796f66 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -180,9 +180,9 @@ def _moin_row_with_attrs(celltag, cell_values, colwidths, colaligns, "decimal": ''} values_with_attrs = ["{0}{1} {2} ".format(celltag, alignment.get(a, ''), - header+c+header) + header + c + header) for c, a in zip(cell_values, colaligns)] - return "".join(values_with_attrs)+"||" + return "".join(values_with_attrs) + "||" def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): @@ -684,7 +684,8 @@ def _format(val, valtype, floatfmt, missingval="", has_invisible=True): def _align_header(header, alignment, width, visible_width): - "Pad string header to width chars given known visible_width of the header." + """Pad string header to width chars given known visible_width of the + header.""" width += len(header) - visible_width if alignment == "left": return _padright(width, header) @@ -704,12 +705,13 @@ def _prepend_row_index(rows, index): print('index=', index) print('rows=', rows) raise ValueError('index must be as long as the number of data rows') - rows = [[v]+list(row) for v, row in zip(index, rows)] + rows = [[v] + list(row) for v, row in zip(index, rows)] return rows def _bool(val): - "A wrapper around standard bool() which doesn't throw on NumPy arrays" + """A wrapper around standard bool() which doesn't throw on NumPy + arrays.""" try: return bool(val) except ValueError: # val is likely to be a numpy array with many elements @@ -717,7 +719,8 @@ def _bool(val): def _normalize_tabular_data(tabular_data, headers, showindex="default"): - """Transform a supported data type to a list of lists, and a list of headers. + """Transform a supported data type to a list of lists, and a list of + headers. Supported tabular data types: @@ -878,7 +881,7 @@ def _normalize_tabular_data(tabular_data, headers, showindex="default"): nhs = len(headers) ncols = len(rows[0]) if nhs < ncols: - headers = [""]*(ncols - nhs) + headers + headers = [""] * (ncols - nhs) + headers return rows, headers @@ -1169,11 +1172,12 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", indices is used to disable number parsing only on those columns e.g. `disable_numparse=[0, 2]` would disable number parsing only on the first and third columns. + """ if tabular_data is None: tabular_data = [] list_of_lists, headers = _normalize_tabular_data( - tabular_data, headers, showindex=showindex) + tabular_data, headers, showindex=showindex) # empty values in the first column of RST tables should be escaped # (issue #82). "" should be escaped as "\\ " or ".." @@ -1207,14 +1211,14 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", else: # if floatfmt is list, tuple etc we have one per column float_formats = list(floatfmt) if len(float_formats) < len(cols): - float_formats.extend((len(cols)-len(float_formats)) * + float_formats.extend((len(cols) - len(float_formats)) * [_DEFAULT_FLOATFMT]) if isinstance(missingval, basestring): missing_vals = len(cols) * [missingval] else: missing_vals = list(missingval) if len(missing_vals) < len(cols): - missing_vals.extend((len(cols)-len(missing_vals)) * + missing_vals.extend((len(cols) - len(missing_vals)) * [_DEFAULT_MISSINGVAL]) cols = [[_format(v, ct, fl_fmt, miss_v, has_invisible) for v in c] for c, ct, fl_fmt, miss_v in zip(cols, coltypes, float_formats, @@ -1223,7 +1227,7 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", # align columns aligns = [numalign if ct in [int, float] else stralign for ct in coltypes] minwidths = [width_fn(h) + MIN_PADDING - for h in headers] if headers else [0]*len(cols) + for h in headers] if headers else [0] * len(cols) cols = [_align_column(c, a, minw, has_invisible) for c, a, minw in zip(cols, aligns, minwidths)] @@ -1247,12 +1251,13 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", def _expand_numparse(disable_numparse, column_count): - """ - Return a list of bools of length `column_count` which indicates whether + """Return a list of bools of length `column_count` which indicates whether number parsing should be used on each column. - If `disable_numparse` is a list of indices, each of those indices are - False, and everything else is True. - If `disable_numparse` is a bool, then the returned list is all the same. + + If `disable_numparse` is a list of indices, each of those indices + are False, and everything else is True. If `disable_numparse` is a + bool, then the returned list is all the same. + """ if isinstance(disable_numparse, Iterable): numparses = [True] * column_count @@ -1346,8 +1351,7 @@ def _format_table(fmt, headers, rows, colwidths, colaligns): def _main(): - """\ - Usage: tabulate [options] [FILE ...] + """\ Usage: tabulate [options] [FILE ...] Pretty-print tabular data. See also https://bitbucket.org/astanin/python-tabulate @@ -1367,6 +1371,7 @@ def _main(): rst, mediawiki, html, latex, latex_raw, latex_booktabs, tsv (default: simple) + """ import getopt import sys From 13991b193a2d764a7baa16937193f37a5bdc3c6e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 16:06:23 -0500 Subject: [PATCH 0375/1025] Add output format changes to changelog. --- changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index eb36b6bab..5bda9ae72 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,8 @@ Features: --------- * Add ability to specify alternative myclirc file. (Thanks: [Dick Marinus]). +* Add new display formats for pretty printing query results. (Thanks: [Amjith + Ramanujam], [Dick Marinus], [Thomas Roten]). Bug Fixes: ---------- From 33e5c26eba1f3261b66d603fcbcb66b8de8f76e2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 16:11:03 -0500 Subject: [PATCH 0376/1025] Add additional table formats to myclirc. --- mycli/myclirc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mycli/myclirc b/mycli/myclirc index 2f3599273..febd7e591 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -30,7 +30,9 @@ log_level = INFO # Timing of sql statments and table rendering. timing = True -# Table format. Possible values: ascii, single, double, or github. +# Table format. Possible values: ascii, single, double, github, +# psql, plain, simple, grid, fancy_grid, pipe, orgtbl, rst, mediawiki, html, +# latex, latex_booktabs, textile, moinmoin, jira, expanded, tsv, csv. # Recommended: ascii table_format = ascii From f06451cb4c12baf30f32d8fd0be3da0042d2f23c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 16:16:13 -0500 Subject: [PATCH 0377/1025] Don't import quote_whitespaces since it's not used. --- mycli/output_formatter/tabulate_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index 26ef40aa2..ee63b5b1d 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -1,5 +1,5 @@ from mycli.packages import tabulate -from .preprocessors import bytes_to_string, align_decimals, quote_whitespaces +from .preprocessors import bytes_to_string, align_decimals tabulate.PRESERVE_WHITESPACE = True From 8f70b12ef9db79e2610eba5c8c45a0b92d07fc93 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 21:11:04 -0500 Subject: [PATCH 0378/1025] Fix outdated docstring. --- mycli/output_formatter/output_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 759fb879a..9f9b6a8e5 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -27,7 +27,7 @@ class OutputFormatter(object): _output_formats = {} def __init__(self, format_name=None): - """Register the supported output formats.""" + """Set the default *format_name*.""" self._format_name = format_name def set_format_name(self, format_name): From fd861d41e041b08da83dface8b6d8a0843f3b2fa Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 21:30:21 -0500 Subject: [PATCH 0379/1025] Make SQLCompleter options pass through refresher. --- mycli/completion_refresher.py | 14 ++++++++------ mycli/main.py | 11 +++++++---- tests/test_completion_refresher.py | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index 33afa009c..d24e03a80 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -13,7 +13,7 @@ def __init__(self): self._completer_thread = None self._restart_refresh = threading.Event() - def refresh(self, executor, callbacks): + def refresh(self, executor, callbacks, completer_options={}): """ Creates a SQLCompleter object and populates it with the relevant completion suggestions in a background thread. @@ -23,14 +23,16 @@ def refresh(self, executor, callbacks): callbacks - A function or a list of functions to call after the thread has completed the refresh. The newly created completion object will be passed in as an argument to each callback. + completer_options - dict of options to pass to SQLCompleter. """ if self.is_refreshing(): self._restart_refresh.set() return [(None, None, None, 'Auto-completion refresh restarted.')] else: - self._completer_thread = threading.Thread(target=self._bg_refresh, - args=(executor, callbacks), - name='completion_refresh') + self._completer_thread = threading.Thread( + target=self._bg_refresh, + args=(executor, callbacks, completer_options), + name='completion_refresh') self._completer_thread.setDaemon(True) self._completer_thread.start() return [(None, None, None, @@ -39,8 +41,8 @@ def refresh(self, executor, callbacks): def is_refreshing(self): return self._completer_thread and self._completer_thread.is_alive() - def _bg_refresh(self, sqlexecute, callbacks): - completer = SQLCompleter(smart_completion=True) + def _bg_refresh(self, sqlexecute, callbacks, completer_options): + completer = SQLCompleter(**completer_options) # Create a new pgexecute method to popoulate the completions. e = sqlexecute diff --git a/mycli/main.py b/mycli/main.py index 64c587057..2a9b04364 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -146,8 +146,9 @@ def __init__(self, sqlexecute=None, prompt=None, # Initialize completer. self.smart_completion = c['main'].as_bool('smart_completion') - self.completer = SQLCompleter(self.smart_completion, - supported_formats=self.formatter.supported_formats) + self.completer = SQLCompleter( + self.smart_completion, + supported_formats=self.formatter.supported_formats()) self._completer_lock = threading.Lock() # Register custom special commands. @@ -665,8 +666,10 @@ def refresh_completions(self, reset=False): if reset: with self._completer_lock: self.completer.reset_completions() - self.completion_refresher.refresh(self.sqlexecute, - self._on_completions_refreshed) + self.completion_refresher.refresh( + self.sqlexecute, self._on_completions_refreshed, + {'smart_completion': self.smart_completion, + 'supported_formats': self.formatter.supported_formats()}) return [(None, None, None, 'Auto-completion refresh started in the background.')] diff --git a/tests/test_completion_refresher.py b/tests/test_completion_refresher.py index a50ad7492..8851eae65 100644 --- a/tests/test_completion_refresher.py +++ b/tests/test_completion_refresher.py @@ -37,7 +37,7 @@ def test_refresh_called_once(refresher): assert len(actual) == 1 assert len(actual[0]) == 4 assert actual[0][3] == 'Auto-completion refresh started in the background.' - bg_refresh.assert_called_with(sqlexecute, callbacks) + bg_refresh.assert_called_with(sqlexecute, callbacks, {}) def test_refresh_called_twice(refresher): From 81c54962c530e1656e0d6575829a67d913daa958 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 12 Apr 2017 21:33:02 -0500 Subject: [PATCH 0380/1025] Pep8radius fixes. --- mycli/completion_refresher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/completion_refresher.py b/mycli/completion_refresher.py index d24e03a80..2bbe32d0e 100644 --- a/mycli/completion_refresher.py +++ b/mycli/completion_refresher.py @@ -14,8 +14,7 @@ def __init__(self): self._restart_refresh = threading.Event() def refresh(self, executor, callbacks, completer_options={}): - """ - Creates a SQLCompleter object and populates it with the relevant + """Creates a SQLCompleter object and populates it with the relevant completion suggestions in a background thread. executor - SQLExecute object, used to extract the credentials to connect @@ -24,6 +23,7 @@ def refresh(self, executor, callbacks, completer_options={}): has completed the refresh. The newly created completion object will be passed in as an argument to each callback. completer_options - dict of options to pass to SQLCompleter. + """ if self.is_refreshing(): self._restart_refresh.set() From 9a23584a03a4e29eaff755768e04cc2589ac90ad Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 08:47:16 -0500 Subject: [PATCH 0381/1025] Fix fancy_grid table format. --- mycli/packages/tabulate.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py index ab3796f66..1e67cea7d 100644 --- a/mycli/packages/tabulate.py +++ b/mycli/packages/tabulate.py @@ -252,12 +252,12 @@ def escape_empty(val): padding=1, with_header_hide=None), "fancy_grid": TableFormat( - lineabove=Line("â•’", "═", "╤", "â••"), - linebelowheader=Line("â•ž", "═", "╪", "â•¡"), - linebetweenrows=Line("├", "─", "┼", "┤"), - linebelow=Line("╘", "═", "╧", "â•›"), - headerrow=DataRow("│", "│", "│"), - datarow=DataRow("│", "│", "│"), + lineabove=Line("╒", "═", "╤", "╕"), + linebelowheader=Line("╞", "═", "╪", "╡"), + linebetweenrows=Line("├", "─", "┼", "┤"), + linebelow=Line("╘", "═", "╧", "╛"), + headerrow=DataRow("│", "│", "│"), + datarow=DataRow("│", "│", "│"), padding=1, with_header_hide=None), "pipe": TableFormat( @@ -1032,13 +1032,13 @@ def tabulate(tabular_data, headers=(), tablefmt="simple", >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], ... ["strings", "numbers"], "fancy_grid")) - ╒═══════════╤═══════════╕ - │ strings │ numbers │ - ╞═══════════╪═══════════╡ - │ spam │ 41.9999 │ - ├───────────┼───────────┤ - │ eggs │ 451 │ - ╘═══════════╧═══════════╛ + ╒═══════════╤═══════════╕ + │ strings │ numbers │ + ╞═══════════╪═══════════╡ + │ spam │ 41.9999 │ + ├───────────┼───────────┤ + │ eggs │ 451 │ + ╘═══════════╧═══════════╛ "pipe" is like tables in PHP Markdown Extra extension or Pandoc pipe_tables: From 4b0de47ebe3e9c91cf6136d8ee28fb59fbf92ee2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 13:40:53 -0500 Subject: [PATCH 0382/1025] Remove single terminaltable format since it's causing problems. --- mycli/myclirc | 2 +- mycli/output_formatter/terminaltables_adapter.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mycli/myclirc b/mycli/myclirc index febd7e591..01a114265 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -30,7 +30,7 @@ log_level = INFO # Timing of sql statments and table rendering. timing = True -# Table format. Possible values: ascii, single, double, github, +# Table format. Possible values: ascii, double, github, # psql, plain, simple, grid, fancy_grid, pipe, orgtbl, rst, mediawiki, html, # latex, latex_booktabs, textile, moinmoin, jira, expanded, tsv, csv. # Recommended: ascii diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py index ac580517f..0c702abe0 100644 --- a/mycli/output_formatter/terminaltables_adapter.py +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -3,7 +3,7 @@ from .preprocessors import (bytes_to_string, align_decimals, override_missing_value) -supported_formats = ('ascii', 'single', 'double', 'github') +supported_formats = ('ascii', 'double', 'github') preprocessors = (bytes_to_string, override_missing_value, align_decimals) @@ -12,7 +12,6 @@ def terminaltables_adapter(data, headers, table_format=None, **_): table_format_handler = { 'ascii': terminaltables.AsciiTable, - 'single': terminaltables.SingleTable, 'double': terminaltables.DoubleTable, 'github': terminaltables.GithubFlavoredMarkdownTable, } From cc76dee6198b759fd0219bff9cafc383a450456b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 14:27:19 -0500 Subject: [PATCH 0383/1025] Use constant for missing value default. --- mycli/output_formatter/output_formatter.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 9f9b6a8e5..3e53d826d 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -16,6 +16,8 @@ terminaltables_adapter, preprocessors as terminaltables_preprocessors, supported_formats as terminaltables_formats) +MISSING_VALUE = '' + OutputFormatHandler = namedtuple( 'OutputFormatHandler', 'format_name preprocessors formatter formatter_args') @@ -77,22 +79,22 @@ def format_output(self, data, headers, format_name=None, **kwargs): OutputFormatter.register_new_formatter('expanded', expanded_table, (override_missing_value, convert_to_string), - {'missing_value': ''}) + {'missing_value': MISSING_VALUE}) for delimiter_format in delimiter_formats: OutputFormatter.register_new_formatter(delimiter_format, delimiter_adapter, delimiter_preprocessors, {'table_format': delimiter_format, - 'missing_value': ''}) + 'missing_value': MISSING_VALUE}) for tabulate_format in tabulate_formats: OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, tabulate_preprocessors, {'table_format': tabulate_format, - 'missing_value': ''}) + 'missing_value': MISSING_VALUE}) for terminaltables_format in terminaltables_formats: OutputFormatter.register_new_formatter( terminaltables_format, terminaltables_adapter, terminaltables_preprocessors, - {'table_format': terminaltables_format, 'missing_value': ''}) + {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) From 39c4aec92645306a3c80ad11ec9213c918437284 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Thu, 13 Apr 2017 14:38:38 -0500 Subject: [PATCH 0384/1025] Make registering a format not require kwargs. --- mycli/output_formatter/output_formatter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index 3e53d826d..c87da5edd 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -49,8 +49,8 @@ def supported_formats(self): return tuple(self._output_formats.keys()) @classmethod - def register_new_formatter(cls, format_name, handler, preprocessors=None, - kwargs=None): + def register_new_formatter(cls, format_name, handler, preprocessors=(), + kwargs={}): """Register a new formatter to format the output.""" cls._output_formats[format_name] = OutputFormatHandler( format_name, preprocessors, handler, kwargs) From 21c600256de4c01fb4c4aa97b84ebd28f3f81c9f Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Fri, 14 Apr 2017 21:22:22 +0200 Subject: [PATCH 0385/1025] remove temporary hack --- changelog.md | 1 + mycli/packages/completion_engine.py | 43 +++++++++++++---------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/changelog.md b/changelog.md index fcf9c198d..8ac468380 100644 --- a/changelog.md +++ b/changelog.md @@ -23,6 +23,7 @@ Internal Changes: Roten]). * Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). * Run pep8 checks in travis (Thanks: [Irina Truong]). +* Remove temporary hack for sqlparse (Thanks: [Dick Marinus]). 1.9.0: ====== diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index 6e2165dca..bd6a7b89c 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -28,32 +28,27 @@ def suggest_type(full_text, text_before_cursor): identifier = None - # This is a temporary hack; the exception handling - # here should be removed once sqlparse has been fixed - try: - # If we've partially typed a word then word_before_cursor won't be an empty - # string. In that case we want to remove the partially typed string before - # sending it to the sqlparser. Otherwise the last token will always be the - # partially typed string which renders the smart completion useless because - # it will always return the list of keywords as completion. - if word_before_cursor: - if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': - parsed = sqlparse.parse(text_before_cursor) - else: - parsed = sqlparse.parse( - text_before_cursor[:-len(word_before_cursor)]) + # If we've partially typed a word then word_before_cursor won't be an empty + # string. In that case we want to remove the partially typed string before + # sending it to the sqlparser. Otherwise the last token will always be the + # partially typed string which renders the smart completion useless because + # it will always return the list of keywords as completion. + if word_before_cursor: + if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': + parsed = sqlparse.parse(text_before_cursor) + else: + parsed = sqlparse.parse( + text_before_cursor[:-len(word_before_cursor)]) - # word_before_cursor may include a schema qualification, like - # "schema_name.partial_name" or "schema_name.", so parse it - # separately - p = sqlparse.parse(word_before_cursor)[0] + # word_before_cursor may include a schema qualification, like + # "schema_name.partial_name" or "schema_name.", so parse it + # separately + p = sqlparse.parse(word_before_cursor)[0] - if p.tokens and isinstance(p.tokens[0], Identifier): - identifier = p.tokens[0] - else: - parsed = sqlparse.parse(text_before_cursor) - except (TypeError, AttributeError): - return [] + if p.tokens and isinstance(p.tokens[0], Identifier): + identifier = p.tokens[0] + else: + parsed = sqlparse.parse(text_before_cursor) if len(parsed) > 1: # Multiple statements being edited -- isolate the current one by From 5d5cf7a46a391a81bdd62042bad5dddfdf594755 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 08:26:56 +0200 Subject: [PATCH 0386/1025] make the code a bit more pythonic --- mycli/packages/completion_engine.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index bd6a7b89c..b97cadf71 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -34,7 +34,8 @@ def suggest_type(full_text, text_before_cursor): # partially typed string which renders the smart completion useless because # it will always return the list of keywords as completion. if word_before_cursor: - if word_before_cursor[-1] == '(' or word_before_cursor[0] == '\\': + if word_before_cursor.endswith( + '(') or word_before_cursor.startswith('\\'): parsed = sqlparse.parse(text_before_cursor) else: parsed = sqlparse.parse( From 034c7ae23766abbdb4c4944ca58b10a8684f5b8c Mon Sep 17 00:00:00 2001 From: "John K. Sterling" Date: Sat, 15 Apr 2017 09:50:09 -0400 Subject: [PATCH 0387/1025] #402 use abbreviated prompt if the default causes it to be huge --- mycli/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index ca3cffc6f..259bdf431 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -77,9 +77,9 @@ def emit(self, record): class MyCli(object): default_prompt = '\\t \\u@\\h:\\d> ' + max_len_prompt = 45 defaults_suffix = None - # In order of being loaded. Files lower in list override earlier ones. cnf_files = [ '/etc/my.cnf', '/etc/mysql/my.cnf', @@ -458,7 +458,10 @@ def run_cli(self): print('Thanks to the contributor -', thanks_picker([author_file, sponsor_file])) def prompt_tokens(cli): - return [(Token.Prompt, self.get_prompt(self.prompt_format))] + prompt = self.get_prompt(self.prompt_format) + if self.prompt_format == self.default_prompt and len(prompt) > self.max_len_prompt: + prompt = self.get_prompt('\\d> ') + return [(Token.Prompt, prompt)] def get_continuation_tokens(cli, width): continuation_prompt = self.get_prompt(self.prompt_continuation_format) From 5c7c544f7fc34cf13a4719c2798e8ce6ae61aac5 Mon Sep 17 00:00:00 2001 From: "John K. Sterling" Date: Sat, 15 Apr 2017 09:52:43 -0400 Subject: [PATCH 0388/1025] #402 keep comment in place --- mycli/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycli/main.py b/mycli/main.py index 259bdf431..0b209b731 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -80,6 +80,7 @@ class MyCli(object): max_len_prompt = 45 defaults_suffix = None + # In order of being loaded. Files lower in list override earlier ones. cnf_files = [ '/etc/my.cnf', '/etc/mysql/my.cnf', From a06675b6cc4801f9b979bbeec0b09722338cc67e Mon Sep 17 00:00:00 2001 From: "John K. Sterling" Date: Sat, 15 Apr 2017 09:59:01 -0400 Subject: [PATCH 0389/1025] #402 update change log --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index fcf9c198d..b393128cb 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,7 @@ Features: --------- * Add ability to specify alternative myclirc file. (Thanks: [Dick Marinus]). +* Add logic to shorten the default prompt if it becomes too long once generated. (Thanks: [John Sterling]) Bug Fixes: ---------- From 7de06479b7d63814bf10c72d205818b31467f06f Mon Sep 17 00:00:00 2001 From: "John K. Sterling" Date: Sat, 15 Apr 2017 09:59:56 -0400 Subject: [PATCH 0390/1025] #402 update change log --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index b393128cb..b9763e4bc 100644 --- a/changelog.md +++ b/changelog.md @@ -5,7 +5,7 @@ Features: --------- * Add ability to specify alternative myclirc file. (Thanks: [Dick Marinus]). -* Add logic to shorten the default prompt if it becomes too long once generated. (Thanks: [John Sterling]) +* Add logic to shorten the default prompt if it becomes too long once generated. (Thanks: [John Sterling]). Bug Fixes: ---------- From 9c70ce12098fc4ebf1567999cd29745cbd767d3a Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 17:10:07 +0200 Subject: [PATCH 0391/1025] don't import as --- mycli/output_formatter/output_formatter.py | 39 ++++++++++------------ 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index c87da5edd..ca8fb4eb4 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -6,15 +6,10 @@ from .expanded import expanded_table from .preprocessors import (override_missing_value, convert_to_string) -from .delimited_output_adapter import (delimiter_adapter, - supported_formats as delimiter_formats, - delimiter_preprocessors) -from .tabulate_adapter import (tabulate_adapter, - supported_formats as tabulate_formats, - preprocessors as tabulate_preprocessors) -from .terminaltables_adapter import ( - terminaltables_adapter, preprocessors as terminaltables_preprocessors, - supported_formats as terminaltables_formats) + +from . import delimited_output_adapter +from . import tabulate_adapter +from . import terminaltables_adapter MISSING_VALUE = '' @@ -81,20 +76,20 @@ def format_output(self, data, headers, format_name=None, **kwargs): convert_to_string), {'missing_value': MISSING_VALUE}) -for delimiter_format in delimiter_formats: - OutputFormatter.register_new_formatter(delimiter_format, delimiter_adapter, - delimiter_preprocessors, - {'table_format': delimiter_format, - 'missing_value': MISSING_VALUE}) +for delimiter_format in delimited_output_adapter.supported_formats: + OutputFormatter.register_new_formatter( + delimiter_format, delimited_output_adapter.delimiter_adapter, + delimited_output_adapter.delimiter_preprocessors, + {'table_format': delimiter_format, 'missing_value': MISSING_VALUE}) -for tabulate_format in tabulate_formats: - OutputFormatter.register_new_formatter(tabulate_format, tabulate_adapter, - tabulate_preprocessors, - {'table_format': tabulate_format, - 'missing_value': MISSING_VALUE}) +for tabulate_format in tabulate_adapter.supported_formats: + OutputFormatter.register_new_formatter( + tabulate_format, tabulate_adapter.tabulate_adapter, + tabulate_adapter.preprocessors, + {'table_format': tabulate_format, 'missing_value': MISSING_VALUE}) -for terminaltables_format in terminaltables_formats: +for terminaltables_format in terminaltables_adapter.supported_formats: OutputFormatter.register_new_formatter( - terminaltables_format, terminaltables_adapter, - terminaltables_preprocessors, + terminaltables_format, terminaltables_adapter.terminaltables_adapter, + terminaltables_adapter.preprocessors, {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) From a5f11e6e023c0bccdc2d93cf8e13bf8c7aced016 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 18:33:38 +0200 Subject: [PATCH 0392/1025] rename all adapter functions to adapter --- mycli/output_formatter/delimited_output_adapter.py | 4 ++-- mycli/output_formatter/output_formatter.py | 8 ++++---- mycli/output_formatter/tabulate_adapter.py | 2 +- mycli/output_formatter/terminaltables_adapter.py | 2 +- tests/test_output_formatter.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py index 8036a5ff7..a01a28433 100644 --- a/mycli/output_formatter/delimited_output_adapter.py +++ b/mycli/output_formatter/delimited_output_adapter.py @@ -8,10 +8,10 @@ from .preprocessors import override_missing_value, bytes_to_string supported_formats = ('csv', 'tsv') -delimiter_preprocessors = (override_missing_value, bytes_to_string) +preprocessors = (override_missing_value, bytes_to_string) -def delimiter_adapter(data, headers, table_format='csv', **_): +def adapter(data, headers, table_format='csv', **_): """Wrap CSV formatting inside a standard function for OutputFormatter.""" with contextlib.closing(StringIO()) as content: if table_format == 'csv': diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py index ca8fb4eb4..61e3c8d52 100644 --- a/mycli/output_formatter/output_formatter.py +++ b/mycli/output_formatter/output_formatter.py @@ -78,18 +78,18 @@ def format_output(self, data, headers, format_name=None, **kwargs): for delimiter_format in delimited_output_adapter.supported_formats: OutputFormatter.register_new_formatter( - delimiter_format, delimited_output_adapter.delimiter_adapter, - delimited_output_adapter.delimiter_preprocessors, + delimiter_format, delimited_output_adapter.adapter, + delimited_output_adapter.preprocessors, {'table_format': delimiter_format, 'missing_value': MISSING_VALUE}) for tabulate_format in tabulate_adapter.supported_formats: OutputFormatter.register_new_formatter( - tabulate_format, tabulate_adapter.tabulate_adapter, + tabulate_format, tabulate_adapter.adapter, tabulate_adapter.preprocessors, {'table_format': tabulate_format, 'missing_value': MISSING_VALUE}) for terminaltables_format in terminaltables_adapter.supported_formats: OutputFormatter.register_new_formatter( - terminaltables_format, terminaltables_adapter.terminaltables_adapter, + terminaltables_format, terminaltables_adapter.adapter, terminaltables_adapter.preprocessors, {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py index ee63b5b1d..b89dcc0bd 100644 --- a/mycli/output_formatter/tabulate_adapter.py +++ b/mycli/output_formatter/tabulate_adapter.py @@ -12,7 +12,7 @@ preprocessors = (bytes_to_string, align_decimals) -def tabulate_adapter(data, headers, table_format=None, missing_value='', **_): +def adapter(data, headers, table_format=None, missing_value='', **_): """Wrap tabulate inside a standard function for OutputFormatter.""" kwargs = {'tablefmt': table_format, 'missingval': missing_value, 'disable_numparse': True} diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py index 0c702abe0..a8f50f985 100644 --- a/mycli/output_formatter/terminaltables_adapter.py +++ b/mycli/output_formatter/terminaltables_adapter.py @@ -7,7 +7,7 @@ preprocessors = (bytes_to_string, override_missing_value, align_decimals) -def terminaltables_adapter(data, headers, table_format=None, **_): +def adapter(data, headers, table_format=None, **_): """Wrap terminaltables inside a standard function for OutputFormatter.""" table_format_handler = { diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index dcc26e62c..a0e0f198d 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -13,11 +13,11 @@ to_string) from mycli.output_formatter.output_formatter import OutputFormatter from mycli.output_formatter.delimited_output_adapter import ( - delimiter_adapter as csv_wrapper) + adapter as csv_wrapper) from mycli.output_formatter.tabulate_adapter import ( - tabulate_adapter as tabulate_wrapper) + adapter as tabulate_wrapper) from mycli.output_formatter.terminaltables_adapter import ( - terminaltables_adapter as terminal_tables_wrapper) + adapter as terminal_tables_wrapper) def test_to_string(): From 8fa96a2f6f49f190b58e1c0f68d5eba788cc14f7 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 15 Apr 2017 21:21:53 +0200 Subject: [PATCH 0393/1025] don't import as (also for tests/test_output_formatter.py) --- tests/test_output_formatter.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/test_output_formatter.py b/tests/test_output_formatter.py index a0e0f198d..9844c1919 100644 --- a/tests/test_output_formatter.py +++ b/tests/test_output_formatter.py @@ -12,12 +12,9 @@ override_missing_value, to_string) from mycli.output_formatter.output_formatter import OutputFormatter -from mycli.output_formatter.delimited_output_adapter import ( - adapter as csv_wrapper) -from mycli.output_formatter.tabulate_adapter import ( - adapter as tabulate_wrapper) -from mycli.output_formatter.terminaltables_adapter import ( - adapter as terminal_tables_wrapper) +from mycli.output_formatter import delimited_output_adapter +from mycli.output_formatter import tabulate_adapter +from mycli.output_formatter import terminaltables_adapter def test_to_string(): @@ -98,7 +95,7 @@ def test_tabulate_wrapper(): """Test the *output_formatter.tabulate_wrapper()* function.""" data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = tabulate_wrapper(data, headers, table_format='psql') + output = tabulate_adapter.adapter(data, headers, table_format='psql') assert output == dedent('''\ +-----------+----------+ | letters | number | @@ -113,7 +110,7 @@ def test_csv_wrapper(): # Test comma-delimited output. data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = csv_wrapper(data, headers) + output = delimited_output_adapter.adapter(data, headers) assert output == dedent('''\ letters,number\r\n\ abc,1\r\n\ @@ -122,7 +119,8 @@ def test_csv_wrapper(): # Test tab-delimited output. data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = csv_wrapper(data, headers, table_format='tsv') + output = delimited_output_adapter.adapter( + data, headers, table_format='tsv') assert output == dedent('''\ letters\tnumber\r\n\ abc\t1\r\n\ @@ -133,7 +131,8 @@ def test_terminal_tables_wrapper(): """Test the *output_formatter.terminal_tables_wrapper()* function.""" data = [['abc', 1], ['d', 456]] headers = ['letters', 'number'] - output = terminal_tables_wrapper(data, headers, table_format='ascii') + output = terminaltables_adapter.adapter( + data, headers, table_format='ascii') assert output == dedent('''\ +---------+--------+ | letters | number | From 6e8e4b85996bfcbd6fc71a461212e98aa5e20115 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:05:03 -0500 Subject: [PATCH 0394/1025] Switch from pycryptodome to cryptography. --- mycli/config.py | 50 ++++++++++++++++++++++++++++++------------------- setup.py | 2 +- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 7f5e0cb25..88bbc48ec 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -6,12 +6,15 @@ from os.path import exists import struct import sys + from configobj import ConfigObj, ConfigObjError +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend + try: basestring except NameError: basestring = str -from Crypto.Cipher import AES logger = logging.getLogger(__name__) @@ -144,7 +147,8 @@ def read_and_decrypt_mylogin_cnf(f): rkey = struct.pack('16B', *rkey) # Create a cipher object using the key. - aes_cipher = AES.new(rkey, AES.MODE_ECB) + aes_cipher = _get_aes_cipher(rkey) + decryptor = aes_cipher.decryptor() # Create a bytes buffer to hold the plaintext. plaintext = BytesIO() @@ -158,24 +162,9 @@ def read_and_decrypt_mylogin_cnf(f): # Read cipher_len bytes from the file and decrypt. cipher = f.read(cipher_len) - pplain = aes_cipher.decrypt(cipher) - - try: - # Determine pad length. - pad_len = ord(pplain[-1:]) - except TypeError: - # ord() was unable to get the value of the byte. - logger.warning('Unable to remove pad.') - continue - - if pad_len > len(pplain) or len(set(pplain[-pad_len:])) != 1: - # Pad length should be less than or equal to the length of the - # plaintext. The pad should have a single unqiue byte. - logger.warning('Invalid pad found in login path file.') + plain = _remove_pad(decryptor.update(cipher)) + if plain is False: continue - - # Get rid of pad. - plain = pplain[:-pad_len] plaintext.write(plain) if plaintext.tell() == 0: @@ -201,3 +190,26 @@ def str_to_bool(s): return False else: raise ValueError('not a recognized boolean value: %s'.format(s)) + +def _get_aes_cipher(key): + """Get the AES cipher object.""" + return Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) + +def _remove_pad(line): + """Remove the pad from the *line*.""" + pad_length = ord(line[-1:]) + try: + # Determine pad length. + pad_length = ord(line[-1:]) + except TypeError: + # ord() was unable to get the value of the byte. + logger.warning('Unable to remove pad.') + return False + + if pad_length > len(line) or len(set(line[-pad_length:])) != 1: + # Pad length should be less than or equal to the length of the + # plaintext. The pad should have a single unqiue byte. + logger.warning('Invalid pad found in login path file.') + return False + + return line[:-pad_length] diff --git a/setup.py b/setup.py index 82cdef6e1..fb05509ef 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ 'PyMySQL >= 0.6.7', 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', - 'pycryptodome >= 3', + 'cryptography >= 1.0.0', ] setup( From 2b049ae41f87ce94218d15f07c2bea97dd3a3335 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:06:20 -0500 Subject: [PATCH 0395/1025] Pep8radius fixes. --- mycli/config.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mycli/config.py b/mycli/config.py index 88bbc48ec..85c29cd7f 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) + def log(logger, level, message): """Logs message to stderr if logging isn't initialized.""" @@ -27,6 +28,7 @@ def log(logger, level, message): else: print(message, file=sys.stderr) + def read_config_file(f): """Read a config file.""" @@ -47,6 +49,7 @@ def read_config_file(f): return config + def read_config_files(files): """Read and merge a list of config files.""" @@ -60,6 +63,7 @@ def read_config_files(files): return config + def write_default_config(source, destination, overwrite=False): destination = os.path.expanduser(destination) if not overwrite and exists(destination): @@ -67,6 +71,7 @@ def write_default_config(source, destination, overwrite=False): shutil.copyfile(source, destination) + def get_mylogin_cnf_path(): """Return the path to the login path file or None if it doesn't exist.""" mylogin_cnf_path = os.getenv('MYSQL_TEST_LOGIN_FILE') @@ -83,6 +88,7 @@ def get_mylogin_cnf_path(): return mylogin_cnf_path return None + def open_mylogin_cnf(name): """Open a readable version of .mylogin.cnf. @@ -105,6 +111,7 @@ def open_mylogin_cnf(name): return TextIOWrapper(plaintext) + def read_and_decrypt_mylogin_cnf(f): """Read and decrypt the contents of .mylogin.cnf. @@ -174,6 +181,7 @@ def read_and_decrypt_mylogin_cnf(f): plaintext.seek(0) return plaintext + def str_to_bool(s): """Convert a string value to its corresponding boolean value.""" if isinstance(s, bool): @@ -191,10 +199,12 @@ def str_to_bool(s): else: raise ValueError('not a recognized boolean value: %s'.format(s)) + def _get_aes_cipher(key): """Get the AES cipher object.""" return Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) + def _remove_pad(line): """Remove the pad from the *line*.""" pad_length = ord(line[-1:]) From 7d91c9280fc33bb45d410131f8b55d8bb62b0c93 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:09:18 -0500 Subject: [PATCH 0396/1025] Add cryptography package change to the changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 1ae07cfd4..43c023f78 100644 --- a/changelog.md +++ b/changelog.md @@ -25,6 +25,7 @@ Internal Changes: * Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). * Run pep8 checks in travis (Thanks: [Irina Truong]). * Remove temporary hack for sqlparse (Thanks: [Dick Marinus]). +* Switch from pycryptodome to cryptography (Thanks: [Thomas Roten]). 1.9.0: ====== From 6d2d1364d0b4f89d4b8ffcb54878dca97ac7836a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:16:38 -0500 Subject: [PATCH 0397/1025] Simplify decryptor/cipher call. --- mycli/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 85c29cd7f..4f17d9f62 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -153,9 +153,8 @@ def read_and_decrypt_mylogin_cnf(f): return None rkey = struct.pack('16B', *rkey) - # Create a cipher object using the key. - aes_cipher = _get_aes_cipher(rkey) - decryptor = aes_cipher.decryptor() + # Create a decryptor object using the key. + decryptor = _get_decryptor(rkey) # Create a bytes buffer to hold the plaintext. plaintext = BytesIO() @@ -200,9 +199,10 @@ def str_to_bool(s): raise ValueError('not a recognized boolean value: %s'.format(s)) -def _get_aes_cipher(key): +def _get_decryptor(key): """Get the AES cipher object.""" - return Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) + c = Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) + return c.decryptor() def _remove_pad(line): From 6927717f9c10d87165a25c8c6dc71dcb7c50e609 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 20:17:30 -0500 Subject: [PATCH 0398/1025] Fix docstring. --- mycli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/config.py b/mycli/config.py index 4f17d9f62..daf5695b3 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -200,7 +200,7 @@ def str_to_bool(s): def _get_decryptor(key): - """Get the AES cipher object.""" + """Get the AES decryptor.""" c = Cipher(algorithms.AES(key), modes.ECB(), backend=default_backend()) return c.decryptor() From 17fa97653f74bc6214ece3ba213d31b188c6c7d0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 18 Apr 2017 21:52:19 -0500 Subject: [PATCH 0399/1025] Releasing version 1.10.0 --- changelog.md | 4 ++-- mycli/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 1ae07cfd4..4ba32f259 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,5 @@ -TBD -=== +1.10.0: +======= Features: --------- diff --git a/mycli/__init__.py b/mycli/__init__.py index e5102d301..52af183e5 100644 --- a/mycli/__init__.py +++ b/mycli/__init__.py @@ -1 +1 @@ -__version__ = '1.9.0' +__version__ = '1.10.0' From 1296aff5d8d19e6ab1f29dde94a29e93e07cddec Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Tue, 18 Apr 2017 21:10:03 -0700 Subject: [PATCH 0400/1025] Update authors and email. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 82cdef6e1..d7e256980 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,8 @@ setup( name='mycli', - author='Amjith Ramanujam', - author_email='amjith[dot]r[at]gmail.com', + author='Mycli Core Team', + author_email='thomas@roten.us', version=version, url='http://mycli.net', packages=find_packages(), From 1087f20c9b5508685a2ee3388d338906e3b80919 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 19 Apr 2017 07:26:26 -0500 Subject: [PATCH 0401/1025] Use mailing list email address for package metadata. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d7e256980..c13d281be 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name='mycli', author='Mycli Core Team', - author_email='thomas@roten.us', + author_email='mycli-users@googlegroups.com', version=version, url='http://mycli.net', packages=find_packages(), From a1da54a24a105d37ccf6446f9e47f550f578f299 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 19 Apr 2017 07:30:07 -0500 Subject: [PATCH 0402/1025] Keep pep8radius happy :) --- setup.py | 64 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/setup.py b/setup.py index c13d281be..d1b2df27b 100644 --- a/setup.py +++ b/setup.py @@ -22,35 +22,35 @@ ] setup( - name='mycli', - author='Mycli Core Team', - author_email='mycli-users@googlegroups.com', - version=version, - url='http://mycli.net', - packages=find_packages(), - package_data={'mycli': ['myclirc', '../AUTHORS', '../SPONSORS']}, - description=description, - long_description=description, - install_requires=install_requirements, - entry_points=''' - [console_scripts] - mycli=mycli.main:cli - ''', - classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Operating System :: Unix', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: SQL', - 'Topic :: Database', - 'Topic :: Database :: Front-Ends', - 'Topic :: Software Development', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - ) + name='mycli', + author='Mycli Core Team', + author_email='mycli-users@googlegroups.com', + version=version, + url='http://mycli.net', + packages=find_packages(), + package_data={'mycli': ['myclirc', '../AUTHORS', '../SPONSORS']}, + description=description, + long_description=description, + install_requires=install_requirements, + entry_points=''' + [console_scripts] + mycli=mycli.main:cli + ''', + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: Unix', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: SQL', + 'Topic :: Database', + 'Topic :: Database :: Front-Ends', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], +) From 081f0d6ca719006bea79a6f5a62ea2d322fad300 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 19 Apr 2017 18:59:19 -0500 Subject: [PATCH 0403/1025] Switch author_email to dev mailing list. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d1b2df27b..fd777e89c 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name='mycli', author='Mycli Core Team', - author_email='mycli-users@googlegroups.com', + author_email='mycli-dev@googlegroups.com', version=version, url='http://mycli.net', packages=find_packages(), From 35abc84cbb0b59c8da64a468d8a5774da7689fed Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Fri, 21 Apr 2017 15:07:50 +0200 Subject: [PATCH 0404/1025] mv authors --- changelog.md | 7 +++++++ AUTHORS => mycli/AUTHORS | 1 + SPONSORS => mycli/SPONSORS | 0 mycli/main.py | 2 +- setup.py | 2 +- tests/test_main.py | 2 +- 6 files changed, 11 insertions(+), 3 deletions(-) rename AUTHORS => mycli/AUTHORS (97%) rename SPONSORS => mycli/SPONSORS (100%) diff --git a/changelog.md b/changelog.md index af5c81fa1..701d9dbce 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,10 @@ +TBD +=== + +Internal Changes: +----------------- +* Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). + 1.10.0: ======= diff --git a/AUTHORS b/mycli/AUTHORS similarity index 97% rename from AUTHORS rename to mycli/AUTHORS index 2eb4af10b..a7bb77650 100644 --- a/AUTHORS +++ b/mycli/AUTHORS @@ -49,6 +49,7 @@ Contributors: * cxbig * chainkite * Michał Górny + * Terje Røsten Creator: -------- diff --git a/SPONSORS b/mycli/SPONSORS similarity index 100% rename from SPONSORS rename to mycli/SPONSORS diff --git a/mycli/main.py b/mycli/main.py index ff3e8d9cb..b24adeadf 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -439,7 +439,7 @@ def run_cli(self): if self.smart_completion: self.refresh_completions() - project_root = os.path.dirname(PACKAGE_ROOT) + project_root = os.path.join(os.path.dirname(PACKAGE_ROOT), 'mycli') author_file = os.path.join(project_root, 'AUTHORS') sponsor_file = os.path.join(project_root, 'SPONSORS') diff --git a/setup.py b/setup.py index 33ab724f4..6e982d9db 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ version=version, url='http://mycli.net', packages=find_packages(), - package_data={'mycli': ['myclirc', '../AUTHORS', '../SPONSORS']}, + package_data={'mycli': ['myclirc', 'AUTHORS', 'SPONSORS']}, description=description, long_description=description, install_requires=install_requirements, diff --git a/tests/test_main.py b/tests/test_main.py index 1271f18da..56c0cb267 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -164,7 +164,7 @@ def test_confirm_destructive_query_notty(executor): assert confirm_destructive_query(sql) is None def test_thanks_picker_utf8(): - project_root = os.path.dirname(PACKAGE_ROOT) + project_root = os.path.join(os.path.dirname(PACKAGE_ROOT), 'mycli') author_file = os.path.join(project_root, 'AUTHORS') sponsor_file = os.path.join(project_root, 'SPONSORS') From 4f77ec7a548ffd787b63c9916c6d955905e20a07 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 22 Apr 2017 20:18:21 +0200 Subject: [PATCH 0405/1025] rename tests/ to test/ rename tests/ to test so it will be included in sdist see: Anything that looks like a test script: test/test*.py (currently, the Distutils don't do anything with test scripts except include them in source distributions, but in the future there will be a standard for testing Python module distributions) https://docs.python.org/3/distutils/sourcedist.html --- .gitignore | 2 +- .travis.yml | 2 +- {tests => test}/conftest.py | 0 {tests => test}/features/basic_commands.feature | 0 {tests => test}/features/crud_database.feature | 0 {tests => test}/features/crud_table.feature | 0 {tests => test}/features/db_utils.py | 0 {tests => test}/features/environment.py | 0 {tests => test}/features/fixture_data/help.txt | 0 .../features/fixture_data/help_commands.txt | 0 {tests => test}/features/fixture_utils.py | 0 {tests => test}/features/iocommands.feature | 0 {tests => test}/features/named_queries.feature | 0 {tests => test}/features/specials.feature | 0 {tests => test}/features/steps/basic_commands.py | 0 {tests => test}/features/steps/crud_database.py | 0 {tests => test}/features/steps/crud_table.py | 0 {tests => test}/features/steps/iocommands.py | 0 {tests => test}/features/steps/named_queries.py | 0 {tests => test}/features/steps/specials.py | 0 {tests => test}/features/steps/wrappers.py | 0 {tests => test}/mylogin.cnf | Bin {tests => test}/test.txt | 0 {tests => test}/test_completion_engine.py | 0 {tests => test}/test_completion_refresher.py | 0 {tests => test}/test_config.py | 0 {tests => test}/test_dbspecial.py | 0 {tests => test}/test_expanded.py | 0 {tests => test}/test_main.py | 0 {tests => test}/test_naive_completion.py | 0 {tests => test}/test_output_formatter.py | 0 {tests => test}/test_parseutils.py | 0 {tests => test}/test_plan.wiki | 0 .../test_smart_completion_public_schema_only.py | 0 {tests => test}/test_special_iocommands.py | 0 {tests => test}/test_sqlexecute.py | 8 ++++---- {tests => test}/test_tabulate.py | 0 {tests => test}/utils.py | 0 38 files changed, 6 insertions(+), 6 deletions(-) rename {tests => test}/conftest.py (100%) rename {tests => test}/features/basic_commands.feature (100%) rename {tests => test}/features/crud_database.feature (100%) rename {tests => test}/features/crud_table.feature (100%) rename {tests => test}/features/db_utils.py (100%) rename {tests => test}/features/environment.py (100%) rename {tests => test}/features/fixture_data/help.txt (100%) rename {tests => test}/features/fixture_data/help_commands.txt (100%) rename {tests => test}/features/fixture_utils.py (100%) rename {tests => test}/features/iocommands.feature (100%) rename {tests => test}/features/named_queries.feature (100%) rename {tests => test}/features/specials.feature (100%) rename {tests => test}/features/steps/basic_commands.py (100%) rename {tests => test}/features/steps/crud_database.py (100%) rename {tests => test}/features/steps/crud_table.py (100%) rename {tests => test}/features/steps/iocommands.py (100%) rename {tests => test}/features/steps/named_queries.py (100%) rename {tests => test}/features/steps/specials.py (100%) rename {tests => test}/features/steps/wrappers.py (100%) rename {tests => test}/mylogin.cnf (100%) rename {tests => test}/test.txt (100%) rename {tests => test}/test_completion_engine.py (100%) rename {tests => test}/test_completion_refresher.py (100%) rename {tests => test}/test_config.py (100%) rename {tests => test}/test_dbspecial.py (100%) rename {tests => test}/test_expanded.py (100%) rename {tests => test}/test_main.py (100%) rename {tests => test}/test_naive_completion.py (100%) rename {tests => test}/test_output_formatter.py (100%) rename {tests => test}/test_parseutils.py (100%) rename {tests => test}/test_plan.wiki (100%) rename {tests => test}/test_smart_completion_public_schema_only.py (100%) rename {tests => test}/test_special_iocommands.py (100%) rename {tests => test}/test_sqlexecute.py (97%) rename {tests => test}/test_tabulate.py (100%) rename {tests => test}/utils.py (100%) diff --git a/.gitignore b/.gitignore index 59fa76be2..e907154fc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ /dist /mycli.egg-info /src -/tests/behave.ini +/test/behave.ini .vagrant *.pyc diff --git a/.travis.yml b/.travis.yml index f78d94b00..021be1c17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: script: - coverage run --source mycli -m py.test - - cd tests + - cd test - behave - cd .. # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. diff --git a/tests/conftest.py b/test/conftest.py similarity index 100% rename from tests/conftest.py rename to test/conftest.py diff --git a/tests/features/basic_commands.feature b/test/features/basic_commands.feature similarity index 100% rename from tests/features/basic_commands.feature rename to test/features/basic_commands.feature diff --git a/tests/features/crud_database.feature b/test/features/crud_database.feature similarity index 100% rename from tests/features/crud_database.feature rename to test/features/crud_database.feature diff --git a/tests/features/crud_table.feature b/test/features/crud_table.feature similarity index 100% rename from tests/features/crud_table.feature rename to test/features/crud_table.feature diff --git a/tests/features/db_utils.py b/test/features/db_utils.py similarity index 100% rename from tests/features/db_utils.py rename to test/features/db_utils.py diff --git a/tests/features/environment.py b/test/features/environment.py similarity index 100% rename from tests/features/environment.py rename to test/features/environment.py diff --git a/tests/features/fixture_data/help.txt b/test/features/fixture_data/help.txt similarity index 100% rename from tests/features/fixture_data/help.txt rename to test/features/fixture_data/help.txt diff --git a/tests/features/fixture_data/help_commands.txt b/test/features/fixture_data/help_commands.txt similarity index 100% rename from tests/features/fixture_data/help_commands.txt rename to test/features/fixture_data/help_commands.txt diff --git a/tests/features/fixture_utils.py b/test/features/fixture_utils.py similarity index 100% rename from tests/features/fixture_utils.py rename to test/features/fixture_utils.py diff --git a/tests/features/iocommands.feature b/test/features/iocommands.feature similarity index 100% rename from tests/features/iocommands.feature rename to test/features/iocommands.feature diff --git a/tests/features/named_queries.feature b/test/features/named_queries.feature similarity index 100% rename from tests/features/named_queries.feature rename to test/features/named_queries.feature diff --git a/tests/features/specials.feature b/test/features/specials.feature similarity index 100% rename from tests/features/specials.feature rename to test/features/specials.feature diff --git a/tests/features/steps/basic_commands.py b/test/features/steps/basic_commands.py similarity index 100% rename from tests/features/steps/basic_commands.py rename to test/features/steps/basic_commands.py diff --git a/tests/features/steps/crud_database.py b/test/features/steps/crud_database.py similarity index 100% rename from tests/features/steps/crud_database.py rename to test/features/steps/crud_database.py diff --git a/tests/features/steps/crud_table.py b/test/features/steps/crud_table.py similarity index 100% rename from tests/features/steps/crud_table.py rename to test/features/steps/crud_table.py diff --git a/tests/features/steps/iocommands.py b/test/features/steps/iocommands.py similarity index 100% rename from tests/features/steps/iocommands.py rename to test/features/steps/iocommands.py diff --git a/tests/features/steps/named_queries.py b/test/features/steps/named_queries.py similarity index 100% rename from tests/features/steps/named_queries.py rename to test/features/steps/named_queries.py diff --git a/tests/features/steps/specials.py b/test/features/steps/specials.py similarity index 100% rename from tests/features/steps/specials.py rename to test/features/steps/specials.py diff --git a/tests/features/steps/wrappers.py b/test/features/steps/wrappers.py similarity index 100% rename from tests/features/steps/wrappers.py rename to test/features/steps/wrappers.py diff --git a/tests/mylogin.cnf b/test/mylogin.cnf similarity index 100% rename from tests/mylogin.cnf rename to test/mylogin.cnf diff --git a/tests/test.txt b/test/test.txt similarity index 100% rename from tests/test.txt rename to test/test.txt diff --git a/tests/test_completion_engine.py b/test/test_completion_engine.py similarity index 100% rename from tests/test_completion_engine.py rename to test/test_completion_engine.py diff --git a/tests/test_completion_refresher.py b/test/test_completion_refresher.py similarity index 100% rename from tests/test_completion_refresher.py rename to test/test_completion_refresher.py diff --git a/tests/test_config.py b/test/test_config.py similarity index 100% rename from tests/test_config.py rename to test/test_config.py diff --git a/tests/test_dbspecial.py b/test/test_dbspecial.py similarity index 100% rename from tests/test_dbspecial.py rename to test/test_dbspecial.py diff --git a/tests/test_expanded.py b/test/test_expanded.py similarity index 100% rename from tests/test_expanded.py rename to test/test_expanded.py diff --git a/tests/test_main.py b/test/test_main.py similarity index 100% rename from tests/test_main.py rename to test/test_main.py diff --git a/tests/test_naive_completion.py b/test/test_naive_completion.py similarity index 100% rename from tests/test_naive_completion.py rename to test/test_naive_completion.py diff --git a/tests/test_output_formatter.py b/test/test_output_formatter.py similarity index 100% rename from tests/test_output_formatter.py rename to test/test_output_formatter.py diff --git a/tests/test_parseutils.py b/test/test_parseutils.py similarity index 100% rename from tests/test_parseutils.py rename to test/test_parseutils.py diff --git a/tests/test_plan.wiki b/test/test_plan.wiki similarity index 100% rename from tests/test_plan.wiki rename to test/test_plan.wiki diff --git a/tests/test_smart_completion_public_schema_only.py b/test/test_smart_completion_public_schema_only.py similarity index 100% rename from tests/test_smart_completion_public_schema_only.py rename to test/test_smart_completion_public_schema_only.py diff --git a/tests/test_special_iocommands.py b/test/test_special_iocommands.py similarity index 100% rename from tests/test_special_iocommands.py rename to test/test_special_iocommands.py diff --git a/tests/test_sqlexecute.py b/test/test_sqlexecute.py similarity index 97% rename from tests/test_sqlexecute.py rename to test/test_sqlexecute.py index a9b5fbec2..157b62341 100644 --- a/tests/test_sqlexecute.py +++ b/test/test_sqlexecute.py @@ -229,7 +229,7 @@ def test_system_command_not_found(executor): @dbtest def test_system_command_output(executor): - test_file_path = os.path.join(os.path.abspath('.'), 'tests/test.txt') + test_file_path = os.path.join(os.path.abspath('.'), 'test', 'test.txt') results = run(executor, 'system cat {0}'.format(test_file_path)) assert len(results) == 1 expected_line = u'mycli rocks!\n' @@ -237,9 +237,9 @@ def test_system_command_output(executor): @dbtest def test_cd_command_current_dir(executor): - tests_path = os.path.join(os.path.abspath('.'), 'tests') - results = run(executor, 'system cd {0}'.format(tests_path)) - assert os.getcwd() == tests_path + test_path = os.path.join(os.path.abspath('.'), 'test') + results = run(executor, 'system cd {0}'.format(test_path)) + assert os.getcwd() == test_path @dbtest def test_unicode_support(executor): diff --git a/tests/test_tabulate.py b/test/test_tabulate.py similarity index 100% rename from tests/test_tabulate.py rename to test/test_tabulate.py diff --git a/tests/utils.py b/test/utils.py similarity index 100% rename from tests/utils.py rename to test/utils.py From 9e20cc6711c89696843363e8c090abf2be235425 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 22 Apr 2017 12:48:37 +0200 Subject: [PATCH 0406/1025] add more files to MANIFEST.in --- MANIFEST.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 1d3bbc866..a50b73686 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ -include LICENSE.txt *.md +include LICENSE.txt *.md *.rst TODO requirements-dev.txt screenshots/* +include conftest.py .coveragerc pytest.ini test tox.ini From 26782c77520d9d0dc08a4820abf5450b90ca0846 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 22 Apr 2017 13:08:25 +0200 Subject: [PATCH 0407/1025] add to changelog.md --- changelog.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/changelog.md b/changelog.md index af5c81fa1..f00fc4359 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,12 @@ +TBD +=== + +Internal Changes: +----------------- + +* Rename tests/ to test/ . (Thanks: [Dick Marinus]). + + 1.10.0: ======= From a9af5d41a4b0e079776a5baf7c214196dbfc57a3 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sat, 22 Apr 2017 20:08:31 +0200 Subject: [PATCH 0408/1025] fix pep8 code style --- test/conftest.py | 3 +- test/features/db_utils.py | 27 +-- test/features/environment.py | 17 +- test/features/fixture_utils.py | 9 +- test/features/steps/basic_commands.py | 33 ++-- test/features/steps/crud_database.py | 60 +++---- test/features/steps/crud_table.py | 63 +++---- test/features/steps/iocommands.py | 7 +- test/features/steps/named_queries.py | 33 ++-- test/features/steps/specials.py | 20 +-- test/features/steps/wrappers.py | 3 +- test/test_completion_engine.py | 170 +++++++++++------- test/test_completion_refresher.py | 15 +- test/test_config.py | 2 +- test/test_dbspecial.py | 3 +- test/test_main.py | 9 + test/test_naive_completion.py | 6 + test/test_parseutils.py | 21 ++- ...est_smart_completion_public_schema_only.py | 79 +++++--- test/test_special_iocommands.py | 15 +- test/test_sqlexecute.py | 30 +++- test/utils.py | 13 +- 22 files changed, 369 insertions(+), 269 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index d24d26bc4..d7d133400 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,5 +1,6 @@ import pytest -from utils import (HOST, USER, PASSWORD, PORT, CHARSET, create_db, db_connection) +from utils import (HOST, USER, PASSWORD, PORT, + CHARSET, create_db, db_connection) import mycli.sqlexecute diff --git a/test/features/db_utils.py b/test/features/db_utils.py index f604c608c..ef0b42ffc 100644 --- a/test/features/db_utils.py +++ b/test/features/db_utils.py @@ -4,15 +4,17 @@ import pymysql + def create_db(hostname='localhost', username=None, password=None, dbname=None): - """ - Create test database. + """Create test database. + :param hostname: string :param username: string :param password: string :param dbname: string :return: + """ cn = pymysql.connect( host=hostname, @@ -23,8 +25,8 @@ def create_db(hostname='localhost', username=None, password=None, ) with cn.cursor() as cr: - cr.execute('drop database if exists '+dbname) - cr.execute('create database '+dbname) + cr.execute('drop database if exists ' + dbname) + cr.execute('create database ' + dbname) cn.close() @@ -33,13 +35,14 @@ def create_db(hostname='localhost', username=None, password=None, def create_cn(hostname, password, username, dbname): - """ - Open connection to database. + """Open connection to database. + :param hostname: :param password: :param username: :param dbname: string :return: psycopg2.connection + """ cn = pymysql.connect( host=hostname, @@ -55,12 +58,13 @@ def create_cn(hostname, password, username, dbname): def drop_db(hostname='localhost', username=None, password=None, dbname=None): - """ - Drop database. + """Drop database. + :param hostname: string :param username: string :param password: string :param dbname: string + """ cn = pymysql.connect( host=hostname, @@ -72,15 +76,16 @@ def drop_db(hostname='localhost', username=None, password=None, ) with cn.cursor() as cr: - cr.execute('drop database if exists '+dbname) + cr.execute('drop database if exists ' + dbname) close_cn(cn) def close_cn(cn=None): - """ - Close connection. + """Close connection. + :param connection: pymysql.connection + """ if cn: cn.close() diff --git a/test/features/environment.py b/test/features/environment.py index 3f55757b8..e79e5740c 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -9,9 +9,7 @@ def before_all(context): - """ - Set env parameters. - """ + """Set env parameters.""" os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" os.environ['PAGER'] = 'cat' @@ -21,7 +19,8 @@ def before_all(context): context.exit_sent = False vi = '_'.join([str(x) for x in sys.version_info[:3]]) - db_name = context.config.userdata.get('my_test_db', None) or "mycli_behave_tests" + db_name = context.config.userdata.get( + 'my_test_db', None) or "mycli_behave_tests" db_name_full = '{0}_{1}'.format(db_name, vi) # Store get params from config/environment variables @@ -40,7 +39,7 @@ def before_all(context): ), 'cli_command': context.config.userdata.get( 'my_cli_command', None) or - sys.executable+' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"', + sys.executable + ' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"', 'dbname': db_name, 'dbname_tmp': db_name_full + '_tmp', 'vi': vi, @@ -54,9 +53,7 @@ def before_all(context): def after_all(context): - """ - Unset env parameters. - """ + """Unset env parameters.""" dbutils.close_cn(context.cn) dbutils.drop_db(context.conf['host'], context.conf['user'], context.conf['pass'], context.conf['dbname']) @@ -70,9 +67,7 @@ def after_all(context): def after_scenario(context, _): - """ - Cleans up after each test complete. - """ + """Cleans up after each test complete.""" if hasattr(context, 'cli') and not context.exit_sent: # Terminate nicely. diff --git a/test/features/fixture_utils.py b/test/features/fixture_utils.py index f3b490c40..a171e34ce 100644 --- a/test/features/fixture_utils.py +++ b/test/features/fixture_utils.py @@ -6,10 +6,11 @@ def read_fixture_lines(filename): - """ - Read lines of text from file. + """Read lines of text from file. + :param filename: string name :return: list of strings + """ lines = [] for line in io.open(filename, 'r', encoding='utf8'): @@ -18,9 +19,7 @@ def read_fixture_lines(filename): def read_fixture_files(): - """ - Read all files inside fixture_data directory. - """ + """Read all files inside fixture_data directory.""" fixture_dict = {} current_dir = os.path.dirname(__file__) diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index 7aa476640..109472b8a 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -14,9 +15,7 @@ @when('we run dbcli') def step_run_cli(context): - """ - Run the process using pexpect. - """ + """Run the process using pexpect.""" run_args = [] if context.conf.get('host', None): run_args.extend(('-h', context.conf['host'])) @@ -26,7 +25,8 @@ def step_run_cli(context): run_args.extend(('-p', context.conf['pass'])) if context.conf.get('dbname', None): run_args.extend(('-D', context.conf['dbname'])) - cli_cmd = context.conf.get('cli_command', None) or sys.executable+' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' + cli_cmd = context.conf.get('cli_command', None) or sys.executable + \ + ' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' cmd_parts = [cli_cmd] + run_args cmd = ' '.join(cmd_parts) @@ -36,27 +36,26 @@ def step_run_cli(context): @when('we wait for prompt') def step_wait_prompt(context): - """ - Make sure prompt is displayed. - """ + """Make sure prompt is displayed.""" user = context.conf['user'] host = context.conf['host'] dbname = context.conf['dbname'] - wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format(user, host, dbname), timeout=5) + wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( + user, host, dbname), timeout=5) @when('we send "ctrl + d"') def step_ctrl_d(context): - """ - Send Ctrl + D to hopefully exit. - """ + """Send Ctrl + D to hopefully exit.""" context.cli.sendcontrol('d') context.exit_sent = True @when('we send "\?" command') def step_send_help(context): - """ - Send \? to see help. + """Send \? + + to see help. + """ context.cli.sendline('\\?') diff --git a/test/features/steps/crud_database.py b/test/features/steps/crud_database.py index 3eab34d9d..d7b8eeb28 100644 --- a/test/features/steps/crud_database.py +++ b/test/features/steps/crud_database.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -14,9 +15,7 @@ @when('we create database') def step_db_create(context): - """ - Send create database. - """ + """Send create database.""" context.cli.sendline('create database {0};'.format( context.conf['dbname_tmp'])) @@ -27,78 +26,67 @@ def step_db_create(context): @when('we drop database') def step_db_drop(context): - """ - Send drop database. - """ + """Send drop database.""" context.cli.sendline('drop database {0};'.format( context.conf['dbname_tmp'])) - wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + wrappers.expect_exact( + context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) context.cli.sendline('y') + @when('we connect to test database') def step_db_connect_test(context): - """ - Send connect to database. - """ + """Send connect to database.""" db_name = context.conf['dbname'] context.cli.sendline('use {0}'.format(db_name)) @when('we connect to dbserver') def step_db_connect_dbserver(context): - """ - Send connect to database. - """ + """Send connect to database.""" context.cli.sendline('use mysql') @then('dbcli exits') def step_wait_exit(context): - """ - Make sure the cli exits. - """ + """Make sure the cli exits.""" wrappers.expect_exact(context, pexpect.EOF, timeout=5) @then('we see dbcli prompt') def step_see_prompt(context): - """ - Wait to see the prompt. - """ + """Wait to see the prompt.""" user = context.conf['user'] host = context.conf['host'] dbname = context.conf['dbname'] - wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format(user, host, dbname), timeout=5) + wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( + user, host, dbname), timeout=5) @then('we see help output') def step_see_help(context): for expected_line in context.fixture_data['help_commands.txt']: - wrappers.expect_exact(context, expected_line+'\r\n', timeout=1) + wrappers.expect_exact(context, expected_line + '\r\n', timeout=1) @then('we see database created') def step_see_db_created(context): - """ - Wait to see create database output. - """ + """Wait to see create database output.""" wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see database dropped') def step_see_db_dropped(context): - """ - Wait to see drop database output. - """ + """Wait to see drop database output.""" wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see database connected') def step_see_db_connected(context): - """ - Wait to see drop database output. - """ - wrappers.expect_exact(context, 'You are now connected to database "', timeout=2) + """Wait to see drop database output.""" + wrappers.expect_exact( + context, 'You are now connected to database "', timeout=2) wrappers.expect_exact(context, '"', timeout=2) - wrappers.expect_exact(context, ' as user "{0}"\r\n'.format(context.conf['user']), timeout=2) + wrappers.expect_exact(context, ' as user "{0}"\r\n'.format( + context.conf['user']), timeout=2) diff --git a/test/features/steps/crud_table.py b/test/features/steps/crud_table.py index b73ff0d6f..34301c890 100644 --- a/test/features/steps/crud_table.py +++ b/test/features/steps/crud_table.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -12,100 +13,78 @@ @when('we create table') def step_create_table(context): - """ - Send create table. - """ + """Send create table.""" context.cli.sendline('create table a(x text);') @when('we insert into table') def step_insert_into_table(context): - """ - Send insert into table. - """ + """Send insert into table.""" context.cli.sendline('''insert into a(x) values('xxx');''') @when('we update table') def step_update_table(context): - """ - Send insert into table. - """ + """Send insert into table.""" context.cli.sendline('''update a set x = 'yyy' where x = 'xxx';''') @when('we select from table') def step_select_from_table(context): - """ - Send select from table. - """ + """Send select from table.""" context.cli.sendline('select * from a;') @when('we delete from table') def step_delete_from_table(context): - """ - Send deete from table. - """ + """Send deete from table.""" context.cli.sendline('''delete from a where x = 'yyy';''') - wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + wrappers.expect_exact( + context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) context.cli.sendline('y') @when('we drop table') def step_drop_table(context): - """ - Send drop table. - """ + """Send drop table.""" context.cli.sendline('drop table a;') - wrappers.expect_exact(context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) + wrappers.expect_exact( + context, 'You\'re about to run a destructive command.\r\nDo you want to proceed? (y/n):', timeout=2) context.cli.sendline('y') @then('we see table created') def step_see_table_created(context): - """ - Wait to see create table output. - """ + """Wait to see create table output.""" wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see record inserted') def step_see_record_inserted(context): - """ - Wait to see insert output. - """ + """Wait to see insert output.""" wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see record updated') def step_see_record_updated(context): - """ - Wait to see update output. - """ + """Wait to see update output.""" wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see data selected') def step_see_data_selected(context): - """ - Wait to see select output. - """ + """Wait to see select output.""" wrappers.expect_exact( context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) @then('we see record deleted') def step_see_data_deleted(context): - """ - Wait to see delete output. - """ + """Wait to see delete output.""" wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see table dropped') def step_see_table_dropped(context): - """ - Wait to see drop output. - """ + """Wait to see drop output.""" wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 885200469..73068fac2 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -8,14 +8,13 @@ @when('we start external editor providing a file name') def step_edit_file(context): - """ - Edit file with external editor. - """ + """Edit file with external editor.""" context.editor_file_name = 'test_file_{0}.sql'.format(context.conf['vi']) if os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) context.cli.sendline('\e {0}'.format(context.editor_file_name)) - wrappers.expect_exact(context, 'Entering Ex mode. Type "visual" to go to Normal mode.', timeout=2) + wrappers.expect_exact( + context, 'Entering Ex mode. Type "visual" to go to Normal mode.', timeout=2) wrappers.expect_exact(context, '\r\n:', timeout=2) diff --git a/test/features/steps/named_queries.py b/test/features/steps/named_queries.py index b53ad47db..60115c5cd 100644 --- a/test/features/steps/named_queries.py +++ b/test/features/steps/named_queries.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -12,48 +13,36 @@ @when('we save a named query') def step_save_named_query(context): - """ - Send \ns command - """ + """Send \ns command.""" context.cli.sendline('\\fs foo SELECT 12345') @when('we use a named query') def step_use_named_query(context): - """ - Send \n command - """ + """Send \n command.""" context.cli.sendline('\\f foo') @when('we delete a named query') def step_delete_named_query(context): - """ - Send \nd command - """ + """Send \nd command.""" context.cli.sendline('\\fd foo') @then('we see the named query saved') def step_see_named_query_saved(context): - """ - Wait to see query saved. - """ + """Wait to see query saved.""" wrappers.expect_exact(context, 'Saved.', timeout=1) @then('we see the named query executed') def step_see_named_query_executed(context): - """ - Wait to see select output. - """ + """Wait to see select output.""" wrappers.expect_exact(context, '12345', timeout=1) wrappers.expect_exact(context, 'SELECT 1', timeout=1) @then('we see the named query deleted') def step_see_named_query_deleted(context): - """ - Wait to see query deleted. - """ + """Wait to see query deleted.""" wrappers.expect_exact(context, 'foo: Deleted', timeout=1) diff --git a/test/features/steps/specials.py b/test/features/steps/specials.py index 790b2476f..c0a3c0feb 100644 --- a/test/features/steps/specials.py +++ b/test/features/steps/specials.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -""" -Steps for behavioral style tests are defined in this module. -Each step is defined by the string decorating it. -This string is used to call the step in "*.feature" file. +"""Steps for behavioral style tests are defined in this module. + +Each step is defined by the string decorating it. This string is used +to call the step in "*.feature" file. + """ from __future__ import unicode_literals @@ -12,15 +13,12 @@ @when('we refresh completions') def step_refresh_completions(context): - """ - Send refresh command. - """ + """Send refresh command.""" context.cli.sendline('rehash') @then('we see completions refresh started') def step_see_refresh_started(context): - """ - Wait to see refresh output. - """ - wrappers.expect_exact(context, 'Auto-completion refresh started in the background', timeout=2) + """Wait to see refresh output.""" + wrappers.expect_exact( + context, 'Auto-completion refresh started in the background', timeout=2) diff --git a/test/features/steps/wrappers.py b/test/features/steps/wrappers.py index eac7c8304..aea742033 100644 --- a/test/features/steps/wrappers.py +++ b/test/features/steps/wrappers.py @@ -9,7 +9,8 @@ def expect_exact(context, expected, timeout): context.cli.expect_exact(expected, timeout=timeout) except: # Strip color codes out of the output. - actual = re.sub(r'\x1b\[([0-9A-Za-z;?])+[m|K]?', '', context.cli.before) + actual = re.sub(r'\x1b\[([0-9A-Za-z;?])+[m|K]?', + '', context.cli.before) raise Exception('Expected:\n---\n{0!r}\n---\n\nActual:\n---\n{1!r}\n---'.format( expected, actual)) diff --git a/test/test_completion_engine.py b/test/test_completion_engine.py index 9b8771083..4f0406b08 100644 --- a/test/test_completion_engine.py +++ b/test/test_completion_engine.py @@ -1,25 +1,28 @@ from mycli.packages.completion_engine import suggest_type import pytest + def sorted_dicts(dicts): - """input is a list of dicts""" + """input is a list of dicts.""" return sorted(tuple(x.items()) for x in dicts) + def test_select_suggests_cols_with_visible_table_scope(): suggestions = suggest_type('SELECT FROM tabl', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + def test_select_suggests_cols_with_qualified_table_scope(): suggestions = suggest_type('SELECT FROM sch.tabl', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [('sch', 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [('sch', 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) @pytest.mark.parametrize('expression', [ @@ -37,10 +40,11 @@ def test_select_suggests_cols_with_qualified_table_scope(): def test_where_suggests_columns_functions(expression): suggestions = suggest_type(expression, expression) assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + @pytest.mark.parametrize('expression', [ 'SELECT * FROM tabl WHERE foo IN (', @@ -49,41 +53,49 @@ def test_where_suggests_columns_functions(expression): def test_where_in_suggests_columns(expression): suggestions = suggest_type(expression, expression) assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + def test_where_equals_any_suggests_columns_or_keywords(): text = 'SELECT * FROM tabl WHERE foo = ANY(' suggestions = suggest_type(text, text) assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'tabl', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}]) + {'type': 'column', 'tables': [(None, 'tabl', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}]) + def test_lparen_suggests_cols(): suggestion = suggest_type('SELECT MAX( FROM tbl', 'SELECT MAX(') assert suggestion == [ {'type': 'column', 'tables': [(None, 'tbl', None)]}] + def test_operand_inside_function_suggests_cols1(): - suggestion = suggest_type('SELECT MAX(col1 + FROM tbl', 'SELECT MAX(col1 + ') + suggestion = suggest_type( + 'SELECT MAX(col1 + FROM tbl', 'SELECT MAX(col1 + ') assert suggestion == [ {'type': 'column', 'tables': [(None, 'tbl', None)]}] + def test_operand_inside_function_suggests_cols2(): - suggestion = suggest_type('SELECT MAX(col1 + col2 + FROM tbl', 'SELECT MAX(col1 + col2 + ') + suggestion = suggest_type( + 'SELECT MAX(col1 + col2 + FROM tbl', 'SELECT MAX(col1 + col2 + ') assert suggestion == [ {'type': 'column', 'tables': [(None, 'tbl', None)]}] + def test_select_suggests_cols_and_funcs(): suggestions = suggest_type('SELECT ', 'SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': []}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': []}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + @pytest.mark.parametrize('expression', [ 'SELECT * FROM ', @@ -102,6 +114,7 @@ def test_expression_suggests_tables_views_and_schemas(expression): {'type': 'view', 'schema': []}, {'type': 'schema'}]) + @pytest.mark.parametrize('expression', [ 'SELECT * FROM sch.', 'INSERT INTO sch.', @@ -118,37 +131,43 @@ def test_expression_suggests_qualified_tables_views_and_schemas(expression): {'type': 'table', 'schema': 'sch'}, {'type': 'view', 'schema': 'sch'}]) + def test_truncate_suggests_tables_and_schemas(): suggestions = suggest_type('TRUNCATE ', 'TRUNCATE ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'table', 'schema': []}, {'type': 'schema'}]) + def test_truncate_suggests_qualified_tables(): suggestions = suggest_type('TRUNCATE sch.', 'TRUNCATE sch.') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'table', 'schema': 'sch'}]) + def test_distinct_suggests_cols(): suggestions = suggest_type('SELECT DISTINCT ', 'SELECT DISTINCT ') assert suggestions == [{'type': 'column', 'tables': []}] + def test_col_comma_suggests_cols(): suggestions = suggest_type('SELECT a, b, FROM tbl', 'SELECT a, b,') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tbl', None)]}, {'type': 'function', 'schema': []}, {'type': 'keyword'}, - ]) + ]) + def test_table_comma_suggests_tables_and_schemas(): suggestions = suggest_type('SELECT a, b FROM tbl1, ', - 'SELECT a, b FROM tbl1, ') + 'SELECT a, b FROM tbl1, ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'table', 'schema': []}, {'type': 'view', 'schema': []}, {'type': 'schema'}]) + def test_into_suggests_tables_and_schemas(): suggestion = suggest_type('INSERT INTO ', 'INSERT INTO ') assert sorted_dicts(suggestion) == sorted_dicts([ @@ -156,26 +175,31 @@ def test_into_suggests_tables_and_schemas(): {'type': 'view', 'schema': []}, {'type': 'schema'}]) + def test_insert_into_lparen_suggests_cols(): suggestions = suggest_type('INSERT INTO abc (', 'INSERT INTO abc (') assert suggestions == [{'type': 'column', 'tables': [(None, 'abc', None)]}] + def test_insert_into_lparen_partial_text_suggests_cols(): suggestions = suggest_type('INSERT INTO abc (i', 'INSERT INTO abc (i') assert suggestions == [{'type': 'column', 'tables': [(None, 'abc', None)]}] + def test_insert_into_lparen_comma_suggests_cols(): suggestions = suggest_type('INSERT INTO abc (id,', 'INSERT INTO abc (id,') assert suggestions == [{'type': 'column', 'tables': [(None, 'abc', None)]}] + def test_partially_typed_col_name_suggests_col_names(): suggestions = suggest_type('SELECT * FROM tabl WHERE col_n', - 'SELECT * FROM tabl WHERE col_n') + 'SELECT * FROM tabl WHERE col_n') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl', None)]}, {'type': 'function', 'schema': []}, {'type': 'keyword'}, - ]) + ]) + def test_dot_suggests_cols_of_a_table_or_schema_qualified_table(): suggestions = suggest_type('SELECT tabl. FROM tabl', 'SELECT tabl.') @@ -185,24 +209,27 @@ def test_dot_suggests_cols_of_a_table_or_schema_qualified_table(): {'type': 'view', 'schema': 'tabl'}, {'type': 'function', 'schema': 'tabl'}]) + def test_dot_suggests_cols_of_an_alias(): suggestions = suggest_type('SELECT t1. FROM tabl1 t1, tabl2 t2', - 'SELECT t1.') + 'SELECT t1.') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'table', 'schema': 't1'}, {'type': 'view', 'schema': 't1'}, {'type': 'column', 'tables': [(None, 'tabl1', 't1')]}, {'type': 'function', 'schema': 't1'}]) + def test_dot_col_comma_suggests_cols_or_schema_qualified_table(): suggestions = suggest_type('SELECT t1.a, t2. FROM tabl1 t1, tabl2 t2', - 'SELECT t1.a, t2.') + 'SELECT t1.a, t2.') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl2', 't2')]}, {'type': 'table', 'schema': 't2'}, {'type': 'view', 'schema': 't2'}, {'type': 'function', 'schema': 't2'}]) + @pytest.mark.parametrize('expression', [ 'SELECT * FROM (', 'SELECT * FROM foo WHERE EXISTS (', @@ -212,6 +239,7 @@ def test_sub_select_suggests_keyword(expression): suggestion = suggest_type(expression, expression) assert suggestion == [{'type': 'keyword'}] + @pytest.mark.parametrize('expression', [ 'SELECT * FROM (S', 'SELECT * FROM foo WHERE EXISTS (S', @@ -221,6 +249,7 @@ def test_sub_select_partial_text_suggests_keyword(expression): suggestion = suggest_type(expression, expression) assert suggestion == [{'type': 'keyword'}] + def test_outer_table_reference_in_exists_subquery_suggests_columns(): q = 'SELECT * FROM foo f WHERE EXISTS (SELECT 1 FROM bar WHERE f.' suggestions = suggest_type(q, q) @@ -230,6 +259,7 @@ def test_outer_table_reference_in_exists_subquery_suggests_columns(): {'type': 'view', 'schema': 'f'}, {'type': 'function', 'schema': 'f'}] + @pytest.mark.parametrize('expression', [ 'SELECT * FROM (SELECT * FROM ', 'SELECT * FROM foo WHERE EXISTS (SELECT * FROM ', @@ -242,32 +272,36 @@ def test_sub_select_table_name_completion(expression): {'type': 'view', 'schema': []}, {'type': 'schema'}]) + def test_sub_select_col_name_completion(): suggestions = suggest_type('SELECT * FROM (SELECT FROM abc', - 'SELECT * FROM (SELECT ') + 'SELECT * FROM (SELECT ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'abc', None)]}, {'type': 'function', 'schema': []}, {'type': 'keyword'}, - ]) + ]) + @pytest.mark.xfail def test_sub_select_multiple_col_name_completion(): suggestions = suggest_type('SELECT * FROM (SELECT a, FROM abc', - 'SELECT * FROM (SELECT a, ') + 'SELECT * FROM (SELECT a, ') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'abc', None)]}, {'type': 'function', 'schema': []}]) + def test_sub_select_dot_col_name_completion(): suggestions = suggest_type('SELECT * FROM (SELECT t. FROM tabl t', - 'SELECT * FROM (SELECT t.') + 'SELECT * FROM (SELECT t.') assert sorted_dicts(suggestions) == sorted_dicts([ {'type': 'column', 'tables': [(None, 'tabl', 't')]}, {'type': 'table', 'schema': 't'}, {'type': 'view', 'schema': 't'}, {'type': 'function', 'schema': 't'}]) + @pytest.mark.parametrize('join_type', ['', 'INNER', 'LEFT', 'RIGHT OUTER']) @pytest.mark.parametrize('tbl_alias', ['', 'foo']) def test_join_suggests_tables_and_schemas(tbl_alias, join_type): @@ -278,6 +312,7 @@ def test_join_suggests_tables_and_schemas(tbl_alias, join_type): {'type': 'view', 'schema': []}, {'type': 'schema'}]) + @pytest.mark.parametrize('sql', [ 'SELECT * FROM abc a JOIN def d ON a.', 'SELECT * FROM abc a JOIN def d ON a.id = d.id AND a.', @@ -290,6 +325,7 @@ def test_join_alias_dot_suggests_cols1(sql): {'type': 'view', 'schema': 'a'}, {'type': 'function', 'schema': 'a'}]) + @pytest.mark.parametrize('sql', [ 'SELECT * FROM abc a JOIN def d ON a.id = d.', 'SELECT * FROM abc a JOIN def d ON a.id = d.id AND a.id2 = d.', @@ -302,6 +338,7 @@ def test_join_alias_dot_suggests_cols2(sql): {'type': 'view', 'schema': 'd'}, {'type': 'function', 'schema': 'd'}]) + @pytest.mark.parametrize('sql', [ 'select a.x, b.y from abc a join bcd b on ', 'select a.x, b.y from abc a join bcd b on a.id = b.id OR ', @@ -310,6 +347,7 @@ def test_on_suggests_aliases(sql): suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['a', 'b']}] + @pytest.mark.parametrize('sql', [ 'select abc.x, bcd.y from abc join bcd on ', 'select abc.x, bcd.y from abc join bcd on abc.id = bcd.id AND ', @@ -318,6 +356,7 @@ def test_on_suggests_tables(sql): suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['abc', 'bcd']}] + @pytest.mark.parametrize('sql', [ 'select a.x, b.y from abc a join bcd b on a.id = ', 'select a.x, b.y from abc a join bcd b on a.id = b.id AND a.id2 = ', @@ -326,6 +365,7 @@ def test_on_suggests_aliases_right_side(sql): suggestions = suggest_type(sql, sql) assert suggestions == [{'type': 'alias', 'aliases': ['a', 'b']}] + @pytest.mark.parametrize('sql', [ 'select abc.x, bcd.y from abc join bcd on ', 'select abc.x, bcd.y from abc join bcd on abc.id = bcd.id and ', @@ -348,57 +388,59 @@ def test_2_statements_2nd_current(): suggestions = suggest_type('select * from a; select * from ', 'select * from a; select * from ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) suggestions = suggest_type('select * from a; select from b', 'select * from a; select ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'b', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'b', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) # Should work even if first statement is invalid suggestions = suggest_type('select * from; select * from ', 'select * from; select * from ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) + def test_2_statements_1st_current(): suggestions = suggest_type('select * from ; select * from b', 'select * from ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) suggestions = suggest_type('select from a; select * from b', 'select ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'a', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'a', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) + def test_3_statements_2nd_current(): suggestions = suggest_type('select * from a; select * from ; select * from c', 'select * from a; select * from ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) suggestions = suggest_type('select * from a; select from b; select * from c', 'select * from a; select ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'column', 'tables': [(None, 'b', None)]}, - {'type': 'function', 'schema': []}, - {'type': 'keyword'}, - ]) + {'type': 'column', 'tables': [(None, 'b', None)]}, + {'type': 'function', 'schema': []}, + {'type': 'keyword'}, + ]) def test_create_db_with_template(): @@ -435,13 +477,15 @@ def test_handle_pre_completion_comma_gracefully(text): assert iter(suggestions) + def test_cross_join(): text = 'select * from v1 cross join v2 JOIN v1.id, ' suggestions = suggest_type(text, text) assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'table', 'schema': []}, - {'type': 'view', 'schema': []}, - {'type': 'schema'}]) + {'type': 'table', 'schema': []}, + {'type': 'view', 'schema': []}, + {'type': 'schema'}]) + @pytest.mark.parametrize('expression', [ 'SELECT 1 AS ', diff --git a/test/test_completion_refresher.py b/test/test_completion_refresher.py index 8851eae65..1ed63774a 100644 --- a/test/test_completion_refresher.py +++ b/test/test_completion_refresher.py @@ -10,10 +10,11 @@ def refresher(): def test_ctor(refresher): - """ - Refresher object should contain a few handlers + """Refresher object should contain a few handlers. + :param refresher: :return: + """ assert len(refresher.refreshers) > 0 actual_handlers = list(refresher.refreshers.keys()) @@ -41,10 +42,11 @@ def test_refresh_called_once(refresher): def test_refresh_called_twice(refresher): - """ - If refresh is called a second time, it should be restarted + """If refresh is called a second time, it should be restarted. + :param refresher: :return: + """ callbacks = Mock() @@ -69,9 +71,10 @@ def dummy_bg_refresh(*args): def test_refresh_with_callbacks(refresher): - """ - Callbacks must be called + """Callbacks must be called. + :param refresher: + """ callbacks = [Mock()] sqlexecute_class = Mock() diff --git a/test/test_config.py b/test/test_config.py index 2a0d26c18..8ef8b7819 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -11,7 +11,7 @@ read_and_decrypt_mylogin_cnf, str_to_bool) with_pycryptodome = ['pycryptodome' in set([package.project_name for package in - pip.get_installed_distributions()])] + pip.get_installed_distributions()])] LOGIN_PATH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'mylogin.cnf')) diff --git a/test/test_dbspecial.py b/test/test_dbspecial.py index 17309b29c..3733c813f 100644 --- a/test/test_dbspecial.py +++ b/test/test_dbspecial.py @@ -2,10 +2,11 @@ from test_completion_engine import sorted_dicts from mycli.packages.special.utils import format_uptime + def test_u_suggests_databases(): suggestions = suggest_type('\\u ', '\\u ') assert sorted_dicts(suggestions) == sorted_dicts([ - {'type': 'database'}]) + {'type': 'database'}]) def test_describe_table(): diff --git a/test/test_main.py b/test/test_main.py index 1271f18da..eb9c8623c 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -18,6 +18,7 @@ CLI_ARGS = ['--user', USER, '--host', HOST, '--port', PORT, '--password', PASSWORD, '_test_db'] + @dbtest def test_execute_arg(executor): run(executor, 'create table test (a text)') @@ -84,6 +85,7 @@ def test_batch_mode(executor): assert result.exit_code == 0 assert 'count(*)\n3\n\na\nabc\n' in result.output + @dbtest def test_batch_mode_table(executor): run(executor, '''create table test(a text)''') @@ -112,6 +114,7 @@ def test_batch_mode_table(executor): assert result.exit_code == 0 assert expected in result.output + @dbtest def test_batch_mode_csv(executor): run(executor, '''create table test(a text, b text)''') @@ -127,6 +130,7 @@ def test_batch_mode_csv(executor): assert result.exit_code == 0 assert expected in result.output + def test_query_starts_with(executor): query = 'USE test;' assert query_starts_with(query, ('use', )) is True @@ -134,10 +138,12 @@ def test_query_starts_with(executor): query = 'DROP DATABASE test;' assert query_starts_with(query, ('use', )) is False + def test_query_starts_with_comment(executor): query = '# comment\nUSE test;' assert query_starts_with(query, ('use', )) is True + def test_queries_start_with(executor): sql = ( '# comment\n' @@ -148,6 +154,7 @@ def test_queries_start_with(executor): assert queries_start_with(sql, ('use', 'drop')) is True assert queries_start_with(sql, ('delete', 'update')) is False + def test_is_destructive(executor): sql = ( 'use test;\n' @@ -156,6 +163,7 @@ def test_is_destructive(executor): ) assert is_destructive(sql) is True + def test_confirm_destructive_query_notty(executor): stdin = click.get_text_stream('stdin') assert stdin.isatty() is False @@ -163,6 +171,7 @@ def test_confirm_destructive_query_notty(executor): sql = 'drop database foo;' assert confirm_destructive_query(sql) is None + def test_thanks_picker_utf8(): project_root = os.path.dirname(PACKAGE_ROOT) author_file = os.path.join(project_root, 'AUTHORS') diff --git a/test/test_naive_completion.py b/test/test_naive_completion.py index 57d738bca..3282c7ed1 100644 --- a/test/test_naive_completion.py +++ b/test/test_naive_completion.py @@ -3,16 +3,19 @@ from prompt_toolkit.completion import Completion from prompt_toolkit.document import Document + @pytest.fixture def completer(): import mycli.sqlcompleter as sqlcompleter return sqlcompleter.SQLCompleter(smart_completion=False) + @pytest.fixture def complete_event(): from mock import Mock return Mock() + def test_empty_string_completion(completer, complete_event): text = '' position = 0 @@ -21,6 +24,7 @@ def test_empty_string_completion(completer, complete_event): complete_event)) assert result == set(map(Completion, completer.all_completions)) + def test_select_keyword_completion(completer, complete_event): text = 'SEL' position = len('SEL') @@ -29,6 +33,7 @@ def test_select_keyword_completion(completer, complete_event): complete_event)) assert result == set([Completion(text='SELECT', start_position=-3)]) + def test_function_name_completion(completer, complete_event): text = 'SELECT MA' position = len('SELECT MA') @@ -39,6 +44,7 @@ def test_function_name_completion(completer, complete_event): Completion(text='MAX', start_position=-2), Completion(text='MASTER', start_position=-2)]) + def test_column_name_completion(completer, complete_event): text = 'SELECT FROM users' position = len('SELECT ') diff --git a/test/test_parseutils.py b/test/test_parseutils.py index e512632c1..e45cab78b 100644 --- a/test/test_parseutils.py +++ b/test/test_parseutils.py @@ -6,50 +6,62 @@ def test_empty_string(): tables = extract_tables('') assert tables == [] + def test_simple_select_single_table(): tables = extract_tables('select * from abc') assert tables == [(None, 'abc', None)] + def test_simple_select_single_table_schema_qualified(): tables = extract_tables('select * from abc.def') assert tables == [('abc', 'def', None)] + def test_simple_select_multiple_tables(): tables = extract_tables('select * from abc, def') assert sorted(tables) == [(None, 'abc', None), (None, 'def', None)] + def test_simple_select_multiple_tables_schema_qualified(): tables = extract_tables('select * from abc.def, ghi.jkl') assert sorted(tables) == [('abc', 'def', None), ('ghi', 'jkl', None)] + def test_simple_select_with_cols_single_table(): tables = extract_tables('select a,b from abc') assert tables == [(None, 'abc', None)] + def test_simple_select_with_cols_single_table_schema_qualified(): tables = extract_tables('select a,b from abc.def') assert tables == [('abc', 'def', None)] + def test_simple_select_with_cols_multiple_tables(): tables = extract_tables('select a,b from abc, def') assert sorted(tables) == [(None, 'abc', None), (None, 'def', None)] + def test_simple_select_with_cols_multiple_tables_with_schema(): tables = extract_tables('select a,b from abc.def, def.ghi') assert sorted(tables) == [('abc', 'def', None), ('def', 'ghi', None)] + def test_select_with_hanging_comma_single_table(): tables = extract_tables('select a, from abc') assert tables == [(None, 'abc', None)] + def test_select_with_hanging_comma_multiple_tables(): tables = extract_tables('select a, from abc, def') assert sorted(tables) == [(None, 'abc', None), (None, 'def', None)] + def test_select_with_hanging_period_multiple_tables(): tables = extract_tables('SELECT t1. FROM tabl1 t1, tabl2 t2') assert sorted(tables) == [(None, 'tabl1', 't1'), (None, 'tabl2', 't2')] + def test_simple_insert_single_table(): tables = extract_tables('insert into abc (id, name) values (1, "def")') @@ -57,27 +69,34 @@ def test_simple_insert_single_table(): # assert tables == [(None, 'abc', None)] assert tables == [(None, 'abc', 'abc')] + @pytest.mark.xfail def test_simple_insert_single_table_schema_qualified(): tables = extract_tables('insert into abc.def (id, name) values (1, "def")') assert tables == [('abc', 'def', None)] + def test_simple_update_table(): tables = extract_tables('update abc set id = 1') assert tables == [(None, 'abc', None)] + def test_simple_update_table_with_schema(): tables = extract_tables('update abc.def set id = 1') assert tables == [('abc', 'def', None)] + def test_join_table(): tables = extract_tables('SELECT * FROM abc a JOIN def d ON a.id = d.num') assert sorted(tables) == [(None, 'abc', 'a'), (None, 'def', 'd')] + def test_join_table_schema_qualified(): - tables = extract_tables('SELECT * FROM abc.def x JOIN ghi.jkl y ON x.id = y.num') + tables = extract_tables( + 'SELECT * FROM abc.def x JOIN ghi.jkl y ON x.id = y.num') assert tables == [('abc', 'def', 'x'), ('ghi', 'jkl', 'y')] + def test_join_as_table(): tables = extract_tables('SELECT * FROM my_table AS m WHERE m.a > 5') assert tables == [(None, 'my_table', 'm')] diff --git a/test/test_smart_completion_public_schema_only.py b/test/test_smart_completion_public_schema_only.py index e99567a07..6cdc9c307 100644 --- a/test/test_smart_completion_public_schema_only.py +++ b/test/test_smart_completion_public_schema_only.py @@ -5,11 +5,12 @@ from prompt_toolkit.document import Document metadata = { - 'users': ['id', 'email', 'first_name', 'last_name'], - 'orders': ['id', 'ordered_date', 'status'], - 'select': ['id', 'insert', 'ABC'], - 'réveillé': ['id', 'insert', 'ABC'] - } + 'users': ['id', 'email', 'first_name', 'last_name'], + 'orders': ['id', 'ordered_date', 'status'], + 'select': ['id', 'insert', 'ABC'], + 'réveillé': ['id', 'insert', 'ABC'] +} + @pytest.fixture def completer(): @@ -30,11 +31,13 @@ def completer(): return comp + @pytest.fixture def complete_event(): from mock import Mock return Mock() + def test_empty_string_completion(completer, complete_event): text = '' position = 0 @@ -44,6 +47,7 @@ def test_empty_string_completion(completer, complete_event): complete_event)) assert set(map(Completion, completer.keywords)) == result + def test_select_keyword_completion(completer, complete_event): text = 'SEL' position = len('SEL') @@ -73,12 +77,14 @@ def test_function_name_completion(completer, complete_event): Completion(text='MASTER', start_position=-2), ]) + def test_suggested_column_names(completer, complete_event): - """ - Suggest column and function names when selecting from table + """Suggest column and function names when selecting from table. + :param completer: :param complete_event: :return: + """ text = 'SELECT from users' position = len('SELECT ') @@ -94,13 +100,15 @@ def test_suggested_column_names(completer, complete_event): list(map(Completion, completer.functions)) + list(map(Completion, completer.keywords))) + def test_suggested_column_names_in_function(completer, complete_event): - """ - Suggest column and function names when selecting multiple - columns from table + """Suggest column and function names when selecting multiple columns from + table. + :param completer: :param complete_event: :return: + """ text = 'SELECT MAX( from users' position = len('SELECT MAX(') @@ -114,12 +122,14 @@ def test_suggested_column_names_in_function(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_column_names_with_table_dot(completer, complete_event): - """ - Suggest column names on table name and dot + """Suggest column names on table name and dot. + :param completer: :param complete_event: :return: + """ text = 'SELECT users. from users' position = len('SELECT users.') @@ -133,12 +143,14 @@ def test_suggested_column_names_with_table_dot(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_column_names_with_alias(completer, complete_event): - """ - Suggest column names on table alias and dot + """Suggest column names on table alias and dot. + :param completer: :param complete_event: :return: + """ text = 'SELECT u. from users u' position = len('SELECT u.') @@ -152,13 +164,15 @@ def test_suggested_column_names_with_alias(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_multiple_column_names(completer, complete_event): - """ - Suggest column and function names when selecting multiple - columns from table + """Suggest column and function names when selecting multiple columns from + table. + :param completer: :param complete_event: :return: + """ text = 'SELECT id, from users u' position = len('SELECT id, ') @@ -174,13 +188,15 @@ def test_suggested_multiple_column_names(completer, complete_event): list(map(Completion, completer.functions)) + list(map(Completion, completer.keywords))) + def test_suggested_multiple_column_names_with_alias(completer, complete_event): - """ - Suggest column names on table alias and dot - when selecting multiple columns from table + """Suggest column names on table alias and dot when selecting multiple + columns from table. + :param completer: :param complete_event: :return: + """ text = 'SELECT u.id, u. from users u' position = len('SELECT u.id, u.') @@ -194,13 +210,15 @@ def test_suggested_multiple_column_names_with_alias(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_multiple_column_names_with_dot(completer, complete_event): - """ - Suggest column names on table names and dot - when selecting multiple columns from table + """Suggest column names on table names and dot when selecting multiple + columns from table. + :param completer: :param complete_event: :return: + """ text = 'SELECT users.id, users. from users u' position = len('SELECT users.id, users.') @@ -214,6 +232,7 @@ def test_suggested_multiple_column_names_with_dot(completer, complete_event): Completion(text='first_name', start_position=0), Completion(text='last_name', start_position=0)]) + def test_suggested_aliases_after_on(completer, complete_event): text = 'SELECT u.name, o.id FROM users u JOIN orders o ON ' position = len('SELECT u.name, o.id FROM users u JOIN orders o ON ') @@ -224,9 +243,11 @@ def test_suggested_aliases_after_on(completer, complete_event): Completion(text='u', start_position=0), Completion(text='o', start_position=0)]) + def test_suggested_aliases_after_on_right_side(completer, complete_event): text = 'SELECT u.name, o.id FROM users u JOIN orders o ON o.user_id = ' - position = len('SELECT u.name, o.id FROM users u JOIN orders o ON o.user_id = ') + position = len( + 'SELECT u.name, o.id FROM users u JOIN orders o ON o.user_id = ') result = set(completer.get_completions( Document(text=text, cursor_position=position), complete_event)) @@ -234,6 +255,7 @@ def test_suggested_aliases_after_on_right_side(completer, complete_event): Completion(text='u', start_position=0), Completion(text='o', start_position=0)]) + def test_suggested_tables_after_on(completer, complete_event): text = 'SELECT users.name, orders.id FROM users JOIN orders ON ' position = len('SELECT users.name, orders.id FROM users JOIN orders ON ') @@ -244,9 +266,11 @@ def test_suggested_tables_after_on(completer, complete_event): Completion(text='users', start_position=0), Completion(text='orders', start_position=0)]) + def test_suggested_tables_after_on_right_side(completer, complete_event): text = 'SELECT users.name, orders.id FROM users JOIN orders ON orders.user_id = ' - position = len('SELECT users.name, orders.id FROM users JOIN orders ON orders.user_id = ') + position = len( + 'SELECT users.name, orders.id FROM users JOIN orders ON orders.user_id = ') result = set(completer.get_completions( Document(text=text, cursor_position=position), complete_event)) @@ -254,6 +278,7 @@ def test_suggested_tables_after_on_right_side(completer, complete_event): Completion(text='users', start_position=0), Completion(text='orders', start_position=0)]) + def test_table_names_after_from(completer, complete_event): text = 'SELECT * FROM ' position = len('SELECT * FROM ') @@ -265,7 +290,8 @@ def test_table_names_after_from(completer, complete_event): Completion(text='orders', start_position=0), Completion(text='`réveillé`', start_position=0), Completion(text='`select`', start_position=0), - ]) + ]) + def test_auto_escaped_col_names(completer, complete_event): text = 'SELECT from `select`' @@ -281,6 +307,7 @@ def test_auto_escaped_col_names(completer, complete_event): list(map(Completion, completer.functions)) + list(map(Completion, completer.keywords))) + def test_un_escaped_table_names(completer, complete_event): text = 'SELECT from réveillé' position = len('SELECT ') diff --git a/test/test_special_iocommands.py b/test/test_special_iocommands.py index 2ed2dbad8..708c7fd77 100644 --- a/test/test_special_iocommands.py +++ b/test/test_special_iocommands.py @@ -23,18 +23,21 @@ def test_set_get_pager(): mycli.packages.special.disable_pager() assert not mycli.packages.special.is_pager_enabled() + def test_set_get_timing(): mycli.packages.special.set_timing_enabled(True) assert mycli.packages.special.is_timing_enabled() mycli.packages.special.set_timing_enabled(False) assert not mycli.packages.special.is_timing_enabled() + def test_set_get_expanded_output(): mycli.packages.special.set_expanded_output(True) assert mycli.packages.special.is_expanded_output() mycli.packages.special.set_expanded_output(False) assert not mycli.packages.special.is_expanded_output() + def test_editor_command(): assert mycli.packages.special.editor_command(r'hello\e') assert mycli.packages.special.editor_command(r'\ehello') @@ -45,14 +48,15 @@ def test_editor_command(): os.environ['EDITOR'] = 'true' mycli.packages.special.open_external_editor(r'select 1') == "select 1" + def test_tee_command(): - mycli.packages.special.write_tee(u"hello world") # write without file set + mycli.packages.special.write_tee(u"hello world") # write without file set with tempfile.NamedTemporaryFile() as f: - mycli.packages.special.execute(None, u"tee "+f.name) + mycli.packages.special.execute(None, u"tee " + f.name) mycli.packages.special.write_tee(u"hello world") assert f.read() == b"hello world\n" - mycli.packages.special.execute(None, u"tee -o "+f.name) + mycli.packages.special.execute(None, u"tee -o " + f.name) mycli.packages.special.write_tee(u"hello world") f.seek(0) assert f.read() == b"hello world\n" @@ -62,6 +66,7 @@ def test_tee_command(): f.seek(0) assert f.read() == b"hello world\n" + def test_tee_command_error(): with pytest.raises(TypeError): mycli.packages.special.execute(None, 'tee') @@ -71,8 +76,10 @@ def test_tee_command_error(): os.chmod(f.name, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) mycli.packages.special.execute(None, 'tee {}'.format(f.name)) + def test_favorite_query(): with utils.db_connection().cursor() as cur: query = u'select "✔"' mycli.packages.special.execute(cur, u'\\fs check {0}'.format(query)) - assert next(mycli.packages.special.execute(cur, u'\\f check'))[0] == "> " + query + assert next(mycli.packages.special.execute( + cur, u'\\f check'))[0] == "> " + query diff --git a/test/test_sqlexecute.py b/test/test_sqlexecute.py index 157b62341..aebfa1a66 100644 --- a/test/test_sqlexecute.py +++ b/test/test_sqlexecute.py @@ -20,6 +20,7 @@ def test_conn(executor): +-----+ 1 row in set""") + @dbtest def test_bools(executor): run(executor, '''create table test(a boolean)''') @@ -33,6 +34,7 @@ def test_bools(executor): +---+ 1 row in set""") + @dbtest def test_binary(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') @@ -46,6 +48,7 @@ def test_binary(executor): +----------------------------------------------------------------------------------------------+ 1 row in set""") + @dbtest def test_binary_expanded(executor): run(executor, '''create table bt(geom linestring NOT NULL)''') @@ -57,6 +60,7 @@ def test_binary_expanded(executor): 1 row in set""") + @dbtest def test_table_and_columns_query(executor): run(executor, "create table a(x text, y text)") @@ -64,25 +68,29 @@ def test_table_and_columns_query(executor): assert set(executor.tables()) == set([('a',), ('b',)]) assert set(executor.table_columns()) == set( - [('a', 'x'), ('a', 'y'), ('b', 'z')]) + [('a', 'x'), ('a', 'y'), ('b', 'z')]) + @dbtest def test_database_list(executor): databases = executor.databases() assert '_test_db' in databases + @dbtest def test_invalid_syntax(executor): with pytest.raises(pymysql.ProgrammingError) as excinfo: run(executor, 'invalid syntax!') assert 'You have an error in your SQL syntax;' in str(excinfo.value) + @dbtest def test_invalid_column_name(executor): with pytest.raises(pymysql.InternalError) as excinfo: run(executor, 'select invalid command') assert "Unknown column 'invalid' in 'field list'" in str(excinfo.value) + @dbtest def test_unicode_support_in_output(executor): run(executor, "create table unicodechars(t text)") @@ -91,6 +99,7 @@ def test_unicode_support_in_output(executor): # See issue #24, this raises an exception without proper handling assert u'é' in run(executor, u"select * from unicodechars", join=True) + @dbtest def test_expanded_output(executor): run(executor, '''create table test(a text)''') @@ -112,19 +121,23 @@ def test_expanded_output(executor): assert results in expected_results + @dbtest def test_multiple_queries_same_line(executor): result = run(executor, "select 'foo'; select 'bar'") - assert len(result) == 4 # 2 for the results and 2 more for status messages. + # 2 for the results and 2 more for status messages. + assert len(result) == 4 assert "foo" in result[0] assert "bar" in result[2] + @dbtest def test_multiple_queries_same_line_syntaxerror(executor): with pytest.raises(pymysql.ProgrammingError) as excinfo: run(executor, "select 'foo'; invalid syntax") assert 'You have an error in your SQL syntax;' in str(excinfo.value) + @dbtest def test_favorite_query(executor): set_expanded_output(False) @@ -147,6 +160,7 @@ def test_favorite_query(executor): results = run(executor, "\\fd test-a") assert results == ['test-a: Deleted'] + @dbtest def test_favorite_query_multiple_statement(executor): set_expanded_output(False) @@ -176,6 +190,7 @@ def test_favorite_query_multiple_statement(executor): results = run(executor, "\\fd test-ad") assert results == ['test-ad: Deleted'] + @dbtest def test_favorite_query_expanded_output(executor): set_expanded_output(False) @@ -206,6 +221,7 @@ def test_favorite_query_expanded_output(executor): results = run(executor, "\\fd test-ae") assert results == ['test-ae: Deleted'] + @dbtest def test_special_command(executor): results = run(executor, '\\?') @@ -213,6 +229,7 @@ def test_special_command(executor): assert len(results) == 1 assert expected_line in results[0] + @dbtest def test_cd_command_without_a_folder_name(executor): results = run(executor, 'system cd') @@ -220,6 +237,7 @@ def test_cd_command_without_a_folder_name(executor): assert len(results) == 1 assert expected_line in results[0] + @dbtest def test_system_command_not_found(executor): results = run(executor, 'system xyz') @@ -227,6 +245,7 @@ def test_system_command_not_found(executor): expected_line = 'OSError:' assert expected_line in results[0] + @dbtest def test_system_command_output(executor): test_file_path = os.path.join(os.path.abspath('.'), 'test', 'test.txt') @@ -235,16 +254,19 @@ def test_system_command_output(executor): expected_line = u'mycli rocks!\n' assert expected_line == results[0] + @dbtest def test_cd_command_current_dir(executor): test_path = os.path.join(os.path.abspath('.'), 'test') results = run(executor, 'system cd {0}'.format(test_path)) assert os.getcwd() == test_path + @dbtest def test_unicode_support(executor): assert u'日本語' in run(executor, u"SELECT '日本語' AS japanese;", join=True) + @dbtest def test_favorite_query_multiline_statement(executor): set_expanded_output(False) @@ -274,6 +296,7 @@ def test_favorite_query_multiline_statement(executor): results = run(executor, "\\fd test-ad") assert results == ['test-ad: Deleted'] + @dbtest def test_timestamp_null(executor): run(executor, '''create table ts_null(a timestamp)''') @@ -287,6 +310,7 @@ def test_timestamp_null(executor): +---------------------+ 1 row in set""") + @dbtest def test_datetime_null(executor): run(executor, '''create table dt_null(a datetime)''') @@ -300,6 +324,7 @@ def test_datetime_null(executor): +---------------------+ 1 row in set""") + @dbtest def test_date_null(executor): run(executor, '''create table date_null(a date)''') @@ -313,6 +338,7 @@ def test_date_null(executor): +------------+ 1 row in set""") + @dbtest def test_time_null(executor): run(executor, '''create table time_null(a time)''') diff --git a/test/utils.py b/test/utils.py index b29e5e072..0d6b6a991 100644 --- a/test/utils.py +++ b/test/utils.py @@ -11,13 +11,15 @@ PORT = getenv('PYTEST_PORT', 3306) CHARSET = getenv('PYTEST_CHARSET', 'utf8') + def db_connection(dbname=None): conn = pymysql.connect(user=USER, host=HOST, port=PORT, database=dbname, password=PASSWORD, - charset=CHARSET, - local_infile=False) + charset=CHARSET, + local_infile=False) conn.autocommit = True return conn + try: db_connection() CAN_CONNECT_TO_DB = True @@ -28,6 +30,7 @@ def db_connection(dbname=None): not CAN_CONNECT_TO_DB, reason="Need a mysql instance at localhost accessible by user 'root'") + def create_db(dbname): with db_connection().cursor() as cur: try: @@ -36,8 +39,9 @@ def create_db(dbname): except: pass + def run(executor, sql, join=False): - " Return string output for the sql to be run " + """Return string output for the sql to be run.""" result = [] # TODO: this needs to go away. `run()` should not test formatted output. @@ -51,6 +55,7 @@ def run(executor, sql, join=False): result = '\n'.join(result) return result + def set_expanded_output(is_expanded): - """ Pass-through for the tests """ + """Pass-through for the tests.""" return special.set_expanded_output(is_expanded) From 072f343989c33352536ee52293e58ad0cf0da60c Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Fri, 21 Apr 2017 20:48:46 +0200 Subject: [PATCH 0409/1025] Add AUTHORS.rst and SPONSORS.rst with links to the text files --- AUTHORS.rst | 3 +++ SPONSORS.rst | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 AUTHORS.rst create mode 100644 SPONSORS.rst diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 000000000..995327f4b --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,3 @@ +Check out our `AUTHORS`_. + +.. _AUTHORS: mycli/AUTHORS diff --git a/SPONSORS.rst b/SPONSORS.rst new file mode 100644 index 000000000..173555c30 --- /dev/null +++ b/SPONSORS.rst @@ -0,0 +1,3 @@ +Check out our `SPONSORS`_. + +.. _SPONSORS: mycli/SPONSORS From 393ee3185423a3c2b8c21acb6965c1b758629cc3 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 22 Apr 2017 12:16:20 -0500 Subject: [PATCH 0410/1025] Simplify author/sponsor file location code. --- mycli/main.py | 5 ++--- tests/test_main.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index b24adeadf..df3f789c3 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -439,9 +439,8 @@ def run_cli(self): if self.smart_completion: self.refresh_completions() - project_root = os.path.join(os.path.dirname(PACKAGE_ROOT), 'mycli') - author_file = os.path.join(project_root, 'AUTHORS') - sponsor_file = os.path.join(project_root, 'SPONSORS') + author_file = os.path.join(PACKAGE_ROOT, 'AUTHORS') + sponsor_file = os.path.join(PACKAGE_ROOT, 'SPONSORS') key_binding_manager = mycli_bindings() diff --git a/tests/test_main.py b/tests/test_main.py index 56c0cb267..791fd5dae 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -164,9 +164,8 @@ def test_confirm_destructive_query_notty(executor): assert confirm_destructive_query(sql) is None def test_thanks_picker_utf8(): - project_root = os.path.join(os.path.dirname(PACKAGE_ROOT), 'mycli') - author_file = os.path.join(project_root, 'AUTHORS') - sponsor_file = os.path.join(project_root, 'SPONSORS') + author_file = os.path.join(PACKAGE_ROOT, 'AUTHORS') + sponsor_file = os.path.join(PACKAGE_ROOT, 'SPONSORS') name = thanks_picker((author_file, sponsor_file)) assert isinstance(name, text_type) From d9490a7c7f080039eff0f8f53a0a59027846495d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 22 Apr 2017 14:03:25 -0500 Subject: [PATCH 0411/1025] Move cryptography changelog item to appropriate section. --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index dfc4c7270..c1643e0d3 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,7 @@ Internal Changes: * Rename tests/ to test/. (Thanks: [Dick Marinus]). * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). +* Switch from pycryptodome to cryptography (Thanks: [Thomas Roten]). 1.10.0: ======= @@ -37,7 +38,6 @@ Internal Changes: * Test mycli using pexpect/python-behave (Thanks: [Dick Marinus]). * Run pep8 checks in travis (Thanks: [Irina Truong]). * Remove temporary hack for sqlparse (Thanks: [Dick Marinus]). -* Switch from pycryptodome to cryptography (Thanks: [Thomas Roten]). 1.9.0: ====== From 13df01d379a4469291d57155e2d8a714b2af9ad7 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 25 Apr 2017 07:38:23 +0200 Subject: [PATCH 0412/1025] behave pager wrapper --- changelog.md | 1 + test/features/environment.py | 4 +++- test/features/steps/basic_commands.py | 2 ++ test/features/steps/crud_database.py | 8 ++++++-- test/features/steps/crud_table.py | 22 +++++++++++++++------- test/features/steps/iocommands.py | 2 +- test/features/steps/named_queries.py | 4 ++-- test/features/steps/specials.py | 6 +++++- test/features/steps/wrappers.py | 5 +++++ test/features/wrappager.py | 16 ++++++++++++++++ 10 files changed, 56 insertions(+), 14 deletions(-) create mode 100755 test/features/wrappager.py diff --git a/changelog.md b/changelog.md index b7cc8c3a3..961eb54c6 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,7 @@ Internal Changes: * Rename tests/ to test/. (Thanks: [Dick Marinus]). * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). +* Add pager wrapper for behave tests (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/features/environment.py b/test/features/environment.py index e79e5740c..376fce723 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -12,7 +12,6 @@ def before_all(context): """Set env parameters.""" os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" - os.environ['PAGER'] = 'cat' os.environ['EDITOR'] = 'ex' os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../.coveragerc" @@ -43,7 +42,10 @@ def before_all(context): 'dbname': db_name, 'dbname_tmp': db_name_full + '_tmp', 'vi': vi, + 'pager_boundary': '---boundary---', } + os.environ['PAGER'] = "{0} {1} {2}".format( + sys.executable, "test/features/wrappager.py", context.conf['pager_boundary']) context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], context.conf['pass'], diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index 109472b8a..845fea17e 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -59,3 +59,5 @@ def step_send_help(context): """ context.cli.sendline('\\?') + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) diff --git a/test/features/steps/crud_database.py b/test/features/steps/crud_database.py index d7b8eeb28..a7616dad3 100644 --- a/test/features/steps/crud_database.py +++ b/test/features/steps/crud_database.py @@ -73,20 +73,24 @@ def step_see_help(context): @then('we see database created') def step_see_db_created(context): """Wait to see create database output.""" - wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see database dropped') def step_see_db_dropped(context): """Wait to see drop database output.""" - wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see database connected') def step_see_db_connected(context): """Wait to see drop database output.""" + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) wrappers.expect_exact( context, 'You are now connected to database "', timeout=2) wrappers.expect_exact(context, '"', timeout=2) wrappers.expect_exact(context, ' as user "{0}"\r\n'.format( context.conf['user']), timeout=2) + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) diff --git a/test/features/steps/crud_table.py b/test/features/steps/crud_table.py index 34301c890..f6e9f044e 100644 --- a/test/features/steps/crud_table.py +++ b/test/features/steps/crud_table.py @@ -9,6 +9,7 @@ import wrappers from behave import when, then +from textwrap import dedent @when('we create table') @@ -56,35 +57,42 @@ def step_drop_table(context): @then('we see table created') def step_see_table_created(context): """Wait to see create table output.""" - wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 0 rows affected\r\n', timeout=2) @then('we see record inserted') def step_see_record_inserted(context): """Wait to see insert output.""" - wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see record updated') def step_see_record_updated(context): """Wait to see update output.""" - wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see data selected') def step_see_data_selected(context): """Wait to see select output.""" - wrappers.expect_exact( - context, '+-----+\r\n| x |\r\n+-----+\r\n| yyy |\r\n+-----+\r\n1 row in set\r\n', timeout=1) + wrappers.expect_pager( + context, dedent("""\ + +-----+\r + | x |\r + +-----+\r + | yyy |\r + +-----+\r + 1 row in set\r + """), timeout=1) @then('we see record deleted') def step_see_data_deleted(context): """Wait to see delete output.""" - wrappers.expect_exact(context, 'Query OK, 1 row affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 1 row affected\r\n', timeout=2) @then('we see table dropped') def step_see_table_dropped(context): """Wait to see drop output.""" - wrappers.expect_exact(context, 'Query OK, 0 rows affected\r\n', timeout=2) + wrappers.expect_pager(context, 'Query OK, 0 rows affected\r\n', timeout=2) diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 73068fac2..83cda6bc7 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -23,7 +23,7 @@ def step_edit_type_sql(context): context.cli.sendline('i') context.cli.sendline('select * from abc') context.cli.sendline('.') - wrappers.expect_exact(context, ':', timeout=2) + wrappers.expect_exact(context, '\r\n:', timeout=2) @when('we exit the editor') diff --git a/test/features/steps/named_queries.py b/test/features/steps/named_queries.py index 60115c5cd..40bf5bc72 100644 --- a/test/features/steps/named_queries.py +++ b/test/features/steps/named_queries.py @@ -32,7 +32,7 @@ def step_delete_named_query(context): @then('we see the named query saved') def step_see_named_query_saved(context): """Wait to see query saved.""" - wrappers.expect_exact(context, 'Saved.', timeout=1) + wrappers.expect_pager(context, 'Saved.\r\n', timeout=1) @then('we see the named query executed') @@ -45,4 +45,4 @@ def step_see_named_query_executed(context): @then('we see the named query deleted') def step_see_named_query_deleted(context): """Wait to see query deleted.""" - wrappers.expect_exact(context, 'foo: Deleted', timeout=1) + wrappers.expect_pager(context, 'foo: Deleted\r\n', timeout=1) diff --git a/test/features/steps/specials.py b/test/features/steps/specials.py index c0a3c0feb..f7715b9ed 100644 --- a/test/features/steps/specials.py +++ b/test/features/steps/specials.py @@ -21,4 +21,8 @@ def step_refresh_completions(context): def step_see_refresh_started(context): """Wait to see refresh output.""" wrappers.expect_exact( - context, 'Auto-completion refresh started in the background', timeout=2) + context, context.conf['pager_boundary'] + '\r\n', timeout=5) + wrappers.expect_exact( + context, 'Auto-completion refresh started in the background.\r\n', timeout=2) + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) diff --git a/test/features/steps/wrappers.py b/test/features/steps/wrappers.py index aea742033..e8d9204ab 100644 --- a/test/features/steps/wrappers.py +++ b/test/features/steps/wrappers.py @@ -14,3 +14,8 @@ def expect_exact(context, expected, timeout): raise Exception('Expected:\n---\n{0!r}\n---\n\nActual:\n---\n{1!r}\n---'.format( expected, actual)) + + +def expect_pager(context, expected, timeout): + expect_exact(context, "{0}\r\n{1}{0}\r\n".format( + context.conf['pager_boundary'], expected), timeout=timeout) diff --git a/test/features/wrappager.py b/test/features/wrappager.py new file mode 100755 index 000000000..51d490956 --- /dev/null +++ b/test/features/wrappager.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +import sys + + +def wrappager(boundary): + print(boundary) + while 1: + buf = sys.stdin.read(2048) + if not buf: + break + sys.stdout.write(buf) + print(boundary) + + +if __name__ == "__main__": + wrappager(sys.argv[1]) From 5074cccdbbb492d5416f6e7b540aae86f70332da Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Wed, 26 Apr 2017 17:21:16 -0700 Subject: [PATCH 0413/1025] Fail on first error in travis script. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 021be1c17..de590fc05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,12 +11,14 @@ install: - pip install git+https://github.com/hayd/pep8radius.git script: + - set -e - coverage run --source mycli -m py.test - cd test - behave - cd .. # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) + - set +e after_success: - coverage combine From 724883308f60eb5ddba31d901bc9b437967a1f6e Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Fri, 28 Apr 2017 21:30:31 +0200 Subject: [PATCH 0414/1025] test using behave the source command --- changelog.md | 1 + test/features/basic_commands.feature | 6 ++++++ test/features/steps/basic_commands.py | 11 +++++++++++ 3 files changed, 18 insertions(+) diff --git a/changelog.md b/changelog.md index 961eb54c6..1df05d693 100644 --- a/changelog.md +++ b/changelog.md @@ -7,6 +7,7 @@ Internal Changes: * Rename tests/ to test/. (Thanks: [Dick Marinus]). * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). * Add pager wrapper for behave tests (Thanks: [Dick Marinus]). +* Behave test source command (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/features/basic_commands.feature b/test/features/basic_commands.feature index 227fe769b..025b58502 100644 --- a/test/features/basic_commands.feature +++ b/test/features/basic_commands.feature @@ -12,6 +12,12 @@ Feature: run the cli, and we send "\?" command then we see help output + Scenario: run source command + When we run dbcli + and we wait for prompt + and we send source command + then we see help output + Scenario: run the cli and exit When we run dbcli and we wait for prompt diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index 845fea17e..37f1e88a5 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -8,6 +8,7 @@ from __future__ import unicode_literals import pexpect +import tempfile from behave import when import wrappers @@ -61,3 +62,13 @@ def step_send_help(context): context.cli.sendline('\\?') wrappers.expect_exact( context, context.conf['pager_boundary'] + '\r\n', timeout=5) + + +@when(u'we send source command') +def step_send_source_command(context): + with tempfile.NamedTemporaryFile() as f: + f.write(b'\?') + f.flush() + context.cli.sendline('\. {0}'.format(f.name)) + wrappers.expect_exact( + context, context.conf['pager_boundary'] + '\r\n', timeout=5) From b2b84bc8af9dd17baff9125ca67bddf6a50bd189 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 07:51:33 +0200 Subject: [PATCH 0415/1025] behave fix clean up In an earlier commit I've changed the current working directory and the removal of a temporary file didn't take that into account. --- changelog.md | 1 + test/features/steps/iocommands.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 1df05d693..8db8c1a16 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,7 @@ Internal Changes: * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). * Add pager wrapper for behave tests (Thanks: [Dick Marinus]). * Behave test source command (Thanks: [Dick Marinus]). +* Behave fix clean up. (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 83cda6bc7..9a713a961 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -9,10 +9,12 @@ @when('we start external editor providing a file name') def step_edit_file(context): """Edit file with external editor.""" - context.editor_file_name = 'test_file_{0}.sql'.format(context.conf['vi']) + context.editor_file_name = '../test_file_{0}.sql'.format( + context.conf['vi']) if os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) - context.cli.sendline('\e {0}'.format(context.editor_file_name)) + context.cli.sendline('\e {0}'.format( + os.path.basename(context.editor_file_name))) wrappers.expect_exact( context, 'Entering Ex mode. Type "visual" to go to Normal mode.', timeout=2) wrappers.expect_exact(context, '\r\n:', timeout=2) From e5b889ba39675de10da9e094ede4a552744c0c9d Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 07:56:04 +0200 Subject: [PATCH 0416/1025] test using behave the tee command --- changelog.md | 1 + test/features/iocommands.feature | 11 +++++++++ test/features/steps/iocommands.py | 39 +++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/changelog.md b/changelog.md index 1df05d693..d031714f6 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,7 @@ Internal Changes: * Move AUTHORS and SPONSORS to mycli directory. (Thanks: [Terje Røsten] []). * Add pager wrapper for behave tests (Thanks: [Dick Marinus]). * Behave test source command (Thanks: [Dick Marinus]). +* Test using behave the tee command (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/features/iocommands.feature b/test/features/iocommands.feature index d043dc2ea..4bcdf6e65 100644 --- a/test/features/iocommands.feature +++ b/test/features/iocommands.feature @@ -8,3 +8,14 @@ Feature: I/O commands and we exit the editor then we see dbcli prompt and we see the sql in prompt + + Scenario: tee output from query + When we run dbcli + and we wait for prompt + and we tee output + and we wait for prompt + and we query "select 123456" + and we wait for prompt + and we notee output + and we wait for prompt + then we see 123456 in tee output diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 83cda6bc7..c8293a10c 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -4,6 +4,7 @@ import wrappers from behave import when, then +from textwrap import dedent @when('we start external editor providing a file name') @@ -41,3 +42,41 @@ def step_edit_done_sql(context): # Cleanup the edited file. if context.editor_file_name and os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) + + +@when(u'we tee output') +def step_tee_ouptut(context): + context.tee_file_name = '../tee_file_{0}.sql'.format(context.conf['vi']) + if os.path.exists(context.tee_file_name): + os.remove(context.tee_file_name) + context.cli.sendline('tee {0}'.format( + os.path.basename(context.tee_file_name))) + wrappers.expect_pager(context, "\r\n", timeout=5) + + +@when(u'we query "select 123456"') +def step_query_select_123456(context): + context.cli.sendline('select 123456') + wrappers.expect_pager(context, dedent("""\ + +--------+\r + | 123456 |\r + +--------+\r + | 123456 |\r + +--------+\r + 1 row in set\r + """), timeout=5) + + +@when(u'we notee output') +def step_notee_output(context): + context.cli.sendline('notee') + wrappers.expect_pager(context, "\r\n", timeout=5) + + +@then(u'we see 123456 in tee output') +def step_see_123456_in_ouput(context): + with open(context.tee_file_name) as f: + assert '123456' in f.read() + if os.path.exists(context.tee_file_name): + os.remove(context.tee_file_name) + context.atprompt = True From 8c6e73842131658fd11ab9907cd59a31c7c8384d Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Thu, 27 Apr 2017 20:39:10 +0200 Subject: [PATCH 0417/1025] behave quit mycli nicely Before this patch mycli is killed by expect and it the coverage data cannot be written. --- changelog.md | 1 + test/features/environment.py | 20 ++++++++++++++++++-- test/features/steps/basic_commands.py | 4 +++- test/features/steps/crud_database.py | 5 ++++- test/features/steps/iocommands.py | 2 +- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 18c332713..c47dd255b 100644 --- a/changelog.md +++ b/changelog.md @@ -11,6 +11,7 @@ Internal Changes: * Behave test source command (Thanks: [Dick Marinus]). * Test using behave the tee command (Thanks: [Dick Marinus]). * Behave fix clean up. (Thanks: [Dick Marinus]). +* Behave quit mycli nicely (Thanks: [Dick Marinus]) 1.10.0: ======= diff --git a/test/features/environment.py b/test/features/environment.py index 376fce723..a0456b99f 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -6,6 +6,7 @@ import sys import db_utils as dbutils import fixture_utils as fixutils +import pexpect def before_all(context): @@ -68,12 +69,27 @@ def after_all(context): # os.environ[k] = v +def before_step(context, _): + context.atprompt = False + + def after_scenario(context, _): """Cleans up after each test complete.""" if hasattr(context, 'cli') and not context.exit_sent: - # Terminate nicely. - context.cli.terminate() + # Quit nicely. + if not context.atprompt: + user = context.conf['user'] + host = context.conf['host'] + dbname = context.currentdb + context.cli.expect_exact( + 'mysql {0}@{1}:{2}> '.format( + user, host, dbname + ), + timeout=5 + ) + context.cli.sendcontrol('d') + context.cli.expect_exact(pexpect.EOF, timeout=5) # TODO: uncomment to debug a failure # def after_step(context, step): diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index 37f1e88a5..df97ee0e7 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -33,6 +33,7 @@ def step_run_cli(context): cmd = ' '.join(cmd_parts) context.cli = pexpect.spawnu(cmd, cwd='..') context.exit_sent = False + context.currentdb = context.conf['dbname'] @when('we wait for prompt') @@ -40,9 +41,10 @@ def step_wait_prompt(context): """Make sure prompt is displayed.""" user = context.conf['user'] host = context.conf['host'] - dbname = context.conf['dbname'] + dbname = context.currentdb wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( user, host, dbname), timeout=5) + context.atprompt = True @when('we send "ctrl + d"') diff --git a/test/features/steps/crud_database.py b/test/features/steps/crud_database.py index a7616dad3..af8580e16 100644 --- a/test/features/steps/crud_database.py +++ b/test/features/steps/crud_database.py @@ -39,12 +39,14 @@ def step_db_drop(context): def step_db_connect_test(context): """Send connect to database.""" db_name = context.conf['dbname'] + context.currentdb = db_name context.cli.sendline('use {0}'.format(db_name)) @when('we connect to dbserver') def step_db_connect_dbserver(context): """Send connect to database.""" + context.currentdb = 'mysql' context.cli.sendline('use mysql') @@ -59,9 +61,10 @@ def step_see_prompt(context): """Wait to see the prompt.""" user = context.conf['user'] host = context.conf['host'] - dbname = context.conf['dbname'] + dbname = context.currentdb wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( user, host, dbname), timeout=5) + context.atprompt = True @then('we see help output') diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 6641ce952..5de640ffb 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -40,7 +40,7 @@ def step_edit_done_sql(context): for match in 'select * from abc'.split(' '): wrappers.expect_exact(context, match, timeout=1) # Cleanup the command line. - context.cli.sendcontrol('u') + context.cli.sendcontrol('c') # Cleanup the edited file. if context.editor_file_name and os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) From 50bf54e04b00909f790457dd791d1c2b91a7d219 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 16:43:59 -0500 Subject: [PATCH 0418/1025] Do not add time from multiple queries together. --- mycli/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index df3f789c3..788e123eb 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -535,8 +535,7 @@ def one_iteration(document=None): max_width) output.extend(formatted) - end = time() - total += end - start + total = time() - start mutating = mutating or is_mutating(status) except KeyboardInterrupt: # get last connection id From 82b65bbc08918037d4d4286eb3fc23927b55266b Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 16:45:28 -0500 Subject: [PATCH 0419/1025] Add timing bugfix to changelog. --- changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.md b/changelog.md index 18c332713..11465d0cd 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Bug Fixes: +---------- + +* Fixed incorrect timekeeping when running queries from a file. (Thanks: [Thomas Roten]). + Internal Changes: ----------------- From 88d470d2fda97dfae835ba7ac0ba99d997c3e907 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 20:41:52 -0500 Subject: [PATCH 0420/1025] Add configurable reserved lines for completion menu. --- changelog.md | 5 +++++ mycli/main.py | 25 ++++++++++++++----------- mycli/myclirc | 3 +++ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/changelog.md b/changelog.md index 18c332713..8422296d1 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Features: +--------- + +* Add option to control how much space is reserved for the completion menu. (Thanks: [Thomas Roten]). + Internal Changes: ----------------- diff --git a/mycli/main.py b/mycli/main.py index df3f789c3..043db60bf 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -113,6 +113,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') + self.min_num_menu_lines = c['main'].as_int('min_num_menu_lines') c_dest_warning = c['main'].as_bool('destructive_warning') self.destructive_warning = c_dest_warning if warn is None else warn self.login_path_as_host = c['main'].as_bool('login_path_as_host') @@ -602,17 +603,19 @@ def one_iteration(document=None): get_toolbar_tokens = create_toolbar_tokens_func(self.completion_refresher.is_refreshing) - layout = create_prompt_layout(lexer=MyCliLexer, - multiline=True, - get_prompt_tokens=prompt_tokens, - get_continuation_tokens=get_continuation_tokens, - get_bottom_toolbar_tokens=get_toolbar_tokens, - display_completions_in_columns=self.wider_completion_menu, - extra_input_processors=[ - ConditionalProcessor( - processor=HighlightMatchingBracketProcessor(chars='[](){}'), - filter=HasFocus(DEFAULT_BUFFER) & ~IsDone()), - ]) + layout = create_prompt_layout( + lexer=MyCliLexer, + multiline=True, + get_prompt_tokens=prompt_tokens, + get_continuation_tokens=get_continuation_tokens, + get_bottom_toolbar_tokens=get_toolbar_tokens, + display_completions_in_columns=self.wider_completion_menu, + extra_input_processors=[ConditionalProcessor( + processor=HighlightMatchingBracketProcessor(chars='[](){}'), + filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() + )], + reserve_space_for_menu=self.min_num_menu_lines + ) with self._completer_lock: buf = CLIBuffer(always_multiline=self.multi_line, completer=self.completer, history=FileHistory(os.path.expanduser(os.environ.get('MYCLI_HISTFILE', '~/.mycli-history'))), diff --git a/mycli/myclirc b/mycli/myclirc index 01a114265..2f45778b5 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -51,6 +51,9 @@ key_bindings = emacs # Enabling this option will show the suggestions in a wider menu. Thus more items are suggested. wider_completion_menu = False +# Number of lines to reserve for the completion menu. +min_num_menu_lines = 8 + # MySQL prompt # \t - Product type (Percona, MySQL, Mariadb) # \u - Username From b3a6839c69e2a37094ef894002e038ef0afc3df5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 21:09:15 -0500 Subject: [PATCH 0421/1025] Add current vi mode to toolbar. --- changelog.md | 5 +++++ mycli/clitoolbar.py | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 11465d0cd..dab38caab 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,11 @@ TBD === +Features: +--------- + +* Display current vi mode in toolbar. (Thanks: [Thomas Roten]). + Bug Fixes: ---------- diff --git a/mycli/clitoolbar.py b/mycli/clitoolbar.py index b62d8edbe..79f2a8a75 100644 --- a/mycli/clitoolbar.py +++ b/mycli/clitoolbar.py @@ -1,5 +1,6 @@ from pygments.token import Token from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode +from prompt_toolkit.key_binding.vi_state import InputMode def create_toolbar_tokens_func(get_is_refreshing): """ @@ -26,7 +27,10 @@ def get_toolbar_tokens(cli): ' (Semi-colon [;] will end the line)')) if cli.editing_mode == EditingMode.VI: - result.append((token.On, '[F4] Vi-mode')) + result.append(( + token.On, + '[F4] Vi-mode ({})'.format(_get_vi_mode(cli)) + )) else: result.append((token.On, '[F4] Emacs-mode')) @@ -35,3 +39,13 @@ def get_toolbar_tokens(cli): return result return get_toolbar_tokens + + +def _get_vi_mode(cli): + """Get the current vi mode for display.""" + return { + InputMode.INSERT: 'I', + InputMode.NAVIGATION: 'N', + InputMode.REPLACE: 'R', + InputMode.INSERT_MULTIPLE: 'M' + }[cli.vi_state.input_mode] From f75ea9bec59121346c20b298828071413dd5e54c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 22:22:17 -0500 Subject: [PATCH 0422/1025] Make reserved space automatically calculate. --- changelog.md | 2 +- mycli/main.py | 10 ++++++++-- mycli/myclirc | 3 --- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index de0b12159..3819ce1a6 100644 --- a/changelog.md +++ b/changelog.md @@ -4,7 +4,7 @@ TBD Features: --------- -* Add option to control how much space is reserved for the completion menu. (Thanks: [Thomas Roten]). +* Handle reserved space for completion menu better in small windows. (Thanks: [Thomas Roten]). Bug Fixes: ---------- diff --git a/mycli/main.py b/mycli/main.py index 5f36cc193..eff09340a 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -113,7 +113,6 @@ def __init__(self, sqlexecute=None, prompt=None, self.less_chatty = c['main'].as_bool('less_chatty') self.cli_style = c['colors'] self.wider_completion_menu = c['main'].as_bool('wider_completion_menu') - self.min_num_menu_lines = c['main'].as_int('min_num_menu_lines') c_dest_warning = c['main'].as_bool('destructive_warning') self.destructive_warning = c_dest_warning if warn is None else warn self.login_path_as_host = c['main'].as_bool('login_path_as_host') @@ -613,7 +612,7 @@ def one_iteration(document=None): processor=HighlightMatchingBracketProcessor(chars='[](){}'), filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() )], - reserve_space_for_menu=self.min_num_menu_lines + reserve_space_for_menu=self.get_reserved_space() ) with self._completer_lock: buf = CLIBuffer(always_multiline=self.multi_line, completer=self.completer, @@ -750,6 +749,13 @@ def format_output(self, title, cur, headers, status, expanded=False, return output + def get_reserved_space(self): + """Get the number of lines to reserve for the completion menu.""" + reserved_space_ratio = .2 + max_reserved_space = 8 + _, height = click.get_terminal_size() + return min(int(height * reserved_space_ratio), max_reserved_space) + @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') diff --git a/mycli/myclirc b/mycli/myclirc index 2f45778b5..01a114265 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -51,9 +51,6 @@ key_bindings = emacs # Enabling this option will show the suggestions in a wider menu. Thus more items are suggested. wider_completion_menu = False -# Number of lines to reserve for the completion menu. -min_num_menu_lines = 8 - # MySQL prompt # \t - Product type (Percona, MySQL, Mariadb) # \u - Username From 535bac41984342e39723ff32419aa3a0efa5375d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:03:34 -0500 Subject: [PATCH 0423/1025] Increase reserved space ratio. --- mycli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index eff09340a..5df93408a 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -751,10 +751,10 @@ def format_output(self, title, cur, headers, status, expanded=False, def get_reserved_space(self): """Get the number of lines to reserve for the completion menu.""" - reserved_space_ratio = .2 + reserved_space_ratio = .45 max_reserved_space = 8 _, height = click.get_terminal_size() - return min(int(height * reserved_space_ratio), max_reserved_space) + return min(round(height * reserved_space_ratio), max_reserved_space) @click.command() From 7ec1c242d5b6a34c8c5e9a88a046b035de43cde2 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:32:41 -0500 Subject: [PATCH 0424/1025] Add CLI Helpers dependency. --- mycli/main.py | 26 +++++++++++++------------- mycli/myclirc | 2 +- setup.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 5df93408a..c3a4aef0d 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -13,6 +13,7 @@ from random import choice from io import open +from cli_helpers.tabular_output import TabularOutputFormatter import click import sqlparse from prompt_toolkit import CommandLineInterface, Application, AbortAction @@ -37,7 +38,6 @@ from .config import (write_default_config, get_mylogin_cnf_path, open_mylogin_cnf, read_config_files, str_to_bool) from .key_bindings import mycli_bindings -from .output_formatter import output_formatter from .encodingutils import utf8tounicode from .lexer import MyCliLexer from .__init__ import __version__ @@ -107,7 +107,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.multi_line = c['main'].as_bool('multi_line') self.key_bindings = c['main']['key_bindings'] special.set_timing_enabled(c['main'].as_bool('timing')) - self.formatter = output_formatter.OutputFormatter( + self.formatter = TabularOutputFormatter( format_name=c['main']['table_format']) self.syntax_style = c['main']['syntax_style'] self.less_chatty = c['main'].as_bool('less_chatty') @@ -149,7 +149,7 @@ def __init__(self, sqlexecute=None, prompt=None, self.smart_completion = c['main'].as_bool('smart_completion') self.completer = SQLCompleter( self.smart_completion, - supported_formats=self.formatter.supported_formats()) + supported_formats=self.formatter.supported_formats) self._completer_lock = threading.Lock() # Register custom special commands. @@ -185,13 +185,13 @@ def register_special_commands(self): def change_table_format(self, arg, **_): try: - self.formatter.set_format_name(arg) + self.formatter.format_name = arg yield (None, None, None, 'Changed table type to {}'.format(arg)) except ValueError: msg = 'Table type {} not yet implemented. Allowed types:'.format( arg) - for table_type in self.formatter.supported_formats(): + for table_type in self.formatter.supported_formats: msg += "\n\t{}".format(table_type) yield (None, None, None, msg) @@ -673,7 +673,7 @@ def refresh_completions(self, reset=False): self.completion_refresher.refresh( self.sqlexecute, self._on_completions_refreshed, {'smart_completion': self.smart_completion, - 'supported_formats': self.formatter.supported_formats()}) + 'supported_formats': self.formatter.supported_formats}) return [(None, None, None, 'Auto-completion refresh started in the background.')] @@ -726,7 +726,7 @@ def run_query(self, query, new_line=True): def format_output(self, title, cur, headers, status, expanded=False, max_width=None): - expanded = expanded or self.formatter.get_format_name() == 'expanded' + expanded = expanded or self.formatter.format_name == 'vertical' output = [] if title: # Only print the title if it's not None. @@ -735,12 +735,12 @@ def format_output(self, title, cur, headers, status, expanded=False, if cur: rows = list(cur) formatted = self.formatter.format_output( - rows, headers, format_name='expanded' if expanded else None) + rows, headers, format_name='vertical' if expanded else None) if (not expanded and max_width and rows and content_exceeds_width(rows[0], max_width) and headers): formatted = self.formatter.format_output( - rows, headers, format_name='expanded') + rows, headers, format_name='vertical') output.append(formatted) @@ -855,9 +855,9 @@ def cli(database, user, host, port, socket, password, dbname, if execute: try: if csv: - mycli.formatter.set_format_name('csv') + mycli.formatter.format_name = 'csv' elif not table: - mycli.formatter.set_format_name('tsv') + mycli.formatter.format_name = 'tsv' mycli.run_query(execute) exit(0) @@ -883,10 +883,10 @@ def cli(database, user, host, port, socket, password, dbname, new_line = True if csv: - mycli.formatter.set_format_name('csv') + mycli.formatter.format_name = 'csv' new_line = False elif not table: - mycli.formatter.set_format_name('tsv') + mycli.formatter.format_name = 'tsv' mycli.run_query(stdin_text, new_line=new_line) exit(0) diff --git a/mycli/myclirc b/mycli/myclirc index 01a114265..ab57a2ed9 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -32,7 +32,7 @@ timing = True # Table format. Possible values: ascii, double, github, # psql, plain, simple, grid, fancy_grid, pipe, orgtbl, rst, mediawiki, html, -# latex, latex_booktabs, textile, moinmoin, jira, expanded, tsv, csv. +# latex, latex_booktabs, textile, moinmoin, jira, vertical, tsv, csv. # Recommended: ascii table_format = ascii diff --git a/setup.py b/setup.py index 27b761d90..5140b7e9b 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ 'sqlparse>=0.2.2,<0.3.0', 'configobj >= 5.0.5', 'cryptography >= 1.0.0', - 'terminaltables >= 3.0.0', + 'cli_helpers >= 0.1.0', ] setup( From 2baed47e9397600c413af235163f3ec9318eadee Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:33:02 -0500 Subject: [PATCH 0425/1025] Remove output formatter code and tests. --- mycli/output_formatter/__init__.py | 0 .../delimited_output_adapter.py | 28 - mycli/output_formatter/expanded.py | 34 - mycli/output_formatter/output_formatter.py | 95 -- mycli/output_formatter/preprocessors.py | 87 - mycli/output_formatter/tabulate_adapter.py | 22 - .../terminaltables_adapter.py | 25 - mycli/packages/tabulate.py | 1432 ----------------- test/test_expanded.py | 19 - test/test_output_formatter.py | 160 -- test/test_tabulate.py | 17 - 11 files changed, 1919 deletions(-) delete mode 100644 mycli/output_formatter/__init__.py delete mode 100644 mycli/output_formatter/delimited_output_adapter.py delete mode 100644 mycli/output_formatter/expanded.py delete mode 100644 mycli/output_formatter/output_formatter.py delete mode 100644 mycli/output_formatter/preprocessors.py delete mode 100644 mycli/output_formatter/tabulate_adapter.py delete mode 100644 mycli/output_formatter/terminaltables_adapter.py delete mode 100644 mycli/packages/tabulate.py delete mode 100644 test/test_expanded.py delete mode 100644 test/test_output_formatter.py delete mode 100644 test/test_tabulate.py diff --git a/mycli/output_formatter/__init__.py b/mycli/output_formatter/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/mycli/output_formatter/delimited_output_adapter.py b/mycli/output_formatter/delimited_output_adapter.py deleted file mode 100644 index a01a28433..000000000 --- a/mycli/output_formatter/delimited_output_adapter.py +++ /dev/null @@ -1,28 +0,0 @@ -import contextlib -import csv -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -from .preprocessors import override_missing_value, bytes_to_string - -supported_formats = ('csv', 'tsv') -preprocessors = (override_missing_value, bytes_to_string) - - -def adapter(data, headers, table_format='csv', **_): - """Wrap CSV formatting inside a standard function for OutputFormatter.""" - with contextlib.closing(StringIO()) as content: - if table_format == 'csv': - writer = csv.writer(content, delimiter=',') - elif table_format == 'tsv': - writer = csv.writer(content, delimiter='\t') - else: - raise ValueError('Invalid table_format specified.') - - writer.writerow(headers) - for row in data: - writer.writerow(row) - - return content.getvalue() diff --git a/mycli/output_formatter/expanded.py b/mycli/output_formatter/expanded.py deleted file mode 100644 index f77c1ee38..000000000 --- a/mycli/output_formatter/expanded.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Format data into a vertical, expanded table layout.""" - -from __future__ import unicode_literals - - -def get_separator(num): - """Get a row separator for row *num*.""" - return "{divider}[ {n}. row ]{divider}\n".format( - divider='*' * 27, n=num + 1) - - -def format_row(headers, row): - """Format a row.""" - formatted_row = [' | '.join(field) for field in zip(headers, row)] - return '\n'.join(formatted_row) - - -def expanded_table(rows, headers, **_): - """Format *rows* and *headers* as an expanded table. - - The values in *rows* and *headers* must be strings. - - """ - header_len = max([len(x) for x in headers]) - padded_headers = [x.ljust(header_len) for x in headers] - formatted_rows = [format_row(padded_headers, row) for row in rows] - - output = [] - for i, result in enumerate(formatted_rows): - output.append(get_separator(i)) - output.append(result) - output.append('\n') - - return ''.join(output) diff --git a/mycli/output_formatter/output_formatter.py b/mycli/output_formatter/output_formatter.py deleted file mode 100644 index 61e3c8d52..000000000 --- a/mycli/output_formatter/output_formatter.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- -"""A generic output formatter interface.""" - -from __future__ import unicode_literals -from collections import namedtuple - -from .expanded import expanded_table -from .preprocessors import (override_missing_value, convert_to_string) - -from . import delimited_output_adapter -from . import tabulate_adapter -from . import terminaltables_adapter - -MISSING_VALUE = '' - -OutputFormatHandler = namedtuple( - 'OutputFormatHandler', - 'format_name preprocessors formatter formatter_args') - - -class OutputFormatter(object): - """A class with a standard interface for various formatting libraries.""" - - _output_formats = {} - - def __init__(self, format_name=None): - """Set the default *format_name*.""" - self._format_name = format_name - - def set_format_name(self, format_name): - """Set the OutputFormatter's default format.""" - if format_name in self.supported_formats(): - self._format_name = format_name - else: - raise ValueError('unrecognized format_name: {}'.format( - format_name)) - - def get_format_name(self): - """Get the OutputFormatter's default format.""" - return self._format_name - - def supported_formats(self): - """Return the supported output format names.""" - return tuple(self._output_formats.keys()) - - @classmethod - def register_new_formatter(cls, format_name, handler, preprocessors=(), - kwargs={}): - """Register a new formatter to format the output.""" - cls._output_formats[format_name] = OutputFormatHandler( - format_name, preprocessors, handler, kwargs) - - def format_output(self, data, headers, format_name=None, **kwargs): - """Format the headers and data using a specific formatter. - - *format_name* must be a formatter available in `supported_formats()`. - - All keyword arguments are passed to the specified formatter. - - """ - format_name = format_name or self._format_name - if format_name not in self.supported_formats(): - raise ValueError('unrecognized format: {}'.format(format_name)) - - (_, preprocessors, formatter, - fkwargs) = self._output_formats[format_name] - fkwargs.update(kwargs) - if preprocessors: - for f in preprocessors: - data, headers = f(data, headers, **fkwargs) - return formatter(data, headers, **fkwargs) - - -OutputFormatter.register_new_formatter('expanded', expanded_table, - (override_missing_value, - convert_to_string), - {'missing_value': MISSING_VALUE}) - -for delimiter_format in delimited_output_adapter.supported_formats: - OutputFormatter.register_new_formatter( - delimiter_format, delimited_output_adapter.adapter, - delimited_output_adapter.preprocessors, - {'table_format': delimiter_format, 'missing_value': MISSING_VALUE}) - -for tabulate_format in tabulate_adapter.supported_formats: - OutputFormatter.register_new_formatter( - tabulate_format, tabulate_adapter.adapter, - tabulate_adapter.preprocessors, - {'table_format': tabulate_format, 'missing_value': MISSING_VALUE}) - -for terminaltables_format in terminaltables_adapter.supported_formats: - OutputFormatter.register_new_formatter( - terminaltables_format, terminaltables_adapter.adapter, - terminaltables_adapter.preprocessors, - {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE}) diff --git a/mycli/output_formatter/preprocessors.py b/mycli/output_formatter/preprocessors.py deleted file mode 100644 index 6f2e459c0..000000000 --- a/mycli/output_formatter/preprocessors.py +++ /dev/null @@ -1,87 +0,0 @@ -from decimal import Decimal - -from mycli import encodingutils - - -def to_string(value): - """Convert *value* to a string.""" - if isinstance(value, encodingutils.binary_type): - return encodingutils.bytes_to_string(value) - else: - return encodingutils.text_type(value) - - -def convert_to_string(data, headers, **_): - """Convert all *data* and *headers* to strings.""" - return ([[to_string(v) for v in row] for row in data], - [to_string(h) for h in headers]) - - -def override_missing_value(data, headers, missing_value='', **_): - """Override missing values in the data with *missing_value*.""" - return ([[missing_value if v is None else v for v in row] for row in data], - headers) - - -def bytes_to_string(data, headers, **_): - """Convert all *data* and *headers* bytes to strings.""" - return ([[encodingutils.bytes_to_string(v) for v in row] for row in data], - [encodingutils.bytes_to_string(h) for h in headers]) - - -def intlen(value): - """Find (character) length. - - >>> intlen('11.1') - 2 - >>> intlen('11') - 2 - >>> intlen('1.1') - 1 - - """ - pos = value.find('.') - if pos < 0: - pos = len(value) - return pos - - -def align_decimals(data, headers, **_): - """Align decimals to decimal point.""" - pointpos = len(headers) * [0] - for row in data: - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - pointpos[i] = max(intlen(v), pointpos[i]) - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - if isinstance(v, Decimal): - v = encodingutils.text_type(v) - result.append((pointpos[i] - intlen(v)) * " " + v) - else: - result.append(v) - results.append(result) - return results, headers - - -def quote_whitespaces(data, headers, quotestyle="'", **_): - """Quote leading/trailing whitespace.""" - quote = len(headers) * [False] - for row in data: - for i, v in enumerate(row): - v = encodingutils.text_type(v) - if v.startswith(' ') or v.endswith(' '): - quote[i] = True - - results = [] - for row in data: - result = [] - for i, v in enumerate(row): - quotation = quotestyle if quote[i] else '' - result.append('{quotestyle}{value}{quotestyle}'.format( - quotestyle=quotation, value=v)) - results.append(result) - return results, headers diff --git a/mycli/output_formatter/tabulate_adapter.py b/mycli/output_formatter/tabulate_adapter.py deleted file mode 100644 index b89dcc0bd..000000000 --- a/mycli/output_formatter/tabulate_adapter.py +++ /dev/null @@ -1,22 +0,0 @@ -from mycli.packages import tabulate -from .preprocessors import bytes_to_string, align_decimals - -tabulate.PRESERVE_WHITESPACE = True - -supported_markup_formats = ('mediawiki', 'html', 'latex', 'latex_booktabs', - 'textile', 'moinmoin', 'jira') -supported_table_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe', - 'orgtbl', 'psql', 'rst') -supported_formats = supported_markup_formats + supported_table_formats - -preprocessors = (bytes_to_string, align_decimals) - - -def adapter(data, headers, table_format=None, missing_value='', **_): - """Wrap tabulate inside a standard function for OutputFormatter.""" - kwargs = {'tablefmt': table_format, 'missingval': missing_value, - 'disable_numparse': True} - if table_format in supported_markup_formats: - kwargs.update(numalign=None, stralign=None) - - return tabulate.tabulate(data, headers, **kwargs) diff --git a/mycli/output_formatter/terminaltables_adapter.py b/mycli/output_formatter/terminaltables_adapter.py deleted file mode 100644 index a8f50f985..000000000 --- a/mycli/output_formatter/terminaltables_adapter.py +++ /dev/null @@ -1,25 +0,0 @@ -import terminaltables - -from .preprocessors import (bytes_to_string, align_decimals, - override_missing_value) - -supported_formats = ('ascii', 'double', 'github') -preprocessors = (bytes_to_string, override_missing_value, align_decimals) - - -def adapter(data, headers, table_format=None, **_): - """Wrap terminaltables inside a standard function for OutputFormatter.""" - - table_format_handler = { - 'ascii': terminaltables.AsciiTable, - 'double': terminaltables.DoubleTable, - 'github': terminaltables.GithubFlavoredMarkdownTable, - } - - try: - table = table_format_handler[table_format] - except KeyError: - raise ValueError('unrecognized table format: {}'.format(table_format)) - - t = table([headers] + data) - return t.table diff --git a/mycli/packages/tabulate.py b/mycli/packages/tabulate.py deleted file mode 100644 index 1e67cea7d..000000000 --- a/mycli/packages/tabulate.py +++ /dev/null @@ -1,1432 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Pretty-print tabular data.""" - -from __future__ import print_function -from __future__ import unicode_literals -from collections import namedtuple, Iterable -from platform import python_version_tuple -import re - - -if python_version_tuple()[0] < "3": - from itertools import izip_longest - from functools import partial - _none_type = type(None) - _bool_type = bool - _int_type = int - _long_type = long - _float_type = float - _text_type = unicode - _binary_type = str - - def _is_file(f): - return isinstance(f, file) - -else: - from itertools import zip_longest as izip_longest - from functools import reduce, partial - _none_type = type(None) - _bool_type = bool - _int_type = int - _long_type = int - _float_type = float - _text_type = str - _binary_type = bytes - basestring = str - - import io - - def _is_file(f): - return isinstance(f, io.IOBase) - -try: - import wcwidth # optional wide-character (CJK) support -except ImportError: - wcwidth = None - - -__all__ = ["tabulate", "tabulate_formats"] -__version__ = "0.8.0" - - -# minimum extra space in headers -MIN_PADDING = 2 - -PRESERVE_WHITESPACE = False - -_DEFAULT_FLOATFMT = "g" -_DEFAULT_MISSINGVAL = "" - - -# if True, enable wide-character (CJK) support -WIDE_CHARS_MODE = wcwidth is not None - - -Line = namedtuple("Line", ["begin", "hline", "sep", "end"]) - - -DataRow = namedtuple("DataRow", ["begin", "sep", "end"]) - - -# A table structure is suppposed to be: -# -# --- lineabove --------- -# headerrow -# --- linebelowheader --- -# datarow -# --- linebewteenrows --- -# ... (more datarows) ... -# --- linebewteenrows --- -# last datarow -# --- linebelow --------- -# -# TableFormat's line* elements can be -# -# - either None, if the element is not used, -# - or a Line tuple, -# - or a function: [col_widths], [col_alignments] -> string. -# -# TableFormat's *row elements can be -# -# - either None, if the element is not used, -# - or a DataRow tuple, -# - or a function: [cell_values], [col_widths], [col_alignments] -> string. -# -# padding (an integer) is the amount of white space around data values. -# -# with_header_hide: -# -# - either None, to display all table elements unconditionally, -# - or a list of elements not to be displayed if the table has column -# headers. -# -TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader", - "linebetweenrows", "linebelow", - "headerrow", "datarow", - "padding", "with_header_hide"]) - - -def _pipe_segment_with_colons(align, colwidth): - """Return a segment of a horizontal line with optional colons which - indicate column's alignment (as in `pipe` output format).""" - w = colwidth - if align in ["right", "decimal"]: - return ('-' * (w - 1)) + ":" - elif align == "center": - return ":" + ('-' * (w - 2)) + ":" - elif align == "left": - return ":" + ('-' * (w - 1)) - else: - return '-' * w - - -def _pipe_line_with_colons(colwidths, colaligns): - """Return a horizontal line with optional colons to indicate column's - alignment (as in `pipe` output format).""" - segments = [_pipe_segment_with_colons(a, w) for a, w in - zip(colaligns, colwidths)] - return "|" + "|".join(segments) + "|" - - -def _mediawiki_row_with_attrs(separator, cell_values, colwidths, colaligns): - alignment = {"left": '', - "right": 'align="right"| ', - "center": 'align="center"| ', - "decimal": 'align="right"| '} - # hard-coded padding _around_ align attribute and value together - # rather than padding parameter which affects only the value - values_with_attrs = [' ' + alignment.get(a, '') + c + ' ' - for c, a in zip(cell_values, colaligns)] - colsep = separator*2 - return (separator + colsep.join(values_with_attrs)).rstrip() - - -def _textile_row_with_attrs(cell_values, colwidths, colaligns): - cell_values[0] += ' ' - alignment = {"left": "<.", "right": ">.", "center": "=.", "decimal": ">."} - values = (alignment.get(a, '') + v for a, v in zip(colaligns, cell_values)) - return '|' + '|'.join(values) + '|' - - -def _html_begin_table_without_header(colwidths_ignore, colaligns_ignore): - # this table header will be suppressed if there is a header row - return "\n".join(["", ""]) - - -def _html_row_with_attrs(celltag, cell_values, colwidths, colaligns): - alignment = {"left": '', - "right": ' style="text-align: right;"', - "center": ' style="text-align: center;"', - "decimal": ' style="text-align: right;"'} - values_with_attrs = ["<{0}{1}>{2}".format( - celltag, alignment.get(a, ''), c) for c, a in - zip(cell_values, colaligns)] - rowhtml = "" + "".join(values_with_attrs).rstrip() + "" - if celltag == "th": # it's a header row, create a new table header - rowhtml = "\n".join(["
", - "", - rowhtml, - "", - ""]) - return rowhtml - - -def _moin_row_with_attrs(celltag, cell_values, colwidths, colaligns, - header=''): - alignment = {"left": '', - "right": '', - "center": '', - "decimal": ''} - values_with_attrs = ["{0}{1} {2} ".format(celltag, - alignment.get(a, ''), - header + c + header) - for c, a in zip(cell_values, colaligns)] - return "".join(values_with_attrs) + "||" - - -def _latex_line_begin_tabular(colwidths, colaligns, booktabs=False): - alignment = {"left": "l", "right": "r", "center": "c", "decimal": "r"} - tabular_columns_fmt = "".join([alignment.get(a, "l") for a in colaligns]) - return "\n".join(["\\begin{tabular}{" + tabular_columns_fmt + "}", - "\\toprule" if booktabs else "\hline"]) - - -LATEX_ESCAPE_RULES = {r"&": r"\&", r"%": r"\%", r"$": r"\$", r"#": r"\#", - r"_": r"\_", r"^": r"\^{}", r"{": r"\{", r"}": r"\}", - r"~": r"\textasciitilde{}", "\\": r"\textbackslash{}", - r"<": r"\ensuremath{<}", r">": r"\ensuremath{>}"} - - -def _latex_row(cell_values, colwidths, colaligns, escrules=LATEX_ESCAPE_RULES): - def escape_char(c): - return escrules.get(c, c) - escaped_values = ["".join(map(escape_char, cell)) for cell in cell_values] - rowfmt = DataRow("", "&", "\\\\") - return _build_simple_row(escaped_values, rowfmt) - - -def _rst_escape_first_column(rows, headers): - def escape_empty(val): - if isinstance(val, (_text_type, _binary_type)) and val.strip() is "": - return ".." - else: - return val - new_headers = list(headers) - new_rows = [] - if headers: - new_headers[0] = escape_empty(headers[0]) - for row in rows: - new_row = list(row) - if new_row: - new_row[0] = escape_empty(row[0]) - new_rows.append(new_row) - return new_rows, new_headers - - -_table_formats = {"simple": - TableFormat( - lineabove=Line("", "-", " ", ""), - linebelowheader=Line("", "-", " ", ""), - linebetweenrows=None, - linebelow=Line("", "-", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, - with_header_hide=["lineabove", "linebelow"]), - "plain": - TableFormat( - lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None), - "grid": - TableFormat( - lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("+", "=", "+", "+"), - linebetweenrows=Line("+", "-", "+", "+"), - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), - "fancy_grid": - TableFormat( - lineabove=Line("╒", "═", "╤", "╕"), - linebelowheader=Line("╞", "═", "╪", "╡"), - linebetweenrows=Line("├", "─", "┼", "┤"), - linebelow=Line("╘", "═", "╧", "╛"), - headerrow=DataRow("│", "│", "│"), - datarow=DataRow("│", "│", "│"), - padding=1, with_header_hide=None), - "pipe": - TableFormat( - lineabove=_pipe_line_with_colons, - linebelowheader=_pipe_line_with_colons, - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, - with_header_hide=["lineabove"]), - "orgtbl": - TableFormat( - lineabove=None, - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), - "jira": - TableFormat( - lineabove=None, - linebelowheader=None, - linebetweenrows=None, - linebelow=None, - headerrow=DataRow("||", "||", "||"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), - "psql": - TableFormat( - lineabove=Line("+", "-", "+", "+"), - linebelowheader=Line("|", "-", "+", "|"), - linebetweenrows=None, - linebelow=Line("+", "-", "+", "+"), - headerrow=DataRow("|", "|", "|"), - datarow=DataRow("|", "|", "|"), - padding=1, with_header_hide=None), - "rst": - TableFormat( - lineabove=Line("", "=", " ", ""), - linebelowheader=Line("", "=", " ", ""), - linebetweenrows=None, - linebelow=Line("", "=", " ", ""), - headerrow=DataRow("", " ", ""), - datarow=DataRow("", " ", ""), - padding=0, with_header_hide=None), - "mediawiki": - TableFormat(lineabove=Line( - "{| class=\"wikitable\" style=\"text-align: left;\"", - "", "", "\n|+ \n|-"), - linebelowheader=Line("|-", "", "", ""), - linebetweenrows=Line("|-", "", "", ""), - linebelow=Line("|}", "", "", ""), - headerrow=partial(_mediawiki_row_with_attrs, "!"), - datarow=partial(_mediawiki_row_with_attrs, "|"), - padding=0, with_header_hide=None), - "moinmoin": - TableFormat( - lineabove=None, - linebelowheader=None, - linebetweenrows=None, - linebelow=None, - headerrow=partial(_moin_row_with_attrs, "||", - header="'''"), - datarow=partial(_moin_row_with_attrs, "||"), - padding=1, with_header_hide=None), - "html": - TableFormat( - lineabove=_html_begin_table_without_header, - linebelowheader="", - linebetweenrows=None, - linebelow=Line("\n
", "", "", ""), - headerrow=partial(_html_row_with_attrs, "th"), - datarow=partial(_html_row_with_attrs, "td"), - padding=0, with_header_hide=["lineabove"]), - "latex": - TableFormat( - lineabove=_latex_line_begin_tabular, - linebelowheader=Line("\\hline", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None), - "latex_raw": - TableFormat( - lineabove=_latex_line_begin_tabular, - linebelowheader=Line("\\hline", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\hline\n\\end{tabular}", "", "", ""), - headerrow=partial(_latex_row, escrules={}), - datarow=partial(_latex_row, escrules={}), - padding=1, with_header_hide=None), - "latex_booktabs": - TableFormat( - lineabove=partial(_latex_line_begin_tabular, - booktabs=True), - linebelowheader=Line("\\midrule", "", "", ""), - linebetweenrows=None, - linebelow=Line("\\bottomrule\n\\end{tabular}", "", "", - ""), - headerrow=_latex_row, - datarow=_latex_row, - padding=1, with_header_hide=None), - "textile": - TableFormat( - lineabove=None, linebelowheader=None, - linebetweenrows=None, linebelow=None, - headerrow=DataRow("|_. ", "|_.", "|"), - datarow=_textile_row_with_attrs, - padding=1, with_header_hide=None)} - - -tabulate_formats = list(sorted(_table_formats.keys())) - - -# ANSI color codes -_invisible_codes = re.compile(r"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") -_invisible_codes_bytes = re.compile(b"\x1b\[\d+[;\d]*m|\x1b\[\d*\;\d*\;\d*m") - - -def _isconvertible(conv, string): - try: - n = conv(string) - return True - except (ValueError, TypeError): - return False - - -def _isnumber(string): - """ - >>> _isnumber("123.45") - True - >>> _isnumber("123") - True - >>> _isnumber("spam") - False - """ - return _isconvertible(float, string) - - -def _isint(string, inttype=int): - """ - >>> _isint("123") - True - >>> _isint("123.45") - False - """ - return type(string) is inttype or\ - (isinstance(string, _binary_type) or isinstance(string, _text_type))\ - and\ - _isconvertible(inttype, string) - - -def _isbool(string): - """ - >>> _isbool(True) - True - >>> _isbool("False") - True - >>> _isbool(1) - False - """ - return type(string) is _bool_type or\ - (isinstance(string, (_binary_type, _text_type)) and - string in ("True", "False")) - - -def _type(string, has_invisible=True, numparse=True): - """The least generic type (type(None), int, float, str, unicode). - - >>> _type(None) is type(None) - True - >>> _type("foo") is type("") - True - >>> _type("1") is type(1) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - >>> _type('\x1b[31m42\x1b[0m') is type(42) - True - - """ - - if has_invisible and \ - (isinstance(string, _text_type) or isinstance(string, _binary_type)): - string = _strip_invisible(string) - - if string is None: - return _none_type - elif hasattr(string, "isoformat"): # datetime.datetime, date, and time - return _text_type - elif _isbool(string): - return _bool_type - elif _isint(string) and numparse: - return int - elif _isint(string, _long_type) and numparse: - return int - elif _isnumber(string) and numparse: - return float - elif isinstance(string, _binary_type): - return _binary_type - else: - return _text_type - - -def _afterpoint(string): - """Symbols after a decimal point, -1 if the string lacks the decimal point. - - >>> _afterpoint("123.45") - 2 - >>> _afterpoint("1001") - -1 - >>> _afterpoint("eggs") - -1 - >>> _afterpoint("123e45") - 2 - - """ - if _isnumber(string): - if _isint(string): - return -1 - else: - pos = string.rfind(".") - pos = string.lower().rfind("e") if pos < 0 else pos - if pos >= 0: - return len(string) - pos - 1 - else: - return -1 # no point - else: - return -1 # not a number - - -def _padleft(width, s): - """Flush right. - - >>> _padleft(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430' - True - - """ - fmt = "{0:>%ds}" % width - return fmt.format(s) - - -def _padright(width, s): - """Flush left. - - >>> _padright(6, '\u044f\u0439\u0446\u0430') == '\u044f\u0439\u0446\u0430 ' - True - - """ - fmt = "{0:<%ds}" % width - return fmt.format(s) - - -def _padboth(width, s): - """Center string. - - >>> _padboth(6, '\u044f\u0439\u0446\u0430') == ' \u044f\u0439\u0446\u0430 ' - True - - """ - fmt = "{0:^%ds}" % width - return fmt.format(s) - - -def _strip_invisible(s): - "Remove invisible ANSI color codes." - if isinstance(s, _text_type): - return re.sub(_invisible_codes, "", s) - else: # a bytestring - return re.sub(_invisible_codes_bytes, "", s) - - -def _visible_width(s): - """Visible width of a printed string. ANSI color codes are removed. - - >>> _visible_width('\x1b[31mhello\x1b[0m'), _visible_width("world") - (5, 5) - - """ - # optional wide-character support - if wcwidth is not None and WIDE_CHARS_MODE: - len_fn = wcwidth.wcswidth - else: - len_fn = len - if isinstance(s, _text_type) or isinstance(s, _binary_type): - return len_fn(_strip_invisible(s)) - else: - return len_fn(_text_type(s)) - - -def _align_column(strings, alignment, minwidth=0, has_invisible=True): - """[string] -> [padded_string] - - >>> list(map(str,_align_column( - ... ["12.345", "-1234.5", "1.23", "1234.5", "1e+234", "1.0e234"], - ... "decimal"))) - [' 12.345 ', '-1234.5 ', ' 1.23 ', ' 1234.5 ', ' 1e+234 ', ' 1.0e234'] - - >>> list(map(str,_align_column(['123.4', '56.7890'], None))) - ['123.4', '56.7890'] - - """ - if alignment == "right": - if not PRESERVE_WHITESPACE: - strings = [s.strip() for s in strings] - padfn = _padleft - elif alignment == "center": - if not PRESERVE_WHITESPACE: - strings = [s.strip() for s in strings] - padfn = _padboth - elif alignment == "decimal": - if has_invisible: - decimals = [_afterpoint(_strip_invisible(s)) for s in strings] - else: - decimals = [_afterpoint(s) for s in strings] - maxdecimals = max(decimals) - strings = [s + (maxdecimals - decs) * " " - for s, decs in zip(strings, decimals)] - padfn = _padleft - elif not alignment: - return strings - else: - if not PRESERVE_WHITESPACE: - strings = [s.strip() for s in strings] - padfn = _padright - - enable_widechars = wcwidth is not None and WIDE_CHARS_MODE - if has_invisible: - width_fn = _visible_width - elif enable_widechars: # optional wide-character support if available - width_fn = wcwidth.wcswidth - else: - width_fn = len - - s_lens = list(map(len, strings)) - s_widths = list(map(width_fn, strings)) - maxwidth = max(max(s_widths), minwidth) - if not enable_widechars and not has_invisible: - padded_strings = [padfn(maxwidth, s) for s in strings] - else: - # enable wide-character width corrections - visible_widths = [maxwidth - (w - l) for w, l in zip(s_widths, s_lens)] - # wcswidth and _visible_width don't count invisible characters; - # padfn doesn't need to apply another correction - padded_strings = [padfn(w, s) for s, w in zip(strings, visible_widths)] - return padded_strings - - -def _more_generic(type1, type2): - types = {_none_type: 0, _bool_type: 1, int: 2, float: 3, _binary_type: 4, - _text_type: 5} - invtypes = {5: _text_type, 4: _binary_type, 3: float, 2: int, - 1: _bool_type, 0: _none_type} - moregeneric = max(types.get(type1, 5), types.get(type2, 5)) - return invtypes[moregeneric] - - -def _column_type(strings, has_invisible=True, numparse=True): - """The least generic type all column values are convertible to. - - >>> _column_type([True, False]) is _bool_type - True - >>> _column_type(["1", "2"]) is _int_type - True - >>> _column_type(["1", "2.3"]) is _float_type - True - >>> _column_type(["1", "2.3", "four"]) is _text_type - True - >>> _column_type(["four", '\u043f\u044f\u0442\u044c']) is _text_type - True - >>> _column_type([None, "brux"]) is _text_type - True - >>> _column_type([1, 2, None]) is _int_type - True - >>> import datetime as dt - >>> _column_type([dt.datetime(1991,2,19), dt.time(17,35)]) is _text_type - True - - """ - types = [_type(s, has_invisible, numparse) for s in strings] - return reduce(_more_generic, types, _bool_type) - - -def _format(val, valtype, floatfmt, missingval="", has_invisible=True): - """Format a value accoding to its type. - - Unicode is supported: - - >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', - ... '\u0446\u0438\u0444\u0440\u0430'] - >>> tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] - >>> good_result = ('\\u0431\\u0443\\u043a\\u0432\\u0430 ' - ... '\\u0446\\u0438\\u0444\\u0440\\u0430\\n------- ' - ... '-------\\n\\u0430\\u0437 ' - ... '2\\n\\u0431\\u0443\\u043a\\u0438 4') - >>> tabulate(tbl, headers=hrow) == good_result - True - - """ - if val is None: - return missingval - - if valtype in [int, _text_type]: - return "{0}".format(val) - elif valtype is _binary_type: - try: - return _text_type(val, "ascii") - except TypeError: - return _text_type(val) - elif valtype is float: - is_a_colored_number = (has_invisible and - isinstance(val, (_text_type, _binary_type))) - if is_a_colored_number: - raw_val = _strip_invisible(val) - formatted_val = format(float(raw_val), floatfmt) - return val.replace(raw_val, formatted_val) - else: - return format(float(val), floatfmt) - else: - return "{0}".format(val) - - -def _align_header(header, alignment, width, visible_width): - """Pad string header to width chars given known visible_width of the - header.""" - width += len(header) - visible_width - if alignment == "left": - return _padright(width, header) - elif alignment == "center": - return _padboth(width, header) - elif not alignment: - return "{0}".format(header) - else: - return _padleft(width, header) - - -def _prepend_row_index(rows, index): - """Add a left-most index column.""" - if index is None or index is False: - return rows - if len(index) != len(rows): - print('index=', index) - print('rows=', rows) - raise ValueError('index must be as long as the number of data rows') - rows = [[v] + list(row) for v, row in zip(index, rows)] - return rows - - -def _bool(val): - """A wrapper around standard bool() which doesn't throw on NumPy - arrays.""" - try: - return bool(val) - except ValueError: # val is likely to be a numpy array with many elements - return False - - -def _normalize_tabular_data(tabular_data, headers, showindex="default"): - """Transform a supported data type to a list of lists, and a list of - headers. - - Supported tabular data types: - - * list-of-lists or another iterable of iterables - - * list of named tuples (usually used with headers="keys") - - * list of dicts (usually used with headers="keys") - - * list of OrderedDicts (usually used with headers="keys") - - * 2D NumPy arrays - - * NumPy record arrays (usually used with headers="keys") - - * dict of iterables (usually used with headers="keys") - - * pandas.DataFrame (usually used with headers="keys") - - The first row can be used as headers if headers="firstrow", - column indices can be used as headers if headers="keys". - - If showindex="default", show row indices of the pandas.DataFrame. - If showindex="always", show row indices for all types of data. - If showindex="never", don't show row indices for all types of data. - If showindex is an iterable, show its values as row indices. - - """ - - try: - bool(headers) - is_headers2bool_broken = False - except ValueError: # numpy.ndarray, pandas.core.index.Index, ... - is_headers2bool_broken = True - headers = list(headers) - - index = None - if hasattr(tabular_data, "keys") and hasattr(tabular_data, "values"): - # dict-like and pandas.DataFrame? - if hasattr(tabular_data.values, "__call__"): - # likely a conventional dict - keys = tabular_data.keys() - # columns have to be transposed - rows = list(izip_longest(*tabular_data.values())) - elif hasattr(tabular_data, "index"): - # values is a property, has .index => it's likely a - # pandas.DataFrame (pandas 0.11.0) - keys = list(tabular_data) - if tabular_data.index.name is not None: - if isinstance(tabular_data.index.name, list): - keys[:0] = tabular_data.index.name - else: - keys[:0] = [tabular_data.index.name] - # values matrix doesn't need to be transposed - vals = tabular_data.values - # for DataFrames add an index per default - index = list(tabular_data.index) - rows = [list(row) for row in vals] - else: - raise ValueError( - "tabular data doesn't appear to be a dict or a DataFrame") - - if headers == "keys": - headers = list(map(_text_type, keys)) # headers should be strings - - else: # it's a usual an iterable of iterables, or a NumPy array - rows = list(tabular_data) - - if (headers == "keys" and not rows): - # an empty table (issue #81) - headers = [] - elif (headers == "keys" and - hasattr(tabular_data, "dtype") and - getattr(tabular_data.dtype, "names")): - # numpy record array - headers = tabular_data.dtype.names - elif (headers == "keys" - and len(rows) > 0 - and isinstance(rows[0], tuple) - and hasattr(rows[0], "_fields")): - # namedtuple - headers = list(map(_text_type, rows[0]._fields)) - elif (len(rows) > 0 - and isinstance(rows[0], dict)): - # dict or OrderedDict - uniq_keys = set() # implements hashed lookup - keys = [] # storage for set - if headers == "firstrow": - firstdict = rows[0] if len(rows) > 0 else {} - keys.extend(firstdict.keys()) - uniq_keys.update(keys) - rows = rows[1:] - for row in rows: - for k in row.keys(): - # Save unique items in input order - if k not in uniq_keys: - keys.append(k) - uniq_keys.add(k) - if headers == 'keys': - headers = keys - elif isinstance(headers, dict): - # a dict of headers for a list of dicts - headers = [headers.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - elif headers == "firstrow": - if len(rows) > 0: - headers = [firstdict.get(k, k) for k in keys] - headers = list(map(_text_type, headers)) - else: - headers = [] - elif headers: - raise ValueError( - 'headers for a list of dicts is not a dict or a keyword') - rows = [[row.get(k) for k in keys] for row in rows] - - elif (headers == "keys" - and hasattr(tabular_data, "description") - and hasattr(tabular_data, "fetchone") - and hasattr(tabular_data, "rowcount")): - # Python Database API cursor object (PEP 0249) - # print tabulate(cursor, headers='keys') - headers = [column[0] for column in tabular_data.description] - - elif headers == "keys" and len(rows) > 0: - # keys are column indices - headers = list(map(_text_type, range(len(rows[0])))) - - # take headers from the first row if necessary - if headers == "firstrow" and len(rows) > 0: - if index is not None: - headers = [index[0]] + list(rows[0]) - index = index[1:] - else: - headers = rows[0] - headers = list(map(_text_type, headers)) # headers should be strings - rows = rows[1:] - - headers = list(map(_text_type, headers)) - rows = list(map(list, rows)) - - # add or remove an index column - showindex_is_a_str = type(showindex) in [_text_type, _binary_type] - if showindex == "default" and index is not None: - rows = _prepend_row_index(rows, index) - elif isinstance(showindex, Iterable) and not showindex_is_a_str: - rows = _prepend_row_index(rows, list(showindex)) - elif (showindex == "always" or - (_bool(showindex) and not showindex_is_a_str)): - if index is None: - index = list(range(len(rows))) - rows = _prepend_row_index(rows, index) - elif (showindex == "never" or - (not _bool(showindex) and not showindex_is_a_str)): - pass - - # pad with empty headers for initial columns if necessary - if headers and len(rows) > 0: - nhs = len(headers) - ncols = len(rows[0]) - if nhs < ncols: - headers = [""] * (ncols - nhs) + headers - - return rows, headers - - -def tabulate(tabular_data, headers=(), tablefmt="simple", - floatfmt=_DEFAULT_FLOATFMT, numalign="decimal", stralign="left", - missingval=_DEFAULT_MISSINGVAL, showindex="default", - disable_numparse=False): - """Format a fixed width table for pretty printing. - - >>> print(tabulate([[1, 2.34], [-56, "8.999"], ["2", "10001"]])) - --- --------- - 1 2.34 - -56 8.999 - 2 10001 - --- --------- - - The first required argument (`tabular_data`) can be a - list-of-lists (or another iterable of iterables), a list of named - tuples, a dictionary of iterables, an iterable of dictionaries, - a two-dimensional NumPy array, NumPy record array, or a Pandas' - dataframe. - - - Table headers - ------------- - - To print nice column headers, supply the second argument (`headers`): - - - `headers` can be an explicit list of column headers - - if `headers="firstrow"`, then the first row of data is used - - if `headers="keys"`, then dictionary keys or column indices are used - - Otherwise a headerless table is produced. - - If the number of headers is less than the number of columns, they - are supposed to be names of the last columns. This is consistent - with the plain-text format of R and Pandas' dataframes. - - >>> print(tabulate([["sex","age"],["Alice","F",24],["Bob","M",19]], - ... headers="firstrow")) - sex age - ----- ----- ----- - Alice F 24 - Bob M 19 - - By default, pandas.DataFrame data have an additional column called - row index. To add a similar column to all other types of data, - use `showindex="always"` or `showindex=True`. To suppress row indices - for all types of data, pass `showindex="never" or `showindex=False`. - To add a custom row index column, pass `showindex=some_iterable`. - - >>> print(tabulate([["F",24],["M",19]], showindex="always")) - - - -- - 0 F 24 - 1 M 19 - - - -- - - - Column alignment - ---------------- - - `tabulate` tries to detect column types automatically, and aligns - the values properly. By default it aligns decimal points of the - numbers (or flushes integer numbers to the right), and flushes - everything else to the left. Possible column alignments - (`numalign`, `stralign`) are: "right", "center", "left", "decimal" - (only for `numalign`), and None (to disable alignment). - - - Table formats - ------------- - - `floatfmt` is a format specification used for columns which - contain numeric data with a decimal point. This can also be - a list or tuple of format strings, one per column. - - `None` values are replaced with a `missingval` string (like - `floatfmt`, this can also be a list of values for different - columns): - - >>> print(tabulate([["spam", 1, None], - ... ["eggs", 42, 3.14], - ... ["other", None, 2.7]], missingval="?")) - ----- -- ---- - spam 1 ? - eggs 42 3.14 - other ? 2.7 - ----- -- ---- - - Various plain-text table formats (`tablefmt`) are supported: - 'plain', 'simple', 'grid', 'pipe', 'orgtbl', 'rst', 'mediawiki', - 'latex', 'latex_raw' and 'latex_booktabs'. Variable `tabulate_formats` - contains the list of currently supported formats. - - "plain" format doesn't use any pseudographics to draw tables, - it separates columns with a double space: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "plain")) - strings numbers - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="plain")) - spam 41.9999 - eggs 451 - - "simple" format is like Pandoc simple_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "simple")) - strings numbers - --------- --------- - spam 41.9999 - eggs 451 - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="simple")) - ---- -------- - spam 41.9999 - eggs 451 - ---- -------- - - "grid" is similar to tables produced by Emacs table.el package or - Pandoc grid_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "grid")) - +-----------+-----------+ - | strings | numbers | - +===========+===========+ - | spam | 41.9999 | - +-----------+-----------+ - | eggs | 451 | - +-----------+-----------+ - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="grid")) - +------+----------+ - | spam | 41.9999 | - +------+----------+ - | eggs | 451 | - +------+----------+ - - "fancy_grid" draws a grid using box-drawing characters: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "fancy_grid")) - ╒═══════════╤═══════════╕ - │ strings │ numbers │ - ╞═══════════╪═══════════╡ - │ spam │ 41.9999 │ - ├───────────┼───────────┤ - │ eggs │ 451 │ - ╘═══════════╧═══════════╛ - - "pipe" is like tables in PHP Markdown Extra extension or Pandoc - pipe_tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "pipe")) - | strings | numbers | - |:----------|----------:| - | spam | 41.9999 | - | eggs | 451 | - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="pipe")) - |:-----|---------:| - | spam | 41.9999 | - | eggs | 451 | - - "orgtbl" is like tables in Emacs org-mode and orgtbl-mode. They - are slightly different from "pipe" format by not using colons to - define column alignment, and using a "+" sign to indicate line - intersections: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "orgtbl")) - | strings | numbers | - |-----------+-----------| - | spam | 41.9999 | - | eggs | 451 | - - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="orgtbl")) - | spam | 41.9999 | - | eggs | 451 | - - "rst" is like a simple table format from reStructuredText; please - note that reStructuredText accepts also "grid" tables: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... ["strings", "numbers"], "rst")) - ========= ========= - strings numbers - ========= ========= - spam 41.9999 - eggs 451 - ========= ========= - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], tablefmt="rst")) - ==== ======== - spam 41.9999 - eggs 451 - ==== ======== - - "mediawiki" produces a table markup used in Wikipedia and on other - MediaWiki-based sites: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], - ... ["eggs", "451.0"]], headers="firstrow", - ... tablefmt="mediawiki")) - {| class="wikitable" style="text-align: left;" - |+ - |- - ! strings !! align="right"| numbers - |- - | spam || align="right"| 41.9999 - |- - | eggs || align="right"| 451 - |} - - "html" produces HTML markup: - - >>> print(tabulate([["strings", "numbers"], ["spam", 41.9999], - ... ["eggs", "451.0"]], headers="firstrow", - ... tablefmt="html")) - - - - - - - - -
strings numbers
spam 41.9999
eggs 451
- - "latex" produces a tabular environment of LaTeX document markup: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="latex")) - \\begin{tabular}{lr} - \\hline - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\hline - \\end{tabular} - - "latex_raw" is similar to "latex", but doesn't escape special characters, - such as backslash and underscore, so LaTeX commands may embedded into - cells' values: - - >>> print(tabulate([["spam$_9$", 41.9999], ["\\\\emph{eggs}", "451.0"]], - ... tablefmt="latex_raw")) - \\begin{tabular}{lr} - \\hline - spam$_9$ & 41.9999 \\\\ - \\emph{eggs} & 451 \\\\ - \\hline - \\end{tabular} - - "latex_booktabs" produces a tabular environment of LaTeX document markup - using the booktabs.sty package: - - >>> print(tabulate([["spam", 41.9999], ["eggs", "451.0"]], - ... tablefmt="latex_booktabs")) - \\begin{tabular}{lr} - \\toprule - spam & 41.9999 \\\\ - eggs & 451 \\\\ - \\bottomrule - \end{tabular} - - Number parsing - -------------- - By default, anything which can be parsed as a number is a number. - This ensures numbers represented as strings are aligned properly. - This can lead to weird results for particular strings such as - specific git SHAs e.g. "42992e1" will be parsed into the number - 429920 and aligned as such. - - To completely disable number parsing (and alignment), use - `disable_numparse=True`. For more fine grained control, a list column - indices is used to disable number parsing only on those columns - e.g. `disable_numparse=[0, 2]` would disable number parsing only on the - first and third columns. - - """ - if tabular_data is None: - tabular_data = [] - list_of_lists, headers = _normalize_tabular_data( - tabular_data, headers, showindex=showindex) - - # empty values in the first column of RST tables should be escaped - # (issue #82). "" should be escaped as "\\ " or ".." - if tablefmt == 'rst': - list_of_lists, headers = _rst_escape_first_column(list_of_lists, - headers) - - # optimization: look for ANSI control codes once, - # enable smart width functions only if a control code is found - plain_text = '\n'.join(['\t'.join(map(_text_type, headers))] + - ['\t'.join(map(_text_type, row)) - for row in list_of_lists]) - - has_invisible = re.search(_invisible_codes, plain_text) - enable_widechars = wcwidth is not None and WIDE_CHARS_MODE - if has_invisible: - width_fn = _visible_width - elif enable_widechars: # optional wide-character support if available - width_fn = wcwidth.wcswidth - else: - width_fn = len - - # format rows and columns, convert numeric values to strings - cols = list(izip_longest(*list_of_lists)) - numparses = _expand_numparse(disable_numparse, len(cols)) - coltypes = [_column_type(col, numparse=np) for col, np in - zip(cols, numparses)] - if isinstance(floatfmt, basestring): # old version - # just duplicate the string to use in each column - float_formats = len(cols) * [floatfmt] - else: # if floatfmt is list, tuple etc we have one per column - float_formats = list(floatfmt) - if len(float_formats) < len(cols): - float_formats.extend((len(cols) - len(float_formats)) * - [_DEFAULT_FLOATFMT]) - if isinstance(missingval, basestring): - missing_vals = len(cols) * [missingval] - else: - missing_vals = list(missingval) - if len(missing_vals) < len(cols): - missing_vals.extend((len(cols) - len(missing_vals)) * - [_DEFAULT_MISSINGVAL]) - cols = [[_format(v, ct, fl_fmt, miss_v, has_invisible) for v in c] - for c, ct, fl_fmt, miss_v in zip(cols, coltypes, float_formats, - missing_vals)] - - # align columns - aligns = [numalign if ct in [int, float] else stralign for ct in coltypes] - minwidths = [width_fn(h) + MIN_PADDING - for h in headers] if headers else [0] * len(cols) - cols = [_align_column(c, a, minw, has_invisible) - for c, a, minw in zip(cols, aligns, minwidths)] - - if headers: - # align headers and add headers - t_cols = cols or [['']] * len(headers) - t_aligns = aligns or [stralign] * len(headers) - minwidths = [max(minw, width_fn(c[0])) - for minw, c in zip(minwidths, t_cols)] - headers = [_align_header(h, a, minw, width_fn(h)) - for h, a, minw in zip(headers, t_aligns, minwidths)] - rows = list(zip(*cols)) - else: - minwidths = [width_fn(c[0]) for c in cols] - rows = list(zip(*cols)) - - if not isinstance(tablefmt, TableFormat): - tablefmt = _table_formats.get(tablefmt, _table_formats["simple"]) - - return _format_table(tablefmt, headers, rows, minwidths, aligns) - - -def _expand_numparse(disable_numparse, column_count): - """Return a list of bools of length `column_count` which indicates whether - number parsing should be used on each column. - - If `disable_numparse` is a list of indices, each of those indices - are False, and everything else is True. If `disable_numparse` is a - bool, then the returned list is all the same. - - """ - if isinstance(disable_numparse, Iterable): - numparses = [True] * column_count - for index in disable_numparse: - numparses[index] = False - return numparses - else: - return [not disable_numparse] * column_count - - -def _build_simple_row(padded_cells, rowfmt): - "Format row according to DataRow format without padding." - begin, sep, end = rowfmt - return (begin + sep.join(padded_cells) + end).rstrip() - - -def _build_row(padded_cells, colwidths, colaligns, rowfmt): - "Return a string which represents a row of data cells." - if not rowfmt: - return None - if hasattr(rowfmt, "__call__"): - return rowfmt(padded_cells, colwidths, colaligns) - else: - return _build_simple_row(padded_cells, rowfmt) - - -def _build_line(colwidths, colaligns, linefmt): - "Return a string which represents a horizontal line." - if not linefmt: - return None - if hasattr(linefmt, "__call__"): - return linefmt(colwidths, colaligns) - else: - begin, fill, sep, end = linefmt - cells = [fill*w for w in colwidths] - return _build_simple_row(cells, (begin, sep, end)) - - -def _pad_row(cells, padding): - if cells: - pad = " "*padding - padded_cells = [pad + cell + pad for cell in cells] - return padded_cells - else: - return cells - - -def _format_table(fmt, headers, rows, colwidths, colaligns): - """Produce a plain-text representation of the table.""" - lines = [] - hidden = fmt.with_header_hide if (headers and fmt.with_header_hide) else [] - pad = fmt.padding - headerrow = fmt.headerrow - - padded_widths = [(w + 2*pad) for w in colwidths] - padded_headers = _pad_row(headers, pad) - padded_rows = [_pad_row(row, pad) for row in rows] - - if fmt.lineabove and "lineabove" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.lineabove)) - - if padded_headers: - lines.append(_build_row(padded_headers, padded_widths, colaligns, - headerrow)) - if fmt.linebelowheader and "linebelowheader" not in hidden: - lines.append(_build_line(padded_widths, colaligns, - fmt.linebelowheader)) - - if padded_rows and fmt.linebetweenrows and "linebetweenrows" not in hidden: - # initial rows with a line below - for row in padded_rows[:-1]: - lines.append(_build_row(row, padded_widths, colaligns, - fmt.datarow)) - lines.append(_build_line(padded_widths, colaligns, - fmt.linebetweenrows)) - # the last row without a line below - lines.append(_build_row(padded_rows[-1], padded_widths, colaligns, - fmt.datarow)) - else: - for row in padded_rows: - lines.append(_build_row(row, padded_widths, colaligns, - fmt.datarow)) - - if fmt.linebelow and "linebelow" not in hidden: - lines.append(_build_line(padded_widths, colaligns, fmt.linebelow)) - - if headers or rows: - return "\n".join(lines) - else: # a completely empty table - return "" - - -def _main(): - """\ Usage: tabulate [options] [FILE ...] - - Pretty-print tabular data. - See also https://bitbucket.org/astanin/python-tabulate - - FILE a filename of the file with tabular data; - if "-" or missing, read data from stdin. - - Options: - - -h, --help show this message - -1, --header use the first row of data as a table header - -o FILE, --output FILE print table to FILE (default: stdout) - -s REGEXP, --sep REGEXP use a custom column separator (default: whitespace) - -F FPFMT, --float FPFMT floating point number format (default: g) - -f FMT, --format FMT set output table format; supported formats: - plain, simple, grid, fancy_grid, pipe, orgtbl, - rst, mediawiki, html, latex, latex_raw, - latex_booktabs, tsv - (default: simple) - - """ - import getopt - import sys - import textwrap - usage = textwrap.dedent(_main.__doc__) - try: - opts, args = getopt.getopt( - sys.argv[1:], "h1o:s:F:f:", - ["help", "header", "output", "sep=", "float=", "format="]) - except getopt.GetoptError as e: - print(e) - print(usage) - sys.exit(2) - headers = [] - floatfmt = _DEFAULT_FLOATFMT - tablefmt = "simple" - sep = r"\s+" - outfile = "-" - for opt, value in opts: - if opt in ["-1", "--header"]: - headers = "firstrow" - elif opt in ["-o", "--output"]: - outfile = value - elif opt in ["-F", "--float"]: - floatfmt = value - elif opt in ["-f", "--format"]: - if value not in tabulate_formats: - print("%s is not a supported table format" % value) - print(usage) - sys.exit(3) - tablefmt = value - elif opt in ["-s", "--sep"]: - sep = value - elif opt in ["-h", "--help"]: - print(usage) - sys.exit(0) - files = [sys.stdin] if not args else args - with (sys.stdout if outfile == "-" else open(outfile, "w")) as out: - for f in files: - if f == "-": - f = sys.stdin - if _is_file(f): - _pprint_file(f, headers=headers, tablefmt=tablefmt, - sep=sep, floatfmt=floatfmt, file=out) - else: - with open(f) as fobj: - _pprint_file(fobj, headers=headers, tablefmt=tablefmt, - sep=sep, floatfmt=floatfmt, file=out) - - -def _pprint_file(fobject, headers, tablefmt, sep, floatfmt, file): - rows = fobject.readlines() - table = [re.split(sep, r.rstrip()) for r in rows if r.strip()] - print(tabulate(table, headers, tablefmt, floatfmt=floatfmt), file=file) - - -if __name__ == "__main__": - _main() diff --git a/test/test_expanded.py b/test/test_expanded.py deleted file mode 100644 index 7233e91ce..000000000 --- a/test/test_expanded.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Test the vertical, expanded table formatter.""" -from textwrap import dedent - -from mycli.output_formatter.expanded import expanded_table -from mycli.encodingutils import text_type - - -def test_expanded_table_renders(): - results = [('hello', text_type(123)), ('world', text_type(456))] - - expected = dedent("""\ - ***************************[ 1. row ]*************************** - name | hello - age | 123 - ***************************[ 2. row ]*************************** - name | world - age | 456 - """) - assert expected == expanded_table(results, ('name', 'age')) diff --git a/test/test_output_formatter.py b/test/test_output_formatter.py deleted file mode 100644 index 9844c1919..000000000 --- a/test/test_output_formatter.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -"""Test the generic output formatter interface.""" - -from __future__ import unicode_literals -from decimal import Decimal -from textwrap import dedent - -from mycli.output_formatter.preprocessors import (align_decimals, - bytes_to_string, - convert_to_string, - quote_whitespaces, - override_missing_value, - to_string) -from mycli.output_formatter.output_formatter import OutputFormatter -from mycli.output_formatter import delimited_output_adapter -from mycli.output_formatter import tabulate_adapter -from mycli.output_formatter import terminaltables_adapter - - -def test_to_string(): - """Test the *output_formatter.to_string()* function.""" - assert 'a' == to_string('a') - assert 'a' == to_string(b'a') - assert '1' == to_string(1) - assert '1.23' == to_string(1.23) - - -def test_convert_to_string(): - """Test the *output_formatter.convert_to_string()* function.""" - data = [[1, 'John'], [2, 'Jill']] - headers = [0, 'name'] - expected = ([['1', 'John'], ['2', 'Jill']], ['0', 'name']) - - assert expected == convert_to_string(data, headers) - - -def test_override_missing_values(): - """Test the *output_formatter.override_missing_values()* function.""" - data = [[1, None], [2, 'Jill']] - headers = [0, 'name'] - expected = ([[1, ''], [2, 'Jill']], [0, 'name']) - - assert expected == override_missing_value(data, headers, - missing_value='') - - -def test_bytes_to_string(): - """Test the *output_formatter.bytes_to_string()* function.""" - data = [[1, 'John'], [2, b'Jill']] - headers = [0, 'name'] - expected = ([[1, 'John'], [2, 'Jill']], [0, 'name']) - - assert expected == bytes_to_string(data, headers) - - -def test_align_decimals(): - """Test the *align_decimals()* function.""" - data = [[Decimal('200'), Decimal('1')], [ - Decimal('1.00002'), Decimal('1.0')]] - headers = ['num1', 'num2'] - expected = ([['200', '1'], [' 1.00002', '1.0']], ['num1', 'num2']) - - assert expected == align_decimals(data, headers) - - -def test_align_decimals_empty_result(): - """Test *align_decimals()* with no results.""" - data = [] - headers = ['num1', 'num2'] - expected = ([], ['num1', 'num2']) - - assert expected == align_decimals(data, headers) - - -def test_quote_whitespaces(): - """Test the *quote_whitespaces()* function.""" - data = [[" before", "after "], [" both ", "none"]] - headers = ['h1', 'h2'] - expected = ([["' before'", "'after '"], ["' both '", "'none'"]], - ['h1', 'h2']) - - assert expected == quote_whitespaces(data, headers) - - -def test_quote_whitespaces_empty_result(): - """Test the *quote_whitespaces()* function with no results.""" - data = [] - headers = ['h1', 'h2'] - expected = ([], ['h1', 'h2']) - - assert expected == quote_whitespaces(data, headers) - - -def test_tabulate_wrapper(): - """Test the *output_formatter.tabulate_wrapper()* function.""" - data = [['abc', 1], ['d', 456]] - headers = ['letters', 'number'] - output = tabulate_adapter.adapter(data, headers, table_format='psql') - assert output == dedent('''\ - +-----------+----------+ - | letters | number | - |-----------+----------| - | abc | 1 | - | d | 456 | - +-----------+----------+''') - - -def test_csv_wrapper(): - """Test the *output_formatter.csv_wrapper()* function.""" - # Test comma-delimited output. - data = [['abc', 1], ['d', 456]] - headers = ['letters', 'number'] - output = delimited_output_adapter.adapter(data, headers) - assert output == dedent('''\ - letters,number\r\n\ - abc,1\r\n\ - d,456\r\n''') - - # Test tab-delimited output. - data = [['abc', 1], ['d', 456]] - headers = ['letters', 'number'] - output = delimited_output_adapter.adapter( - data, headers, table_format='tsv') - assert output == dedent('''\ - letters\tnumber\r\n\ - abc\t1\r\n\ - d\t456\r\n''') - - -def test_terminal_tables_wrapper(): - """Test the *output_formatter.terminal_tables_wrapper()* function.""" - data = [['abc', 1], ['d', 456]] - headers = ['letters', 'number'] - output = terminaltables_adapter.adapter( - data, headers, table_format='ascii') - assert output == dedent('''\ - +---------+--------+ - | letters | number | - +---------+--------+ - | abc | 1 | - | d | 456 | - +---------+--------+''') - - -def test_output_formatter(): - """Test the *output_formatter.OutputFormatter* class.""" - data = [['abc', Decimal(1)], ['defg', Decimal('11.1')], - ['hi', Decimal('1.1')]] - headers = ['text', 'numeric'] - expected = dedent('''\ - +------+---------+ - | text | numeric | - +------+---------+ - | abc | 1 | - | defg | 11.1 | - | hi | 1.1 | - +------+---------+''') - - assert expected == OutputFormatter().format_output(data, headers, - format_name='ascii') diff --git a/test/test_tabulate.py b/test/test_tabulate.py deleted file mode 100644 index ae7c25ce1..000000000 --- a/test/test_tabulate.py +++ /dev/null @@ -1,17 +0,0 @@ -from textwrap import dedent - -from mycli.packages import tabulate - -tabulate.PRESERVE_WHITESPACE = True - - -def test_dont_strip_leading_whitespace(): - data = [[' abc']] - headers = ['xyz'] - tbl = tabulate.tabulate(data, headers, tablefmt='psql') - assert tbl == dedent(''' - +---------+ - | xyz | - |---------| - | abc | - +---------+ ''').strip() From 02ae6b9d8d7e9c18bc87e0087b3c01e753575c15 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:34:41 -0500 Subject: [PATCH 0426/1025] Add cli_helpers dependency to changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 5001405c9..1e00f27e9 100644 --- a/changelog.md +++ b/changelog.md @@ -22,6 +22,7 @@ Internal Changes: * Behave test source command (Thanks: [Dick Marinus]). * Test using behave the tee command (Thanks: [Dick Marinus]). * Behave fix clean up. (Thanks: [Dick Marinus]). +* Remove output formatter code in favor of CLI Helpers dependency (Thanks: [Thomas Roten]). 1.10.0: ======= From 317eae173e07af337a004a9ec4f4fc73a6e8539c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 1 May 2017 23:44:16 -0500 Subject: [PATCH 0427/1025] Remove unused function. --- mycli/encodingutils.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/mycli/encodingutils.py b/mycli/encodingutils.py index 1a8b5bbb1..078e67726 100644 --- a/mycli/encodingutils.py +++ b/mycli/encodingutils.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import binascii import sys PY2 = sys.version_info[0] == 2 @@ -37,22 +36,3 @@ def utf8tounicode(arg): if PY2 and isinstance(arg, binary_type): return arg.decode('utf-8') return arg - - -def bytes_to_string(b): - """Convert bytes to a string. Hexlify bytes that can't be decoded. - - >>> print(bytes_to_string(b"\\xff")) - 0xff - >>> print(bytes_to_string('abc')) - abc - >>> print(bytes_to_string('✌')) - ✌ - - """ - if isinstance(b, binary_type): - try: - return b.decode('utf8') - except UnicodeDecodeError: - return '0x' + binascii.hexlify(b).decode('ascii') - return b From a6fb51809371ecb633591068fa5bd35efb9bfb71 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 21:33:36 +0200 Subject: [PATCH 0428/1025] Added a regression test to test_completion_engine.py for sqlparse >= 0.2.3 --- changelog.md | 1 + test/test_completion_engine.py | 1 + 2 files changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index 18c332713..8acc448b1 100644 --- a/changelog.md +++ b/changelog.md @@ -11,6 +11,7 @@ Internal Changes: * Behave test source command (Thanks: [Dick Marinus]). * Test using behave the tee command (Thanks: [Dick Marinus]). * Behave fix clean up. (Thanks: [Dick Marinus]). +* Added a regression test for sqlparse >= 0.2.3 (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/test/test_completion_engine.py b/test/test_completion_engine.py index 4f0406b08..42f4e2e02 100644 --- a/test/test_completion_engine.py +++ b/test/test_completion_engine.py @@ -234,6 +234,7 @@ def test_dot_col_comma_suggests_cols_or_schema_qualified_table(): 'SELECT * FROM (', 'SELECT * FROM foo WHERE EXISTS (', 'SELECT * FROM foo WHERE bar AND NOT EXISTS (', + 'SELECT 1 AS', ]) def test_sub_select_suggests_keyword(expression): suggestion = suggest_type(expression, expression) From 02629c910f6af7e7bc02660287ae8d831fe0d22c Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 21:30:51 +0200 Subject: [PATCH 0429/1025] Revert "remove temporary hack" This reverts commit 21c600256de4c01fb4c4aa97b84ebd28f3f81c9f. --- changelog.md | 1 + mycli/packages/completion_engine.py | 44 ++++++++++++++++------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/changelog.md b/changelog.md index 8acc448b1..feedcf4da 100644 --- a/changelog.md +++ b/changelog.md @@ -12,6 +12,7 @@ Internal Changes: * Test using behave the tee command (Thanks: [Dick Marinus]). * Behave fix clean up. (Thanks: [Dick Marinus]). * Added a regression test for sqlparse >= 0.2.3 (Thanks: [Dick Marinus]). +* Reverted removal of temporary hack for sqlparse (Thanks: [Dick Marinus]). 1.10.0: ======= diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index b97cadf71..efbd41f61 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -28,28 +28,32 @@ def suggest_type(full_text, text_before_cursor): identifier = None - # If we've partially typed a word then word_before_cursor won't be an empty - # string. In that case we want to remove the partially typed string before - # sending it to the sqlparser. Otherwise the last token will always be the - # partially typed string which renders the smart completion useless because - # it will always return the list of keywords as completion. - if word_before_cursor: - if word_before_cursor.endswith( - '(') or word_before_cursor.startswith('\\'): - parsed = sqlparse.parse(text_before_cursor) - else: - parsed = sqlparse.parse( - text_before_cursor[:-len(word_before_cursor)]) + # here should be removed once sqlparse has been fixed + try: + # If we've partially typed a word then word_before_cursor won't be an empty + # string. In that case we want to remove the partially typed string before + # sending it to the sqlparser. Otherwise the last token will always be the + # partially typed string which renders the smart completion useless because + # it will always return the list of keywords as completion. + if word_before_cursor: + if word_before_cursor.endswith( + '(') or word_before_cursor.startswith('\\'): + parsed = sqlparse.parse(text_before_cursor) + else: + parsed = sqlparse.parse( + text_before_cursor[:-len(word_before_cursor)]) - # word_before_cursor may include a schema qualification, like - # "schema_name.partial_name" or "schema_name.", so parse it - # separately - p = sqlparse.parse(word_before_cursor)[0] + # word_before_cursor may include a schema qualification, like + # "schema_name.partial_name" or "schema_name.", so parse it + # separately + p = sqlparse.parse(word_before_cursor)[0] - if p.tokens and isinstance(p.tokens[0], Identifier): - identifier = p.tokens[0] - else: - parsed = sqlparse.parse(text_before_cursor) + if p.tokens and isinstance(p.tokens[0], Identifier): + identifier = p.tokens[0] + else: + parsed = sqlparse.parse(text_before_cursor) + except (TypeError, AttributeError): + return [] if len(parsed) > 1: # Multiple statements being edited -- isolate the current one by From 21cd71d0f4cf4dfb21882aa8bfca349a40809779 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Mon, 1 May 2017 21:32:30 +0200 Subject: [PATCH 0430/1025] make test/test_completion_engine.py happy --- mycli/packages/completion_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/completion_engine.py b/mycli/packages/completion_engine.py index efbd41f61..88fabfa13 100644 --- a/mycli/packages/completion_engine.py +++ b/mycli/packages/completion_engine.py @@ -53,7 +53,7 @@ def suggest_type(full_text, text_before_cursor): else: parsed = sqlparse.parse(text_before_cursor) except (TypeError, AttributeError): - return [] + return [{'type': 'keyword'}] if len(parsed) > 1: # Multiple statements being edited -- isolate the current one by From 4bd32370a44713dcd266f25974d0152f4eb0619e Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Tue, 2 May 2017 09:08:04 +0200 Subject: [PATCH 0431/1025] move boiler plate code to before_scenario --- conftest.py | 7 ++++++ test/features/__init__.py | 0 test/features/basic_commands.feature | 16 +++----------- test/features/crud_database.feature | 8 ++----- test/features/crud_table.feature | 4 +--- test/features/environment.py | 7 ++++++ test/features/iocommands.feature | 8 ++----- test/features/named_queries.feature | 4 +--- test/features/specials.feature | 4 +--- test/features/steps/__init__.py | 0 test/features/steps/basic_commands.py | 31 +++----------------------- test/features/steps/wrappers.py | 32 +++++++++++++++++++++++++++ 12 files changed, 59 insertions(+), 62 deletions(-) create mode 100644 test/features/__init__.py create mode 100644 test/features/steps/__init__.py diff --git a/conftest.py b/conftest.py index d2cd1336c..41e72adae 100644 --- a/conftest.py +++ b/conftest.py @@ -3,4 +3,11 @@ "setup.py", "mycli/magic.py", "mycli/packages/parseutils.py", + "test/features/environment.py", + "test/features/steps/basic_commands.py", + "test/features/steps/crud_database.py", + "test/features/steps/crud_table.py", + "test/features/steps/iocommands.py", + "test/features/steps/named_queries.py", + "test/features/steps/specials.py", ] diff --git a/test/features/__init__.py b/test/features/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/features/basic_commands.feature b/test/features/basic_commands.feature index 025b58502..249a68e6e 100644 --- a/test/features/basic_commands.feature +++ b/test/features/basic_commands.feature @@ -2,24 +2,14 @@ Feature: run the cli, call the help command, exit the cli - Scenario: run the cli - When we run dbcli - then we see dbcli prompt - Scenario: run "\?" command - When we run dbcli - and we wait for prompt - and we send "\?" command + When we send "\?" command then we see help output Scenario: run source command - When we run dbcli - and we wait for prompt - and we send source command + When we send source command then we see help output Scenario: run the cli and exit - When we run dbcli - and we wait for prompt - and we send "ctrl + d" + When we send "ctrl + d" then dbcli exits diff --git a/test/features/crud_database.feature b/test/features/crud_database.feature index c72468c30..32d1e38d4 100644 --- a/test/features/crud_database.feature +++ b/test/features/crud_database.feature @@ -2,9 +2,7 @@ Feature: manipulate databases: create, drop, connect, disconnect Scenario: create and drop temporary database - When we run dbcli - and we wait for prompt - and we create database + When we create database then we see database created when we drop database then we see database dropped @@ -12,9 +10,7 @@ Feature: manipulate databases: then we see database connected Scenario: connect and disconnect from test database - When we run dbcli - and we wait for prompt - and we connect to test database + When we connect to test database then we see database connected when we connect to dbserver then we see database connected diff --git a/test/features/crud_table.feature b/test/features/crud_table.feature index d2209fd0e..bcef9292e 100644 --- a/test/features/crud_table.feature +++ b/test/features/crud_table.feature @@ -2,9 +2,7 @@ Feature: manipulate tables: create, insert, update, select, delete from, drop Scenario: create, insert, select from, update, drop table - When we run dbcli - and we wait for prompt - and we connect to test database + When we connect to test database then we see database connected when we create table then we see table created diff --git a/test/features/environment.py b/test/features/environment.py index a0456b99f..138f88f15 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -8,6 +8,8 @@ import fixture_utils as fixutils import pexpect +from steps.wrappers import run_cli, wait_prompt + def before_all(context): """Set env parameters.""" @@ -73,6 +75,11 @@ def before_step(context, _): context.atprompt = False +def before_scenario(context, _): + run_cli(context) + wait_prompt(context) + + def after_scenario(context, _): """Cleans up after each test complete.""" diff --git a/test/features/iocommands.feature b/test/features/iocommands.feature index 4bcdf6e65..38efbbb0a 100644 --- a/test/features/iocommands.feature +++ b/test/features/iocommands.feature @@ -1,18 +1,14 @@ Feature: I/O commands Scenario: edit sql in file with external editor - When we run dbcli - and we wait for prompt - and we start external editor providing a file name + When we start external editor providing a file name and we type sql in the editor and we exit the editor then we see dbcli prompt and we see the sql in prompt Scenario: tee output from query - When we run dbcli - and we wait for prompt - and we tee output + When we tee output and we wait for prompt and we query "select 123456" and we wait for prompt diff --git a/test/features/named_queries.feature b/test/features/named_queries.feature index 79f31ac3a..74201b92a 100644 --- a/test/features/named_queries.feature +++ b/test/features/named_queries.feature @@ -2,9 +2,7 @@ Feature: named queries: save, use and delete named queries Scenario: save, use and delete named queries - When we run dbcli - and we wait for prompt - and we connect to test database + When we connect to test database then we see database connected when we save a named query then we see the named query saved diff --git a/test/features/specials.feature b/test/features/specials.feature index 9bacec45a..bb3675784 100644 --- a/test/features/specials.feature +++ b/test/features/specials.feature @@ -2,8 +2,6 @@ Feature: Special commands @wip Scenario: run refresh command - When we run dbcli - and we wait for prompt - and we refresh completions + When we refresh completions and we wait for prompt then we see completions refresh started diff --git a/test/features/steps/__init__.py b/test/features/steps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/features/steps/basic_commands.py b/test/features/steps/basic_commands.py index df97ee0e7..299893def 100644 --- a/test/features/steps/basic_commands.py +++ b/test/features/steps/basic_commands.py @@ -7,44 +7,19 @@ """ from __future__ import unicode_literals -import pexpect -import tempfile - from behave import when +import tempfile import wrappers @when('we run dbcli') def step_run_cli(context): - """Run the process using pexpect.""" - run_args = [] - if context.conf.get('host', None): - run_args.extend(('-h', context.conf['host'])) - if context.conf.get('user', None): - run_args.extend(('-u', context.conf['user'])) - if context.conf.get('pass', None): - run_args.extend(('-p', context.conf['pass'])) - if context.conf.get('dbname', None): - run_args.extend(('-D', context.conf['dbname'])) - cli_cmd = context.conf.get('cli_command', None) or sys.executable + \ - ' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' - - cmd_parts = [cli_cmd] + run_args - cmd = ' '.join(cmd_parts) - context.cli = pexpect.spawnu(cmd, cwd='..') - context.exit_sent = False - context.currentdb = context.conf['dbname'] + wrappers.run_cli(context) @when('we wait for prompt') def step_wait_prompt(context): - """Make sure prompt is displayed.""" - user = context.conf['user'] - host = context.conf['host'] - dbname = context.currentdb - wrappers.expect_exact(context, 'mysql {0}@{1}:{2}> '.format( - user, host, dbname), timeout=5) - context.atprompt = True + wrappers.wait_prompt(context) @when('we send "ctrl + d"') diff --git a/test/features/steps/wrappers.py b/test/features/steps/wrappers.py index e8d9204ab..3855ede07 100644 --- a/test/features/steps/wrappers.py +++ b/test/features/steps/wrappers.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import re +import pexpect def expect_exact(context, expected, timeout): @@ -19,3 +20,34 @@ def expect_exact(context, expected, timeout): def expect_pager(context, expected, timeout): expect_exact(context, "{0}\r\n{1}{0}\r\n".format( context.conf['pager_boundary'], expected), timeout=timeout) + + +def run_cli(context): + """Run the process using pexpect.""" + run_args = [] + if context.conf.get('host', None): + run_args.extend(('-h', context.conf['host'])) + if context.conf.get('user', None): + run_args.extend(('-u', context.conf['user'])) + if context.conf.get('pass', None): + run_args.extend(('-p', context.conf['pass'])) + if context.conf.get('dbname', None): + run_args.extend(('-D', context.conf['dbname'])) + cli_cmd = context.conf.get('cli_command', None) or sys.executable + \ + ' -c "import coverage ; coverage.process_startup(); import mycli.main; mycli.main.cli()"' + + cmd_parts = [cli_cmd] + run_args + cmd = ' '.join(cmd_parts) + context.cli = pexpect.spawnu(cmd, cwd='..') + context.exit_sent = False + context.currentdb = context.conf['dbname'] + + +def wait_prompt(context): + """Make sure prompt is displayed.""" + user = context.conf['user'] + host = context.conf['host'] + dbname = context.currentdb + expect_exact(context, 'mysql {0}@{1}:{2}> '.format( + user, host, dbname), timeout=5) + context.atprompt = True From 1a24358e1c4cab3101f4353b4ca2792345e89036 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 2 May 2017 23:03:29 -0500 Subject: [PATCH 0432/1025] Edit last command in external editor. --- mycli/main.py | 9 +++++++-- mycli/packages/special/iocommands.py | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 5df93408a..7d81f5c56 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -419,8 +419,9 @@ def handle_editor_command(self, cli, document): saved_callables = cli.application.pre_run_callables while special.editor_command(document.text): filename = special.get_filename(document.text) - sql, message = special.open_external_editor(filename, - sql=document.text) + sql, message = special.open_external_editor( + filename, sql=document.text, + default_text=self.get_last_query() or '') if message: # Something went wrong. Raise an exception and bail. raise RuntimeError(message) @@ -756,6 +757,10 @@ def get_reserved_space(self): _, height = click.get_terminal_size() return min(round(height * reserved_space_ratio), max_reserved_space) + def get_last_query(self): + """Get the last query executed or None.""" + return self.query_history[-1][0] if self.query_history else None + @click.command() @click.option('-h', '--host', envvar='MYSQL_HOST', help='Host address of the database.') diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 4d4fcc655..2cdc86ca6 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -102,7 +102,7 @@ def get_filename(sql): return filename.strip() or None @export -def open_external_editor(filename=None, sql=''): +def open_external_editor(filename=None, sql='', default_text=''): """ Open external editor, wait for the user to type in his query, return the query. @@ -118,6 +118,8 @@ def open_external_editor(filename=None, sql=''): while pattern.search(sql): sql = pattern.sub('', sql) + text = sql if sql else default_text + message = None filename = filename.strip().split(' ', 1)[0] if filename else None @@ -125,7 +127,7 @@ def open_external_editor(filename=None, sql=''): # Populate the editor buffer with the partial sql (if available) and a # placeholder comment. - query = click.edit(sql + '\n\n' + MARKER, filename=filename, + query = click.edit(text + '\n\n' + MARKER, filename=filename, extension='.sql') if filename: From 52881034f8f77a1ae7f33be840698c9dcbc59cef Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 2 May 2017 23:04:47 -0500 Subject: [PATCH 0433/1025] Simplify conditional logic. --- mycli/packages/special/iocommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 2cdc86ca6..5398d0c69 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -118,7 +118,7 @@ def open_external_editor(filename=None, sql='', default_text=''): while pattern.search(sql): sql = pattern.sub('', sql) - text = sql if sql else default_text + text = sql or default_text message = None filename = filename.strip().split(' ', 1)[0] if filename else None From eac5dbc70ada397dd1fbf8f66584725a5e811524 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 2 May 2017 23:10:50 -0500 Subject: [PATCH 0434/1025] Add external editor change to changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 5001405c9..51a8748c6 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,7 @@ Features: * Handle reserved space for completion menu better in small windows. (Thanks: [Thomas Roten]). * Display current vi mode in toolbar. (Thanks: [Thomas Roten]). +* Opening an external editor will edit the last-run query. (Thanks: [Thomas Roten]). Bug Fixes: ---------- From b4a8ea41a956cd7735de1e63b88d0758758b2fb1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:41:00 -0500 Subject: [PATCH 0435/1025] Simplify editor command logic. --- mycli/main.py | 7 ++++--- mycli/packages/special/iocommands.py | 21 ++++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 7d81f5c56..9676161d6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -419,9 +419,10 @@ def handle_editor_command(self, cli, document): saved_callables = cli.application.pre_run_callables while special.editor_command(document.text): filename = special.get_filename(document.text) - sql, message = special.open_external_editor( - filename, sql=document.text, - default_text=self.get_last_query() or '') + query = (special.get_editor_query(document.text) or + self.get_last_query() or '') + sql, message = special.open_external_editor(filename, + sql=query) if message: # Something went wrong. Raise an exception and bail. raise RuntimeError(message) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 5398d0c69..abcdfa3c6 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -102,13 +102,8 @@ def get_filename(sql): return filename.strip() or None @export -def open_external_editor(filename=None, sql='', default_text=''): - """ - Open external editor, wait for the user to type in his query, - return the query. - :return: list with one tuple, query as first element. - """ - +def get_editor_query(sql): + """Get the query part of an editor command.""" sql = sql.strip() # The reason we can't simply do .strip('\e') is that it strips characters, @@ -118,7 +113,15 @@ def open_external_editor(filename=None, sql='', default_text=''): while pattern.search(sql): sql = pattern.sub('', sql) - text = sql or default_text + return sql + +@export +def open_external_editor(filename=None, sql='', default_text=''): + """ + Open external editor, wait for the user to type in his query, + return the query. + :return: list with one tuple, query as first element. + """ message = None filename = filename.strip().split(' ', 1)[0] if filename else None @@ -127,7 +130,7 @@ def open_external_editor(filename=None, sql='', default_text=''): # Populate the editor buffer with the partial sql (if available) and a # placeholder comment. - query = click.edit(text + '\n\n' + MARKER, filename=filename, + query = click.edit(sql + '\n\n' + MARKER, filename=filename, extension='.sql') if filename: From 541929ff7ed2cb8c8c2903694f547c20bd2130dc Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:43:24 -0500 Subject: [PATCH 0436/1025] Remove unused argument. --- mycli/packages/special/iocommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index abcdfa3c6..8472792b5 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -116,7 +116,7 @@ def get_editor_query(sql): return sql @export -def open_external_editor(filename=None, sql='', default_text=''): +def open_external_editor(filename=None, sql=''): """ Open external editor, wait for the user to type in his query, return the query. From b5cd5f31f4cac078e2050cf8524b58393a57a1a4 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:44:19 -0500 Subject: [PATCH 0437/1025] Pep8 fix. --- mycli/packages/special/iocommands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 8472792b5..0e09a9ecc 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -117,9 +117,9 @@ def get_editor_query(sql): @export def open_external_editor(filename=None, sql=''): - """ - Open external editor, wait for the user to type in his query, - return the query. + """Open external editor, wait for the user to type in their query, return + the query. + :return: list with one tuple, query as first element. """ From 466d0f3ea01d7f83e9c5470e9b61221d4a2c7165 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:48:40 -0500 Subject: [PATCH 0438/1025] Simplify editing logic. --- mycli/main.py | 2 +- mycli/packages/special/iocommands.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 9676161d6..af535ca30 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -420,7 +420,7 @@ def handle_editor_command(self, cli, document): while special.editor_command(document.text): filename = special.get_filename(document.text) query = (special.get_editor_query(document.text) or - self.get_last_query() or '') + self.get_last_query()) sql, message = special.open_external_editor(filename, sql=query) if message: diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 0e09a9ecc..29a9f99e5 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -116,7 +116,7 @@ def get_editor_query(sql): return sql @export -def open_external_editor(filename=None, sql=''): +def open_external_editor(filename=None, sql=None): """Open external editor, wait for the user to type in their query, return the query. @@ -126,12 +126,13 @@ def open_external_editor(filename=None, sql=''): message = None filename = filename.strip().split(' ', 1)[0] if filename else None + sql = sql or '' MARKER = '# Type your query above this line.\n' # Populate the editor buffer with the partial sql (if available) and a # placeholder comment. - query = click.edit(sql + '\n\n' + MARKER, filename=filename, - extension='.sql') + query = click.edit("{sql}\n\n{marker}".format(sql=sql, marker=MARKER), + filename=filename, extension='.sql') if filename: try: From b68c3b2b21730cf69da24f1cd8ad9a8886b5ea56 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:49:36 -0500 Subject: [PATCH 0439/1025] Single quotes instead of double. --- mycli/packages/special/iocommands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 29a9f99e5..87d682737 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -131,7 +131,7 @@ def open_external_editor(filename=None, sql=None): # Populate the editor buffer with the partial sql (if available) and a # placeholder comment. - query = click.edit("{sql}\n\n{marker}".format(sql=sql, marker=MARKER), + query = click.edit('{sql}\n\n{marker}'.format(sql=sql, marker=MARKER), filename=filename, extension='.sql') if filename: From f34c042075dd1df10c32a69d7e5460f0409e2e3a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:50:15 -0500 Subject: [PATCH 0440/1025] Extra newlines for PEP 8. --- mycli/packages/special/iocommands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 87d682737..19c2dc735 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -101,6 +101,7 @@ def get_filename(sql): command, _, filename = sql.partition(' ') return filename.strip() or None + @export def get_editor_query(sql): """Get the query part of an editor command.""" @@ -115,6 +116,7 @@ def get_editor_query(sql): return sql + @export def open_external_editor(filename=None, sql=None): """Open external editor, wait for the user to type in their query, return From 133e197eb1e3af2e339c07039e4839496fce1d8f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Wed, 3 May 2017 12:56:14 -0500 Subject: [PATCH 0441/1025] Last pep8 fix :) --- mycli/main.py | 3 +-- mycli/packages/special/iocommands.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index af535ca30..2e2913d00 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -421,8 +421,7 @@ def handle_editor_command(self, cli, document): filename = special.get_filename(document.text) query = (special.get_editor_query(document.text) or self.get_last_query()) - sql, message = special.open_external_editor(filename, - sql=query) + sql, message = special.open_external_editor(filename, sql=query) if message: # Something went wrong. Raise an exception and bail. raise RuntimeError(message) diff --git a/mycli/packages/special/iocommands.py b/mycli/packages/special/iocommands.py index 19c2dc735..92773f8de 100644 --- a/mycli/packages/special/iocommands.py +++ b/mycli/packages/special/iocommands.py @@ -123,6 +123,7 @@ def open_external_editor(filename=None, sql=None): the query. :return: list with one tuple, query as first element. + """ message = None From e8bbe5eb30c683c99d359bfd4c7425c98a065b2a Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 07:34:04 -0500 Subject: [PATCH 0442/1025] Add docstring and use io.open. --- release.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/release.py b/release.py index 817695838..f75277f03 100755 --- a/release.py +++ b/release.py @@ -1,7 +1,10 @@ #!/usr/bin/env python +"""A script to publish a release of mycli to PyPI.""" + from __future__ import print_function import re import ast +import io import subprocess import sys from optparse import OptionParser @@ -51,9 +54,8 @@ def run_step(*args): def version(version_file): _version_re = re.compile(r'__version__\s+=\s+(.*)') - with open(version_file, 'rb') as f: - ver = str(ast.literal_eval(_version_re.search( - f.read().decode('utf-8')).group(1))) + with io.open(version_file, encoding='utf-8') as f: + ver = str(ast.literal_eval(_version_re.search(f.read()).group(1))) return ver @@ -61,7 +63,8 @@ def version(version_file): def commit_for_release(version_file, ver): run_step('git', 'reset') run_step('git', 'add', version_file) - run_step('git', 'commit', '--message', 'Releasing version %s' % ver) + run_step('git', 'commit', '--message', + 'Releasing version {}'.format(ver)) def create_git_tag(tag_name): @@ -128,7 +131,7 @@ def checklist(questions): sys.exit(1) commit_for_release('mycli/__init__.py', ver) - create_git_tag('v%s' % ver) + create_git_tag('v{}'.format(ver)) register_with_pypi() create_distribution_files() push_to_github() From 3cd6492b6105a0babdf45ca0b58d09745fcf2e00 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 07:36:46 -0500 Subject: [PATCH 0443/1025] Don't evaluate Python code. --- release.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release.py b/release.py index f75277f03..02d0e1f5d 100755 --- a/release.py +++ b/release.py @@ -2,9 +2,8 @@ """A script to publish a release of mycli to PyPI.""" from __future__ import print_function -import re -import ast import io +import re import subprocess import sys from optparse import OptionParser @@ -52,10 +51,11 @@ def run_step(*args): def version(version_file): - _version_re = re.compile(r'__version__\s+=\s+(.*)') + _version_re = re.compile( + r'__version__\s+=\s+(?P[\'"])(?P.*)(?P=quote)') with io.open(version_file, encoding='utf-8') as f: - ver = str(ast.literal_eval(_version_re.search(f.read()).group(1))) + ver = _version_re.search(f.read()).group('version') return ver From f7194c6b19b57879ebee6abac6a7ab22b553680f Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 09:29:02 -0500 Subject: [PATCH 0444/1025] Use click's confirm for release script. --- release.py | 17 +++++------------ requirements-dev.txt | 1 + 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/release.py b/release.py index 02d0e1f5d..499b9cb0b 100755 --- a/release.py +++ b/release.py @@ -3,15 +3,12 @@ from __future__ import print_function import io +from optparse import OptionParser import re import subprocess import sys -from optparse import OptionParser -try: - input = raw_input -except NameError: - pass +import click DEBUG = False CONFIRM_STEPS = False @@ -26,9 +23,7 @@ def skip_step(): global CONFIRM_STEPS if CONFIRM_STEPS: - choice = input("--- Confirm step? (y/N) [y] ") - if choice.lower() == 'n': - return True + return not click.confirm('--- Run this step?', default=True) return False @@ -93,8 +88,7 @@ def push_tags_to_github(): def checklist(questions): for question in questions: - choice = input(question + ' (y/N) [n] ') - if choice.lower() != 'y': + if not click.confirm('--- {}'.format(question), default=False): sys.exit(1) @@ -126,8 +120,7 @@ def checklist(questions): CONFIRM_STEPS = popts.confirm_steps DRY_RUN = popts.dry_run - choice = input('Are you sure? (y/N) [n] ') - if choice.lower() != 'y': + if not click.confirm('Are you sure?', default=False): sys.exit(1) commit_for_release('mycli/__init__.py', ver) diff --git a/requirements-dev.txt b/requirements-dev.txt index b7e6e2dca..cf552f484 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,4 @@ behave pexpect coverage==4.3.4 pep8radius +click==6.7 From 945744537348a185b9bfe6a7ad7ad0d2237e2153 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 09:30:19 -0500 Subject: [PATCH 0445/1025] Remove unneeded register step. --- release.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/release.py b/release.py index 499b9cb0b..18e1b8f27 100755 --- a/release.py +++ b/release.py @@ -66,10 +66,6 @@ def create_git_tag(tag_name): run_step('git', 'tag', tag_name) -def register_with_pypi(): - run_step('python', 'setup.py', 'register') - - def create_distribution_files(): run_step('python', 'setup.py', 'sdist', 'bdist_wheel') @@ -125,7 +121,6 @@ def checklist(questions): commit_for_release('mycli/__init__.py', ver) create_git_tag('v{}'.format(ver)) - register_with_pypi() create_distribution_files() push_to_github() push_tags_to_github() From 1f1c9369d1fb862d6120ae3540205ff78fb996bd Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 09:47:22 -0500 Subject: [PATCH 0446/1025] Pin the correct version of pep8radius for development. --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b7e6e2dca..19df802e9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,4 @@ twine==1.8.1 behave pexpect coverage==4.3.4 -pep8radius +git+https://github.com/hayd/pep8radius.git@c8aebd0e1d272160896124e104773b97a6249c3e#egg=pep8radius From c27a2939002aad4c65da75279b46935a10a33fc7 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 10:18:31 -0500 Subject: [PATCH 0447/1025] Add Makefile. --- Makefile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ad6cfbbcc --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +.PHONY: clean lint lint-fix test test-all + +help: + @echo "clean - remove all build artifacts" + @echo "lint - check code changes against PEP 8" + @echo "lint-fix - automatically fix PEP 8 violations" + @echo "test - run tests quickly with the current Python" + @echo "test-all - run tests in all environments" + +clean: + rm -rf build dist egg *.egg-info + find . -name '*.py[co]' -exec rm -f {} + + +lint: + pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) + +lint-fix: + pep8radius master --docformatter --in-place + +test: + if pytest ; then cd test && behave ; fi + +test-all: + tox From a0c38677e3323773129466dd3499633d7cd219e1 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 10:22:03 -0500 Subject: [PATCH 0448/1025] Add Makefile to manifest. --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index a50b73686..f787b9455 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include LICENSE.txt *.md *.rst TODO requirements-dev.txt screenshots/* +include LICENSE.txt *.md *.rst TODO requirements-dev.txt Makefile screenshots/* include conftest.py .coveragerc pytest.ini test tox.ini From 469e0adbb144419da9d89bfd2475b8874b1955e0 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 10:22:43 -0500 Subject: [PATCH 0449/1025] Add Makefile to changelog. --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 1e9fd5fda..ad9edfb12 100644 --- a/changelog.md +++ b/changelog.md @@ -27,6 +27,7 @@ Internal Changes: * Better handle common before/after scenarios in behave. (Thanks: [Dick Marinus]) * Added a regression test for sqlparse >= 0.2.3 (Thanks: [Dick Marinus]). * Reverted removal of temporary hack for sqlparse (Thanks: [Dick Marinus]). +* Add Makefile to simplify development tasks (Thanks: [Thomas Roten]). 1.10.0: ======= From 5f75dc18942f62f72a7dbfe94270c43fa3a8f775 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 14:13:57 -0500 Subject: [PATCH 0450/1025] Streamline development guide using make commands. --- DEVELOP.rst | 99 ++++++++++++++++++++++++----------------------------- 1 file changed, 44 insertions(+), 55 deletions(-) diff --git a/DEVELOP.rst b/DEVELOP.rst index 0d53e6ad7..c4699f626 100644 --- a/DEVELOP.rst +++ b/DEVELOP.rst @@ -1,86 +1,75 @@ Development Guide ----------------- + This is a guide for developers who would like to contribute to this project. +If you're interested in contributing to mycli, thank you. We'd love your help! +You'll always get credit for your work. + GitHub Workflow --------------- -If you're interested in contributing to mycli, first of all my heart felt -thanks. `Fork the project `_ in github. Then -clone your fork into your computer (``git clone ``). Make -the changes and create the commits in your local machine. Then push those -changes to your fork. Then click on the pull request icon on github and create -a new pull request. Add a description about the change and send it along. I -promise to review the pull request in a reasonable window of time and get back -to you. +1. `Fork the repository `_ on GitHub. +2. Clone your fork locally:: -In order to keep your fork up to date with any changes from mainline, add a new -git remote to your local copy called 'upstream' and point it to the main mycli -repo. + $ git clone -:: +3. Add the official repository (``upstream``) as a remote repository:: - $ git remote add upstream git@github.com:dbcli/mycli.git + $ git remote add upstream git@github.com:dbcli/mycli.git -Once the 'upstream' end point is added you can then periodically do a ``git -pull upstream master`` to update your local copy and then do a ``git push -origin master`` to keep your own fork up to date. +4. Set up a `virtual environment `_ + for development:: -Local Setup ------------ + $ cd mycli + $ pip install virtualenv + $ virtualenv mycli_dev -The installation instructions in the README file are intended for users of -mycli. If you're developing mycli, you'll need to install it in a slightly -different way so you can see the effects of your changes right away without -having to go through the install cycle everytime you change the code. + We've just created a virtual environment that we'll use to install all the dependencies + and tools we need to work on mycli. Whenever you want to work on mycli, you + need to activate the virtual environment:: -It is highly recommended to use virtualenv for development. If you don't know -what a virtualenv is, this `guide `_ -will help you get started. + $ source mycli_dev/bin/activate -Create a virtualenv (let's call it mycli-dev). Activate it: +5. Install the dependencies and development tools:: -:: + $ pip install -r requirements-dev.txt + $ pip install --editable . - source ./mycli-dev/bin/activate +6. Create a branch for your bugfix or feature:: -Once the virtualenv is activated, `cd` into the local clone of mycli folder -and install mycli using pip as follows: + $ git checkout -b -:: +7. While you work on your bugfix or feature, be sure to pull the latest changes from ``upstream``. This ensures that your local codebase is up-to-date:: - $ pip install --editable . + $ git pull upstream master + + +Running the Tests +----------------- - or +While you work on mycli, it's important to run the tests to make sure your code +hasn't broken any existing functionality. To run the tests, just type in:: - $ pip install -e . + $ make test -This will install the necessary dependencies as well as install mycli from the -working folder into the virtualenv. By installing it using `pip install -e` -we've linked the mycli installation with the working copy. So any changes made -to the code is immediately available in the installed version of mycli. This -makes it easy to change something in the code, launch mycli and check the -effects of your change. +Mycli supports Python 2.7 and 3.3+. You can test against multiple versions of +Python by running:: -Building DEB package from scratch --------------------- + $ make test-all -First pip install `make-deb`. Then run make-deb. It will create a debian folder -after asking a few questions like maintainer name, email etc. -$ vagrant up +Coding Style +------------ -PEP8 checks ------------ +Mycli requires code submissions to adhere to +`PEP 8 `_. +It's easy to check the style of your code, just run:: -When you submit a PR, the changeset is checked for pep8 compliance using -`pep8radius `_. If you see a build failing because -of these checks, install pep8radius and apply style fixes: + $ make lint -:: +If you see any PEP 8 style issues, you can automatically fix them by running:: - $ pip install pep8radius - $ pep8radius --docformatter --diff # view a diff of proposed fixes - $ pep8radius --docformatter --in-place # apply the fixes + $ make lint-fix -Then commit and push the fixes. +Be sure to commit and push any PEP 8 fixes. From 615564c34b52b022816d64fe22f254834b099c19 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 14:18:43 -0500 Subject: [PATCH 0451/1025] Remove outdated release procedure file. --- release_procedure.txt | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 release_procedure.txt diff --git a/release_procedure.txt b/release_procedure.txt deleted file mode 100644 index 1b935b624..000000000 --- a/release_procedure.txt +++ /dev/null @@ -1,14 +0,0 @@ -# vi: ft=vimwiki - -* Bump the version number in mycli/__init__.py -* Commit with message: 'Releasing version X.X.X.' -* Create a tag: git tag vX.X.X -* Register with pypi for new version: python setup.py register -* Fix the image url in PyPI to point to github raw content. https://raw.githubusercontent.com/dbcli/mysql-cli/master/screenshots/image01.png -* Create source dist tar ball: python setup.py sdist -* Test this by installing it in a fresh new virtualenv. Run SanityChecks [./sanity_checks.txt]. -* Upload the source dist to PyPI: https://pypi.python.org/pypi/mycli -* pip install mycli -* Run SanityChecks. -* Push the version back to github: git push --tags origin master -* Done! From 593e68b0d60efc0ead15e782e04594e992b8a245 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 21:09:01 -0500 Subject: [PATCH 0452/1025] Remove unused files. --- MANIFEST.in | 2 +- TODO | 10 ------ Vagrantfile | 30 ------------------ create_deb.sh | 31 ------------------ debian/changelog | 74 ------------------------------------------- debian/compat | 1 - debian/control | 13 -------- debian/mycli.triggers | 8 ----- debian/postinst | 5 --- debian/postrm | 5 --- debian/rules | 4 --- release_procedure.txt | 14 -------- 12 files changed, 1 insertion(+), 196 deletions(-) delete mode 100644 TODO delete mode 100644 Vagrantfile delete mode 100755 create_deb.sh delete mode 100644 debian/changelog delete mode 100644 debian/compat delete mode 100644 debian/control delete mode 100644 debian/mycli.triggers delete mode 100644 debian/postinst delete mode 100644 debian/postrm delete mode 100644 debian/rules delete mode 100644 release_procedure.txt diff --git a/MANIFEST.in b/MANIFEST.in index a50b73686..d9bc01869 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include LICENSE.txt *.md *.rst TODO requirements-dev.txt screenshots/* +include LICENSE.txt *.md *.rst requirements-dev.txt screenshots/* include conftest.py .coveragerc pytest.ini test tox.ini diff --git a/TODO b/TODO deleted file mode 100644 index 64c9842af..000000000 --- a/TODO +++ /dev/null @@ -1,10 +0,0 @@ -# vi: ft=vimwiki - -* [ ] Check if views are available in mysql. -* [ ] Create waffle.io page. -* [ ] Setup gitter. -* [ ] Setup a landing page for mycli.net. -* [ ] Send out invites to backers, pgcli contributors. -* [ ] Write a blog post on personal blog about the experience of kickstarter. -* [ ] Check mycli against MariaDB and Percona. -* [ ] Use error codes instead of matching error strings for reconnect, auto-password prompt etc. diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index a514d1efd..000000000 --- a/Vagrantfile +++ /dev/null @@ -1,30 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure(2) do |config| - - config.vm.synced_folder ".", "/mycli" - - config.vm.define "debian" do |debian| - debian.vm.box = "debian/jessie64" - debian.vm.provision "shell", inline: <<-SHELL - echo "-> Building DEB" - sudo apt-get update - sudo echo "deb http://ppa.launchpad.net/spotify-jyrki/dh-virtualenv/ubuntu trusty main" >> /etc/apt/sources.list - sudo echo "deb-src http://ppa.launchpad.net/spotify-jyrki/dh-virtualenv/ubuntu trusty main" >> /etc/apt/sources.list - sudo apt-get update - sudo apt-get install -y --force-yes python-virtualenv dh-virtualenv debhelper build-essential python-setuptools python-dev - echo "-> Cleaning up old workspace" - rm -rf build - mkdir -p build - cp -r /mycli build/. - cd build/mycli - - echo "-> Creating mycli deb" - dpkg-buildpackage -us -uc - cp ../*.deb /mycli/. - SHELL - end - -end - diff --git a/create_deb.sh b/create_deb.sh deleted file mode 100755 index 8e75ebb31..000000000 --- a/create_deb.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -set -e - -make-deb -cd debian - -cat > postinst <<- EOM -#!/bin/bash - -echo "Setting up symlink to mycli" -ln -sf /usr/share/python/mycli/bin/mycli /usr/local/bin/mycli -EOM -echo "Created postinst file." - -cat > postrm <<- EOM -#!/bin/bash - -echo "Removing symlink to mycli" -rm /usr/local/bin/mycli -EOM -echo "Created postrm file." - -for f in * -do - echo "" >> $f; -done - -echo "INFO: debian folder is setup and ready." -echo "INFO: 1. Update the changelog with real changes." -echo "INFO: 2. Run:\n\tvagrant provision || vagrant up" diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 5135bf68a..000000000 --- a/debian/changelog +++ /dev/null @@ -1,74 +0,0 @@ -mycli (1.7.0) unstable; urgency=medium - - * Add stdin batch mode. (Thanks: Thomas Roten). - * Add warn/no-warn command-line options. (Thanks: Thomas Roten). - * Upgrade sqlparse dependency to 0.1.19. (Thanks: [Amjith Ramanujam]). - * Update features list in README.md. (Thanks: Matheus Rosa). - * Remove extra \n in features list in README.md. (Thanks: Matheus Rosa). - * Enable history search via . (Thanks: [Amjith Ramanujam]). - * Upgrade prompt_toolkit to 1.0.0. (Thanks: Jonathan Slenders) - - -- Casper Langemeijer Fri, 27 May 2016 12:03:31 +0200 - -mycli (1.6.0) unstable; urgency=medium - - * Change continuation prompt for multi-line mode to match default mysql. - * Add status command to match mysql's status command. (Thanks: Thomas Roten). - * Add SSL support for mycli. (Thanks: Artem Bezsmertnyi). - * Add auto-completion and highlight support for OFFSET keyword. (Thanks: Matheus Rosa). - * Add support for MYSQL_TEST_LOGIN_FILE env variable to specify alternate login file. (Thanks: Thomas Roten). - * Add support for --auto-vertical-output to automatically switch to vertical output if the output doesn't fit in the table format. - * Add support for system-wide config. Now /etc/myclirc will be honored. (Thanks: Thomas Roten). - * Add support for nopager and \n to turn off the pager. (Thanks: Thomas Roten). - * Add support for --local-infile command-line option. (Thanks: Thomas Roten). - * Remove -S from less option which was clobbering the scroll back in history. (Thanks: Thomas Roten). - * Make system command work with Python 3. (Thanks: Thomas Roten). - * Support \G terminator for \f queries. (Thanks: Terseus). - * Upgrade prompt_toolkit to 0.60. - * Add Python 3.5 to test environments. (Thanks: Thomas Roten). - * Remove license meta-data. (Thanks: Thomas Roten). - * Skip binary tests if PyMySQL version does not support it. (Thanks: Thomas Roten). - * Refactor pager handling. (Thanks: Thomas Roten) - * Capture warnings to log file. (Thanks: Mikhail Borisov). - * Make syntax_style a tiny bit more intuitive. (Thanks: Phil Cohen). - - -- Casper Langemeijer Fri, 27 May 2016 12:03:31 +0200 - -mycli (1.5.2) unstable; urgency=low - - * Protect against port number being None when no port is specified in command line. - * Cast the value of port read from my.cnf to int. - * Make a config option to enable `audit_log`. (Thanks: [Matheus Rosa]). - * Add support for reading .mylogin.cnf to get user credentials. (Thanks: [Thomas Roten]). - * Register the special command `prompt` with the `\R` as alias. (Thanks: [Matheus Rosa]). - * Perform completion refresh in a background thread. Now mycli can handle - * Add support for `system` command. (Thanks: [Matheus Rosa]). - * Caught and hexed binary fields in MySQL. (Thanks: [Daniel West]). - * Treat enter key as tab when the suggestion menu is open. (Thanks: [Matheus Rosa]) - * Add "delete" and "truncate" as destructive commands. (Thanks: [Martijn Engler]). - * Change \dt syntax to add an optional table name. (Thanks: [Shoma Suzuki]). - * Add TRANSACTION related keywords. - * Treat DESC and EXPLAIN as DESCRIBE. (Thanks: [spacewander]). - * Fix the removal of whitespace from table output. - * Add ability to make suggestions for compound join clauses. (Thanks: [Matheus Rosa]). - * Fix the incorrect reporting of command time. - * Add type validation for port argument. (Thanks [Matheus Rosa]) - * Make pycrypto optional and only install it in \*nix systems. (Thanks: [Iryna Cherniavska]). - * Add badge for PyPI version to README. (Thanks: [Shoma Suzuki]). - * Updated release script with a --dry-run and --confirm-steps option. (Thanks: [Iryna Cherniavska]). - * Adds support for PyMySQL 0.6.2 and above. This is useful for debian package builders. (Thanks: [Thomas Roten]). - * Disable click warning. - - -- Casper Langemeijer Sun, 15 Nov 2015 10:26:24 +0100 - -mycli (1.4.0) unstable; urgency=low - - * Add `source` command. This allows running sql statement from a file. - * Added a config option to make the warning before destructive commands optional. (Thanks: [Daniel West](https://github.com/danieljwest)) - * Add completion support for CHANGE TO and other master/slave commands. This is still preliminary and it will be enhanced in the future. - * Add custom styles to color the menus and toolbars. - * Upgrade prompt_toolkit to 0.46. (Thanks: [Jonathan Slenders](https://github.com/jonathanslenders)) - * Fix keyword completion after the `WHERE` clause. - * Add `\g` and `\G` as valid query terminators. Previously in multi-line mode ending a query with a `\G` wouldn't run the query. This is now fixed. - - -- Amjith Ramanujam Sun, 23 Aug 2015 20:14:45 +0000 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144f..000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control deleted file mode 100644 index 418383263..000000000 --- a/debian/control +++ /dev/null @@ -1,13 +0,0 @@ -Source: mycli -Section: python -Priority: extra -Maintainer: Amjith Ramanujam -Build-Depends: debhelper (>= 9), python, dh-virtualenv (>= 0.7), python-setuptools, python-dev -Standards-Version: 3.9.5 - -Package: mycli -Architecture: any -Pre-Depends: dpkg (>= 1.16.1), python2.7-minimal, ${misc:Pre-Depends} -Depends: ${python:Depends}, ${misc:Depends} -Description: CLI for MySQL Database. With auto-completion and syntax highlighting. - CLI for MySQL Database. With auto-completion and syntax highlighting. diff --git a/debian/mycli.triggers b/debian/mycli.triggers deleted file mode 100644 index b0b1d2184..000000000 --- a/debian/mycli.triggers +++ /dev/null @@ -1,8 +0,0 @@ -# Register interest in Python interpreter changes (Python 2 for now); and -# don't make the Python package dependent on the virtualenv package -# processing (noawait) -interest-noawait /usr/bin/python2.7 - -# Also provide a symbolic trigger for all dh-virtualenv packages -interest dh-virtualenv-interpreter-update - diff --git a/debian/postinst b/debian/postinst deleted file mode 100644 index 122ff285d..000000000 --- a/debian/postinst +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -echo "Setting up symlink to mycli" -ln -sf /usr/share/python/mycli/bin/mycli /usr/local/bin/mycli - diff --git a/debian/postrm b/debian/postrm deleted file mode 100644 index 850d6e8d8..000000000 --- a/debian/postrm +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -echo "Removing symlink to mycli" -rm /usr/local/bin/mycli - diff --git a/debian/rules b/debian/rules deleted file mode 100644 index 299e30912..000000000 --- a/debian/rules +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/make -f - -%: - dh $@ --with python-virtualenv diff --git a/release_procedure.txt b/release_procedure.txt deleted file mode 100644 index 1b935b624..000000000 --- a/release_procedure.txt +++ /dev/null @@ -1,14 +0,0 @@ -# vi: ft=vimwiki - -* Bump the version number in mycli/__init__.py -* Commit with message: 'Releasing version X.X.X.' -* Create a tag: git tag vX.X.X -* Register with pypi for new version: python setup.py register -* Fix the image url in PyPI to point to github raw content. https://raw.githubusercontent.com/dbcli/mysql-cli/master/screenshots/image01.png -* Create source dist tar ball: python setup.py sdist -* Test this by installing it in a fresh new virtualenv. Run SanityChecks [./sanity_checks.txt]. -* Upload the source dist to PyPI: https://pypi.python.org/pypi/mycli -* pip install mycli -* Run SanityChecks. -* Push the version back to github: git push --tags origin master -* Done! From 1136a1f88c4fe333348b9031341fc5ed2acbbcc5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 21:13:27 -0500 Subject: [PATCH 0453/1025] Move coverage config to setup.cfg. --- .coveragerc | 3 --- setup.cfg | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index ae818eefb..000000000 --- a/.coveragerc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -parallel=True -source=mycli diff --git a/setup.cfg b/setup.cfg index 2a9acf13d..c017a54d9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,6 @@ [bdist_wheel] universal = 1 + +[coverage:run] +parallel=True +source=mycli From ffd74e6979acc81c5e21957716d740dc3643e343 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 21:14:36 -0500 Subject: [PATCH 0454/1025] Move pytest config to setup.cfg. --- pytest.ini | 2 -- setup.cfg | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 7c5b52b77..000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts=--capture=sys --showlocals --doctest-modules diff --git a/setup.cfg b/setup.cfg index c017a54d9..b92934a4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,3 +4,6 @@ universal = 1 [coverage:run] parallel=True source=mycli + +[pytest] +addopts=--capture=sys --showlocals --doctest-modules From ead3a32ae7c370b834e3d7de1e91bcd1f1db9d59 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 21:19:11 -0500 Subject: [PATCH 0455/1025] Update pytest section to tool:pytest. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b92934a4b..994fc8d84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,5 +5,5 @@ universal = 1 parallel=True source=mycli -[pytest] +[tool:pytest] addopts=--capture=sys --showlocals --doctest-modules From ff7bc198ee6c9311bcf8451654c64992a6d8393c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 22:11:07 -0500 Subject: [PATCH 0456/1025] Move conftest.py into setup.cfg. --- conftest.py | 13 ------------- setup.cfg | 8 +++++++- 2 files changed, 7 insertions(+), 14 deletions(-) delete mode 100644 conftest.py diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 41e72adae..000000000 --- a/conftest.py +++ /dev/null @@ -1,13 +0,0 @@ -import sys -collect_ignore = [ - "setup.py", - "mycli/magic.py", - "mycli/packages/parseutils.py", - "test/features/environment.py", - "test/features/steps/basic_commands.py", - "test/features/steps/crud_database.py", - "test/features/steps/crud_table.py", - "test/features/steps/iocommands.py", - "test/features/steps/named_queries.py", - "test/features/steps/specials.py", -] diff --git a/setup.cfg b/setup.cfg index 994fc8d84..0854bbe41 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,4 +6,10 @@ parallel=True source=mycli [tool:pytest] -addopts=--capture=sys --showlocals --doctest-modules +addopts = --capture=sys + --showlocals + --doctest-modules + --ignore=setup.py + --ignore=mycli/magic.py + --ignore=mycli/packages/parseutils.py + --ignore=test/features From 0a4675dd65e1b6379d1505379e2e8a776a772eb5 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 22:11:24 -0500 Subject: [PATCH 0457/1025] Standardize spacing in setup.cfg. --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0854bbe41..041a857b1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,8 +2,8 @@ universal = 1 [coverage:run] -parallel=True -source=mycli +parallel = True +source = mycli [tool:pytest] addopts = --capture=sys From befb8a02dd63b98ba2558a8213244a41cc07c75d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 22:21:08 -0500 Subject: [PATCH 0458/1025] Use setup.cfg for coverage subprocess tests. --- test/features/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/features/environment.py b/test/features/environment.py index 138f88f15..0caf4793d 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -16,7 +16,7 @@ def before_all(context): os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" os.environ['EDITOR'] = 'ex' - os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../.coveragerc" + os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../setup.cfg" context.exit_sent = False From e9179a7a8a2d0ff9b24ce019251d5f571d2cf990 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 23:12:31 -0500 Subject: [PATCH 0459/1025] Use pytest-cov plugin. --- .travis.yml | 5 +++-- requirements-dev.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index de590fc05..5e2dea07e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,13 @@ python: - "3.6" install: - - pip install PyMySQL . pytest mock codecov pexpect behave + - pip install PyMySQL pytest mock codecov pexpect behave pytest-cov - pip install git+https://github.com/hayd/pep8radius.git + - pip install -e . script: - set -e - - coverage run --source mycli -m py.test + - pytest --cov-report= --cov=mycli - cd test - behave - cd .. diff --git a/requirements-dev.txt b/requirements-dev.txt index cf552f484..c7e7c2c4f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ mock pytest +pytest-cov==2.4.0 tox twine==1.8.1 behave From ec4fad0716952c15a6e081399ea49f23ead83849 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 23:21:38 -0500 Subject: [PATCH 0460/1025] Travis linter says to drop on_start. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5e2dea07e..7a1023d54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,4 +31,3 @@ notifications: - YOUR_WEBHOOK_URL on_success: change # options: [always|never|change] default: always on_failure: always # options: [always|never|change] default: always - on_start: false # default: false From 3065c0d2bf72c57ca373c800e697324f84e6c4ab Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sat, 6 May 2017 23:25:20 -0500 Subject: [PATCH 0461/1025] pytest isn't found on <3.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7a1023d54..c68e6a9ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ install: script: - set -e - - pytest --cov-report= --cov=mycli + - py.test --cov-report= --cov=mycli - cd test - behave - cd .. From d06ccd48ba0f2c51a262ba8f2e57f3eaab391fdb Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 00:02:39 -0500 Subject: [PATCH 0462/1025] Make behave work from project root as well. --- .travis.yml | 4 +--- setup.cfg | 2 ++ test/features/environment.py | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index c68e6a9ba..49ec67d90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,9 +14,7 @@ install: script: - set -e - py.test --cov-report= --cov=mycli - - cd test - - behave - - cd .. + - behave test/features # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) - set +e diff --git a/setup.cfg b/setup.cfg index 041a857b1..b7286a9bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,6 +9,8 @@ source = mycli addopts = --capture=sys --showlocals --doctest-modules + --cov-report= + --cov=mycli --ignore=setup.py --ignore=mycli/magic.py --ignore=mycli/packages/parseutils.py diff --git a/test/features/environment.py b/test/features/environment.py index 0caf4793d..42ed8df11 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -10,13 +10,16 @@ from steps.wrappers import run_cli, wait_prompt +PACKAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + def before_all(context): """Set env parameters.""" os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" os.environ['EDITOR'] = 'ex' - os.environ["COVERAGE_PROCESS_START"] = os.getcwd() + "/../setup.cfg" + os.environ["COVERAGE_PROCESS_START"] = os.path.join(PACKAGE_ROOT, + 'setup.cfg') context.exit_sent = False @@ -48,7 +51,9 @@ def before_all(context): 'pager_boundary': '---boundary---', } os.environ['PAGER'] = "{0} {1} {2}".format( - sys.executable, "test/features/wrappager.py", context.conf['pager_boundary']) + sys.executable, + os.path.join(PACKAGE_ROOT, '/test/features/wrappager.py'), + context.conf['pager_boundary']) context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], context.conf['pass'], From 88d001ed77935cf2d44fb6f5524a51edfe55e6f8 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 00:03:43 -0500 Subject: [PATCH 0463/1025] Don't run coverage for every py.test. --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index b7286a9bf..041a857b1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,8 +9,6 @@ source = mycli addopts = --capture=sys --showlocals --doctest-modules - --cov-report= - --cov=mycli --ignore=setup.py --ignore=mycli/magic.py --ignore=mycli/packages/parseutils.py From ea92f9e42e7a62340db21b854553281c2fc4403c Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 00:08:22 -0500 Subject: [PATCH 0464/1025] Fix wrappager path. --- test/features/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/features/environment.py b/test/features/environment.py index 42ed8df11..61a579e53 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -52,7 +52,7 @@ def before_all(context): } os.environ['PAGER'] = "{0} {1} {2}".format( sys.executable, - os.path.join(PACKAGE_ROOT, '/test/features/wrappager.py'), + os.path.join(PACKAGE_ROOT, 'test/features/wrappager.py'), context.conf['pager_boundary']) context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], From adf6b939196d63cd3db44167ddc67cabfcc5753d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 06:44:10 -0500 Subject: [PATCH 0465/1025] Drop pytest-cov. --- .travis.yml | 4 ++-- requirements-dev.txt | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 49ec67d90..10cb01a8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,13 +7,13 @@ python: - "3.6" install: - - pip install PyMySQL pytest mock codecov pexpect behave pytest-cov + - pip install PyMySQL pytest mock codecov pexpect behave - pip install git+https://github.com/hayd/pep8radius.git - pip install -e . script: - set -e - - py.test --cov-report= --cov=mycli + - coverage run -m py.test - behave test/features # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) diff --git a/requirements-dev.txt b/requirements-dev.txt index c7e7c2c4f..cf552f484 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,5 @@ mock pytest -pytest-cov==2.4.0 tox twine==1.8.1 behave From 47dcafb642adfa9a3853629a1fe0293fea451c07 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Sun, 7 May 2017 06:44:19 -0500 Subject: [PATCH 0466/1025] Use absolute path for package root. --- mycli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 3eb513c85..ca03900f6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -56,7 +56,7 @@ # Query tuples are used for maintaining history Query = namedtuple('Query', ['query', 'successful', 'mutating']) -PACKAGE_ROOT = os.path.dirname(__file__) +PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__)) # no-op logging handler class NullHandler(logging.Handler): From d4d1f0d88e8a1bd0325bdbbdab658d938371df47 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 08:16:22 -0500 Subject: [PATCH 0467/1025] Move coverage config back to coveragerc. --- .coveragerc | 3 +++ setup.cfg | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..8d3149f62 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +parallel = True +source = mycli diff --git a/setup.cfg b/setup.cfg index 041a857b1..5d578a99a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,6 @@ [bdist_wheel] universal = 1 -[coverage:run] -parallel = True -source = mycli - [tool:pytest] addopts = --capture=sys --showlocals From b471f5005c5b91963852fdbfca7d4c49697dd397 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 08:16:50 -0500 Subject: [PATCH 0468/1025] Make behave support running from project/test dir. --- test/features/environment.py | 12 +++++++----- test/features/steps/wrappers.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/features/environment.py b/test/features/environment.py index 61a579e53..f43ff8426 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -10,16 +10,18 @@ from steps.wrappers import run_cli, wait_prompt -PACKAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - def before_all(context): """Set env parameters.""" os.environ['LINES'] = "100" os.environ['COLUMNS'] = "100" os.environ['EDITOR'] = 'ex' - os.environ["COVERAGE_PROCESS_START"] = os.path.join(PACKAGE_ROOT, - 'setup.cfg') + + context.package_root = os.path.abspath( + os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + + os.environ["COVERAGE_PROCESS_START"] = os.path.join(context.package_root, + '.coveragerc') context.exit_sent = False @@ -52,7 +54,7 @@ def before_all(context): } os.environ['PAGER'] = "{0} {1} {2}".format( sys.executable, - os.path.join(PACKAGE_ROOT, 'test/features/wrappager.py'), + os.path.join(context.package_root, 'test/features/wrappager.py'), context.conf['pager_boundary']) context.cn = dbutils.create_db(context.conf['host'], context.conf['user'], diff --git a/test/features/steps/wrappers.py b/test/features/steps/wrappers.py index 3855ede07..5070f053c 100644 --- a/test/features/steps/wrappers.py +++ b/test/features/steps/wrappers.py @@ -38,7 +38,7 @@ def run_cli(context): cmd_parts = [cli_cmd] + run_args cmd = ' '.join(cmd_parts) - context.cli = pexpect.spawnu(cmd, cwd='..') + context.cli = pexpect.spawnu(cmd, cwd=context.package_root) context.exit_sent = False context.currentdb = context.conf['dbname'] From 21cd18b556fcb7b6ea891c2f68ae29d128713bdf Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 08:34:11 -0500 Subject: [PATCH 0469/1025] Make tee files relative to package root. --- test/features/steps/iocommands.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/features/steps/iocommands.py b/test/features/steps/iocommands.py index 5de640ffb..6fe36f4a1 100644 --- a/test/features/steps/iocommands.py +++ b/test/features/steps/iocommands.py @@ -10,8 +10,8 @@ @when('we start external editor providing a file name') def step_edit_file(context): """Edit file with external editor.""" - context.editor_file_name = '../test_file_{0}.sql'.format( - context.conf['vi']) + context.editor_file_name = os.path.join( + context.package_root, 'test_file_{0}.sql'.format(context.conf['vi'])) if os.path.exists(context.editor_file_name): os.remove(context.editor_file_name) context.cli.sendline('\e {0}'.format( @@ -48,7 +48,8 @@ def step_edit_done_sql(context): @when(u'we tee output') def step_tee_ouptut(context): - context.tee_file_name = '../tee_file_{0}.sql'.format(context.conf['vi']) + context.tee_file_name = os.path.join( + context.package_root, 'tee_file_{0}.sql'.format(context.conf['vi'])) if os.path.exists(context.tee_file_name): os.remove(context.tee_file_name) context.cli.sendline('tee {0}'.format( From ba063c11e38c616ad047f4c02a0cc62dcaf9690d Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 09:59:02 -0500 Subject: [PATCH 0470/1025] Fix test files and config in MANIFEST.in. --- MANIFEST.in | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index d9bc01869..798f07a8a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,6 @@ include LICENSE.txt *.md *.rst requirements-dev.txt screenshots/* -include conftest.py .coveragerc pytest.ini test tox.ini +include .coveragerc tox.ini +recursive-include test *.cnf +recursive-include test *.feature +recursive-include test *.py +recursive-include test *.txt From c606deb3e8e86e73294d92bf714298370d6b1e05 Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Mon, 8 May 2017 17:03:24 -0500 Subject: [PATCH 0471/1025] Revert "Drop pytest-cov." This reverts commit adf6b939196d63cd3db44167ddc67cabfcc5753d. --- .travis.yml | 4 ++-- requirements-dev.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 10cb01a8a..49ec67d90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,13 +7,13 @@ python: - "3.6" install: - - pip install PyMySQL pytest mock codecov pexpect behave + - pip install PyMySQL pytest mock codecov pexpect behave pytest-cov - pip install git+https://github.com/hayd/pep8radius.git - pip install -e . script: - set -e - - coverage run -m py.test + - py.test --cov-report= --cov=mycli - behave test/features # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) diff --git a/requirements-dev.txt b/requirements-dev.txt index cf552f484..c7e7c2c4f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ mock pytest +pytest-cov==2.4.0 tox twine==1.8.1 behave From 229a38c381f52ff7fc13c33b0ae1952791ed9b5e Mon Sep 17 00:00:00 2001 From: Thomas Roten Date: Tue, 9 May 2017 00:49:35 -0500 Subject: [PATCH 0472/1025] Move Makefile tasks to setup.py. --- .travis.yml | 3 +- MANIFEST.in | 4 +- Makefile | 24 ------------ setup.py | 14 ++++--- tasks.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 2 +- 6 files changed, 117 insertions(+), 34 deletions(-) delete mode 100644 Makefile mode change 100644 => 100755 setup.py create mode 100644 tasks.py diff --git a/.travis.yml b/.travis.yml index de590fc05..fff93d10e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,7 @@ script: - cd test - behave - cd .. - # check for pep8 errors, only looking at branch vs master. If there are errors, show diff and return an error code. - - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) + - ./setup.py lint - set +e after_success: diff --git a/MANIFEST.in b/MANIFEST.in index f787b9455..af8dd71a8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include LICENSE.txt *.md *.rst TODO requirements-dev.txt Makefile screenshots/* -include conftest.py .coveragerc pytest.ini test tox.ini +include LICENSE.txt *.md *.rst TODO requirements-dev.txt screenshots/* +include tasks.py conftest.py .coveragerc pytest.ini test tox.ini diff --git a/Makefile b/Makefile deleted file mode 100644 index ad6cfbbcc..000000000 --- a/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -.PHONY: clean lint lint-fix test test-all - -help: - @echo "clean - remove all build artifacts" - @echo "lint - check code changes against PEP 8" - @echo "lint-fix - automatically fix PEP 8 violations" - @echo "test - run tests quickly with the current Python" - @echo "test-all - run tests in all environments" - -clean: - rm -rf build dist egg *.egg-info - find . -name '*.py[co]' -exec rm -f {} + - -lint: - pep8radius master --docformatter --error-status || ( pep8radius master --docformatter --diff; false ) - -lint-fix: - pep8radius master --docformatter --in-place - -test: - if pytest ; then cd test && behave ; fi - -test-all: - tox diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 5140b7e9b..bc39b29de --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ +#!/usr/bin/env python + import re import ast -import platform from setuptools import setup, find_packages _version_re = re.compile(r'__version__\s+=\s+(.*)') @@ -33,10 +34,13 @@ description=description, long_description=description, install_requires=install_requirements, - entry_points=''' - [console_scripts] - mycli=mycli.main:cli - ''', + entry_points={ + 'console_scripts': ['mycli = mycli.main:cli'], + 'distutils.commands': [ + 'lint = tasks:lint', + 'test = tasks:test', + ], + }, classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', diff --git a/tasks.py b/tasks.py new file mode 100644 index 000000000..b65c019e1 --- /dev/null +++ b/tasks.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +"""Common development tasks for setup.py to use.""" + +import re +import subprocess +import sys + +from setuptools import Command + + +class BaseCommand(Command, object): + """The base command for project tasks.""" + + user_options = [] + + default_cmd_options = ('verbose', 'quiet', 'dry_run') + + def __init__(self, *args, **kwargs): + super(BaseCommand, self).__init__(*args, **kwargs) + self.verbose = False + + def initialize_options(self): + """Override the distutils abstract method.""" + pass + + def finalize_options(self): + """Override the distutils abstract method.""" + # Distutils uses incrementing integers for verbosity. + self.verbose = bool(self.verbose) + + def call_and_exit(self, cmd, shell=True): + """Run the *cmd* and exit with the proper exit code.""" + sys.exit(subprocess.call(cmd, shell=shell)) + + def call_in_sequence(self, cmds, shell=True): + """Run multiple commmands in a row, exiting if one fails.""" + for cmd in cmds: + if subprocess.call(cmd, shell=shell) == 1: + sys.exit(1) + + def apply_options(self, cmd, options=()): + """Apply command-line options.""" + for option in (self.default_cmd_options + options): + cmd = self.apply_option(cmd, option, + active=getattr(self, option, False)) + return cmd + + def apply_option(self, cmd, option, active=True): + """Apply a command-line option.""" + return re.sub(r'{{{}\:(?P