Skip to content

Commit f261e09

Browse files
rchiodorchiodo
authored
Fix types other than scalars being used for columns in the Data Viewer (microsoft#5514)
* Fix column types that aren't serializable * Add more tests around breaking cases * Better check for indexColumn * Add news entry * Review feedback and new check for dicts * More imports cleanup
1 parent 97e9495 commit f261e09

7 files changed

Lines changed: 122 additions & 55 deletions

File tree

‎news/2 Fixes/5452.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Handle missing index columns and non trivial data types for columns.

‎pythonFiles/datascience/getJupyterVariableDataFrameInfo.py‎

Lines changed: 56 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Query Jupyter server for the info about a dataframe
22
import json as _VSCODE_json
33
import pandas as _VSCODE_pd
4+
import pandas.io.json as _VSCODE_pd_json
45

56
# _VSCode_sub_supportsDataExplorer will contain our list of data explorer supported types
67
_VSCode_supportsDataExplorer = "['list', 'Series', 'dict', 'ndarray', 'DataFrame']"
@@ -19,40 +20,62 @@
1920
del _VSCode_supportsDataExplorer
2021
_VSCODE_evalResult = eval(_VSCODE_targetVariable['name'])
2122

22-
# First list out the columns of the data frame (assuming it is one for now)
23-
_VSCODE_columnTypes = []
24-
_VSCODE_columnNames = []
25-
if _VSCODE_targetVariable['type'] == 'list':
26-
_VSCODE_evalResult = _VSCODE_pd.DataFrame(_VSCODE_evalResult)
27-
_VSCODE_columnTypes = list(_VSCODE_evalResult.dtypes)
28-
_VSCODE_columnNames = list(_VSCODE_evalResult)
29-
elif _VSCODE_targetVariable['type'] == 'Series':
30-
_VSCODE_evalResult = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult)
31-
_VSCODE_columnTypes = list(_VSCODE_evalResult.dtypes)
32-
_VSCODE_columnNames = list(_VSCODE_evalResult)
33-
elif _VSCODE_targetVariable['type'] == 'dict':
23+
# Figure out shape if not already there. Use the shape to compute the row count
24+
if (hasattr(_VSCODE_evalResult, 'shape')):
25+
try:
26+
# Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it
27+
if isinstance(_VSCODE_evalResult.shape, tuple):
28+
_VSCODE_targetVariable['rowCount'] = _VSCODE_evalResult.shape[0]
29+
except TypeError:
30+
_VSCODE_targetVariable['rowCount'] = 0
31+
elif (hasattr(_VSCODE_evalResult, '__len__')):
32+
try:
33+
_VSCODE_targetVariable['rowCount'] = len(_VSCODE_evalResult)
34+
except TypeError:
35+
_VSCODE_targetVariable['rowCount'] = 0
36+
37+
# Turn the eval result into a df
38+
_VSCODE_df = _VSCODE_evalResult
39+
if isinstance(_VSCODE_evalResult, list):
40+
_VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult)
41+
elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series):
42+
_VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult)
43+
elif isinstance(_VSCODE_evalResult, dict):
3444
_VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult)
35-
_VSCODE_evalResult = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult)
36-
_VSCODE_columnTypes = list(_VSCODE_evalResult.dtypes)
37-
_VSCODE_columnNames = list(_VSCODE_evalResult)
45+
_VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult)
3846
elif _VSCODE_targetVariable['type'] == 'ndarray':
39-
_VSCODE_evalResult = _VSCODE_pd.DataFrame(_VSCODE_evalResult)
40-
_VSCODE_columnTypes = list(_VSCODE_evalResult.dtypes)
41-
_VSCODE_columnNames = list(_VSCODE_evalResult)
42-
elif _VSCODE_targetVariable['type'] == 'DataFrame':
43-
_VSCODE_columnTypes = list(_VSCODE_evalResult.dtypes)
44-
_VSCODE_columnNames = list(_VSCODE_evalResult)
47+
_VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult)
48+
49+
# If any rows, use pandas json to convert a single row to json. Extract
50+
# the column names and types from the json so we match what we'll fetch when
51+
# we ask for all of the rows
52+
if _VSCODE_targetVariable['rowCount']:
53+
try:
54+
_VSCODE_row = _VSCODE_df.iloc[0:1]
55+
_VSCODE_json_row = _VSCODE_pd_json.to_json(None, _VSCODE_row, date_format='iso')
56+
_VSCODE_columnNames = list(_VSCODE_json.loads(_VSCODE_json_row))
57+
del _VSCODE_row
58+
del _VSCODE_json_row
59+
except:
60+
_VSCODE_columnNames = list(_VSCODE_df)
61+
else:
62+
_VSCODE_columnNames = list(_VSCODE_df)
63+
64+
# Compute the index column. It may have been renamed
65+
_VSCODE_indexColumn = _VSCODE_df.index.name if _VSCODE_df.index.name else 'index'
66+
_VSCODE_columnTypes = list(_VSCODE_df.dtypes)
67+
del _VSCODE_df
4568

46-
# Make sure we have an index column (see code in getJupyterVariableDataFrameRows.py)
47-
if 'index' not in _VSCODE_columnNames:
48-
_VSCODE_columnNames.insert(0, 'index')
69+
# Make sure the index column exists
70+
if _VSCODE_indexColumn not in _VSCODE_columnNames:
71+
_VSCODE_columnNames.insert(0, _VSCODE_indexColumn)
4972
_VSCODE_columnTypes.insert(0, 'int64')
5073

5174
# Then loop and generate our output json
5275
_VSCODE_columns = []
5376
for _VSCODE_n in range(0, len(_VSCODE_columnNames)):
54-
_VSCODE_column_name = _VSCODE_columnNames[_VSCODE_n]
5577
_VSCODE_column_type = _VSCODE_columnTypes[_VSCODE_n]
78+
_VSCODE_column_name = str(_VSCODE_columnNames[_VSCODE_n])
5679
_VSCODE_colobj = {}
5780
_VSCODE_colobj['key'] = _VSCODE_column_name
5881
_VSCODE_colobj['name'] = _VSCODE_column_name
@@ -66,16 +89,16 @@
6689

6790
# Save this in our target
6891
_VSCODE_targetVariable['columns'] = _VSCODE_columns
92+
_VSCODE_targetVariable['indexColumn'] = _VSCODE_indexColumn
6993
del _VSCODE_columns
94+
del _VSCODE_indexColumn
7095

71-
# Figure out shape if not already there. Use the shape to compute the row count
72-
if (hasattr(_VSCODE_evalResult, "shape")):
73-
_VSCODE_targetVariable['rowCount'] = _VSCODE_evalResult.shape[0]
74-
elif _VSCODE_targetVariable['type'] == 'list':
75-
_VSCODE_targetVariable['rowCount'] = len(_VSCODE_evalResult)
76-
else:
77-
_VSCODE_targetVariable['rowCount'] = 0
7896

7997
# Transform this back into a string
8098
print(_VSCODE_json.dumps(_VSCODE_targetVariable))
81-
del _VSCODE_targetVariable
99+
del _VSCODE_targetVariable
100+
101+
# Cleanup imports
102+
del _VSCODE_json
103+
del _VSCODE_pd
104+
del _VSCODE_pd_json

‎pythonFiles/datascience/getJupyterVariableDataFrameRows.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@
1515

1616
# Assume we have a dataframe. If not, turn our eval result into a dataframe
1717
_VSCODE_df = _VSCODE_evalResult
18-
if (_VSCODE_targetVariable['type'] == 'list'):
18+
if isinstance(_VSCODE_evalResult, list):
1919
_VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult)
20-
elif (_VSCODE_targetVariable['type'] == 'Series'):
20+
elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series):
2121
_VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult)
22-
elif _VSCODE_targetVariable['type'] == 'dict':
22+
elif isinstance(_VSCODE_evalResult, dict):
2323
_VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult)
2424
_VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult)
2525
elif _VSCODE_targetVariable['type'] == 'ndarray':
@@ -38,4 +38,7 @@
3838
del _VSCODE_endRow
3939
del _VSCODE_startRow
4040
del _VSCODE_rows
41-
del _VSCODE_result
41+
del _VSCODE_result
42+
del _VSCODE_json
43+
del _VSCODE_pd
44+
del _VSCODE_pd_json

‎pythonFiles/tests/ipython/test_variables.py‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,17 @@ def test_dataframe_info(capsys):
5151
se = pd.Series(ls)
5252
np1 = np.array(ls)
5353
np2 = np.array([[1, 2, 3], [4, 5, 6]])
54+
dict1 = {'Name': 'Zara', 'Age': 7, 'Class': 'First'}
5455
obj = {}
56+
col = pd.Series(data=np.random.random_sample((7,))*100)
57+
dfInit = {}
58+
idx = pd.date_range('2007-01-01', periods=7, freq='M')
59+
for i in range(30):
60+
dfInit[i] = col
61+
dfInit['idx'] = idx
62+
df2 = pd.DataFrame(dfInit).set_index('idx')
63+
df3 = df2.iloc[:, [0,1]]
64+
se2 = df2.loc[df2.index[0], :]
5565
''')
5666
vars = get_variables(capsys)
5767
df = get_variable_value(vars, 'df', capsys)
@@ -60,25 +70,38 @@ def test_dataframe_info(capsys):
6070
np2 = get_variable_value(vars, 'np2', capsys)
6171
ls = get_variable_value(vars, 'ls', capsys)
6272
obj = get_variable_value(vars, 'obj', capsys)
73+
df3 = get_variable_value(vars, 'df3', capsys)
74+
se2 = get_variable_value(vars, 'se2', capsys)
75+
dict1 = get_variable_value(vars, 'dict1', capsys)
6376
assert df
6477
assert se
6578
assert np
6679
assert ls
6780
assert obj
68-
verify_dataframe_info(vars, 'df', capsys, True)
69-
verify_dataframe_info(vars, 'se', capsys, True)
70-
verify_dataframe_info(vars, 'np1', capsys, True)
71-
verify_dataframe_info(vars, 'ls', capsys, True)
72-
verify_dataframe_info(vars, 'np2', capsys, True)
73-
verify_dataframe_info(vars, 'obj', capsys, False)
81+
assert df3
82+
assert se2
83+
assert dict1
84+
verify_dataframe_info(vars, 'df', 'index', capsys, True)
85+
verify_dataframe_info(vars, 'se', 'index', capsys, True)
86+
verify_dataframe_info(vars, 'np1', 'index', capsys, True)
87+
verify_dataframe_info(vars, 'ls', 'index', capsys, True)
88+
verify_dataframe_info(vars, 'np2', 'index', capsys, True)
89+
verify_dataframe_info(vars, 'obj', 'index', capsys, False)
90+
verify_dataframe_info(vars, 'df3', 'idx', capsys, True)
91+
verify_dataframe_info(vars, 'se2', 'index', capsys, True)
92+
verify_dataframe_info(vars, 'df2', 'idx', capsys, True)
93+
verify_dataframe_info(vars, 'dict1', 'index', capsys, True)
7494

75-
def verify_dataframe_info(vars, name, capsys, hasInfo):
95+
def verify_dataframe_info(vars, name, indexColumn, capsys, hasInfo):
7696
info = get_data_frame_info(vars, name, capsys)
7797
assert info
7898
assert 'columns' in info
7999
assert len(info['columns']) > 0 if hasInfo else True
80100
assert 'rowCount' in info
81-
assert info['rowCount'] > 0 if hasInfo else info['rowCount'] == 0
101+
if hasInfo:
102+
assert info['rowCount'] > 0
103+
assert info['indexColumn']
104+
assert info['indexColumn'] == indexColumn
82105

83106
@pytest.mark.skipif(not haveIPython,
84107
reason="Can't run variable tests without IPython console")

‎src/client/datascience/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@ export interface IJupyterVariable {
271271
truncated: boolean;
272272
columns?: { key: string; type: string }[];
273273
rowCount?: number;
274+
indexColumn?: string;
274275
}
275276

276277
export const IJupyterVariables = Symbol('IJupyterVariables');

‎src/datascience-ui/data-explorer/cellFormatter.tsx‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,12 @@ import { getLocString } from '../react-common/locReactSide';
1111
interface ICellFormatterProps {
1212
value: string | number | object | boolean;
1313
row: JSONObject | string;
14-
dependentValues: string | undefined;
14+
dependentValues: ICellFormatterMetaData | undefined;
15+
}
16+
17+
export interface ICellFormatterMetaData {
18+
columnType: string;
19+
columnValue: string | number | object | boolean;
1520
}
1621

1722
export class CellFormatter extends React.Component<ICellFormatterProps> {
@@ -29,7 +34,7 @@ export class CellFormatter extends React.Component<ICellFormatterProps> {
2934

3035
// Render based on type
3136
if (this.props.dependentValues && this.props.value !== null) {
32-
switch (this.props.dependentValues) {
37+
switch (this.props.dependentValues.columnType) {
3338
case 'bool':
3439
return this.renderBool(this.props.value as boolean);
3540
break;

‎src/datascience-ui/data-explorer/mainPanel.tsx‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import {
2020
import { IJupyterVariable } from '../../client/datascience/types';
2121
import { IMessageHandler, PostOffice } from '../react-common/postOffice';
2222
import { StyleInjector } from '../react-common/styleInjector';
23-
import { CellFormatter } from './cellFormatter';
23+
import { CellFormatter, ICellFormatterMetaData } from './cellFormatter';
2424
import { EmptyRows } from './emptyRowsView';
2525
import { ProgressBar } from './progressBar';
2626
import { generateTestData } from './testData';
@@ -56,6 +56,7 @@ interface IMainPanelState {
5656
gridHeight: number;
5757
sortDirection: string;
5858
sortColumn: string | number;
59+
indexColumn: string;
5960
}
6061

6162
export class MainPanel extends React.Component<IMainPanelProps, IMainPanelState> implements IMessageHandler {
@@ -81,7 +82,8 @@ export class MainPanel extends React.Component<IMainPanelProps, IMainPanelState>
8182
filters: {},
8283
gridHeight: 100,
8384
sortColumn: 'index',
84-
sortDirection: 'NONE'
85+
sortDirection: 'NONE',
86+
indexColumn: 'index'
8587
};
8688
} else {
8789
this.state = {
@@ -93,7 +95,8 @@ export class MainPanel extends React.Component<IMainPanelProps, IMainPanelState>
9395
filters: {},
9496
gridHeight: 100,
9597
sortColumn: 'index',
96-
sortDirection: 'NONE'
98+
sortDirection: 'NONE',
99+
indexColumn: 'index'
97100
};
98101
}
99102
}
@@ -194,14 +197,16 @@ export class MainPanel extends React.Component<IMainPanelProps, IMainPanelState>
194197
const totalRowCount = variable.rowCount ? variable.rowCount : 0;
195198
const initialRows: JSONArray = [];
196199
const paddedRows = this.padRows(initialRows, totalRowCount);
200+
const indexColumn = variable.indexColumn ? variable.indexColumn : 'index';
197201

198202
this.setState(
199203
{
200204
gridColumns: columns,
201205
actualGridRows: paddedRows,
202206
currentGridRows: paddedRows,
203207
actualRowCount: totalRowCount,
204-
fetchedRowCount: initialRows.length
208+
fetchedRowCount: initialRows.length,
209+
indexColumn: indexColumn
205210
}
206211
);
207212

@@ -303,14 +308,20 @@ export class MainPanel extends React.Component<IMainPanelProps, IMainPanelState>
303308
return [];
304309
}
305310

306-
private getRowMetaData(_row: object, column?: AdazzleReactDataGrid.Column<object>): any {
311+
private getRowMetaData(row: any, column?: AdazzleReactDataGrid.Column<object>): ICellFormatterMetaData {
312+
let columnValue = '';
313+
let columnType = 'string';
307314
if (column) {
308315
const obj = column as any;
309316
if (obj.type) {
310-
return obj.type.toString();
317+
columnType = obj.type.toString();
318+
columnValue = row[obj.name];
311319
}
312320
}
313-
return '';
321+
return {
322+
columnType,
323+
columnValue
324+
};
314325
}
315326

316327
private updateDimensions = () => {
@@ -354,7 +365,7 @@ export class MainPanel extends React.Component<IMainPanelProps, IMainPanelState>
354365

355366
// Default to the index column
356367
if (sortDirection === 'NONE') {
357-
sortColumn = 'index';
368+
sortColumn = this.state.indexColumn;
358369
sortDirection = 'ASC';
359370
}
360371

0 commit comments

Comments
 (0)