Skip to content

Commit 07c1f9f

Browse files
wiedldwiedld
authored
fix(5901): display of query metadata (table and row counts) in QxBuilder and DataExplorer (#6016)
* chore: two additional existence checks, to reduce flake * chore(5901): extract test utils methods for selectBucket() and resetSchema(), and add tests for metadata display in results pane for Qx Builder * fix(5901): the presentation of zero tables, is because we assigned lastTableValue of 0 || -1 to -1, then added +1 to make zero tableCnt whenever we had 1 table (with tableId 0). Additionally, we are handling a use case when we should error (e.g. when table id is a bool) with a presumed incorrect behavior. * feat(5901): message to user when data is truncated, and don't display metadata * chore: skip flaky test and comment reasons for flake (to later debug in another PR). * fix(5901): add existence check for NewScript action. Make flows tableNum be updated only upon state change with results and loading.
1 parent 537e744 commit 07c1f9f

File tree

8 files changed

+207
-73
lines changed

8 files changed

+207
-73
lines changed

‎cypress/e2e/shared/fluxQueryBuilder.test.ts‎

Lines changed: 135 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,18 @@ describe('Script Builder', () => {
2121
const bucketName = 'defbuck'
2222
const measurement = 'ndbc'
2323

24-
const selectSchema = () => {
25-
cy.log('select bucket')
24+
const selectBucket = (bucketName: string) => {
2625
cy.getByTestID('bucket-selector--dropdown-button').click()
27-
cy.getByTestID('bucket-selector--search-bar').type(bucketName)
2826
cy.getByTestID(`bucket-selector--dropdown--${bucketName}`).click()
2927
cy.getByTestID('bucket-selector--dropdown-button').should(
3028
'contain',
3129
bucketName
3230
)
31+
}
32+
33+
const selectSchema = () => {
34+
cy.log('select bucket')
35+
selectBucket(bucketName)
3336

3437
cy.log('select measurement')
3538
cy.getByTestID('measurement-selector--dropdown-button')
@@ -62,6 +65,17 @@ describe('Script Builder', () => {
6265
})
6366
}
6467

68+
const DEFAULT_EDITOR_TEXT =
69+
'// Start by selecting data from the schema browser or typing flux here'
70+
71+
const resetSession = () => {
72+
window.sessionStorage.setItem(
73+
'dataExplorer.schema',
74+
JSON.parse(JSON.stringify(DEFAULT_SCHEMA))
75+
)
76+
window.sessionStorage.setItem('dataExplorer.query', DEFAULT_EDITOR_TEXT)
77+
}
78+
6579
beforeEach(() => {
6680
cy.flush().then(() => {
6781
cy.signin().then(() => {
@@ -72,6 +86,8 @@ describe('Script Builder', () => {
7286
newDataExplorer: true,
7387
}).then(() => {
7488
cy.createBucket(id, name, 'defbuck2')
89+
cy.createBucket(id, name, 'defbuck3')
90+
cy.createBucket(id, name, 'defbuck4')
7591
cy.writeData(writeData, 'defbuck')
7692
cy.wait(100)
7793
cy.getByTestID('flux-query-builder-toggle').then($toggle => {
@@ -91,10 +107,7 @@ describe('Script Builder', () => {
91107

92108
describe('Results display', () => {
93109
beforeEach(() => {
94-
window.sessionStorage.setItem(
95-
'dataExplorer.schema',
96-
JSON.parse(JSON.stringify(DEFAULT_SCHEMA))
97-
)
110+
resetSession()
98111
cy.setFeatureFlags({
99112
schemaComposition: true,
100113
newDataExplorer: true,
@@ -104,6 +117,7 @@ describe('Script Builder', () => {
104117
cy.wait(1200).then(() => {
105118
cy.reload()
106119
cy.getByTestID('flux-sync--toggle')
120+
cy.getByTestID('flux-editor', {timeout: 30000})
107121
})
108122
})
109123

@@ -126,7 +140,6 @@ describe('Script Builder', () => {
126140
})
127141

128142
it('will allow querying of different data ranges', () => {
129-
cy.getByTestID('flux-editor', {timeout: 30000})
130143
selectSchema()
131144
confirmSchemaComposition()
132145

@@ -142,6 +155,112 @@ describe('Script Builder', () => {
142155
cy.getByTestID('time-machine-submit-button').should('exist').click()
143156
cy.wait('@query -15m')
144157
})
158+
159+
describe('data completeness', () => {
160+
const runTest = (
161+
tableCnt: number,
162+
rowCnt: number,
163+
truncated: boolean
164+
) => {
165+
cy.log('confirm on 1hr')
166+
cy.getByTestID('timerange-dropdown').within(() => {
167+
cy.getByTestID('dropdown--button').should('exist').click()
168+
})
169+
cy.getByTestID('dropdown-item-past1h').should('exist').click()
170+
cy.getByTestID('time-machine-submit-button').should('exist').click()
171+
172+
cy.log('run query')
173+
cy.getByTestID('time-machine-submit-button').should('exist').click()
174+
cy.wait('@query -1h')
175+
cy.wait(1000)
176+
177+
cy.log('table metadata displayed to user is correct')
178+
if (truncated) {
179+
cy.getByTestID('query-stat')
180+
.should('contain', 'truncated')
181+
.should('not.contain', 'rows')
182+
.should('not.contain', 'tables')
183+
} else {
184+
cy.getByTestID('query-stat')
185+
.should('not.contain', 'truncated')
186+
.contains(`${tableCnt} tables`)
187+
cy.getByTestID('query-stat').contains(`${rowCnt} rows`)
188+
}
189+
}
190+
191+
it('will return 0 tables and 0 rows, for an empty dataset', () => {
192+
cy.log('select empty dataset')
193+
selectBucket('defbuck3')
194+
cy.getByTestID('flux-editor').contains(`from(bucket: "defbuck3")`, {
195+
timeout: 30000,
196+
})
197+
198+
runTest(0, 0, false)
199+
})
200+
201+
describe('will return 1 table', () => {
202+
beforeEach(() => {
203+
const writeData: string[] = []
204+
writeData.push(`ndbc3,air_temp_degc=70_degrees station_id=${i}`)
205+
cy.writeData(writeData, 'defbuck4')
206+
cy.wait(1200)
207+
})
208+
209+
it('for a dataset with only 1 table', () => {
210+
cy.log('select dataset with 1 table')
211+
selectBucket('defbuck4')
212+
213+
runTest(1, 1, false)
214+
})
215+
})
216+
217+
it('will return the complete dataset for smaller payloads', () => {
218+
cy.log('select smaller dataset')
219+
selectSchema()
220+
confirmSchemaComposition()
221+
222+
runTest(30, 30, false)
223+
})
224+
225+
describe('for larger payloads', () => {
226+
beforeEach(() => {
227+
const writeData: string[] = []
228+
for (let i = 0; i < 500; i++) {
229+
writeData.push(
230+
`ndbc3,air_temp_degc=70_degrees station_id_${i}=${i}`
231+
)
232+
writeData.push(
233+
`ndbc4,air_temp_degc=70_degrees station_id_${i}=${i}`
234+
)
235+
writeData.push(
236+
`ndbc5,air_temp_degc=70_degrees station_id_${i}=${i}`
237+
)
238+
writeData.push(`ndbc6,air_temp_degc=70_degrees station_id=${i}`)
239+
writeData.push(`ndbc7,air_temp_degc=70_degrees station_id=${i}`)
240+
}
241+
cy.writeData(writeData, 'defbuck2')
242+
cy.setFeatureFlags({
243+
newDataExplorer: true,
244+
schemaComposition: true,
245+
dataExplorerCsvLimit: 10000 as any,
246+
}).then(() => {
247+
// cy.wait($time) is necessary to consistently ensure sufficient time for the feature flag override.
248+
// The flag reset happens via redux, (it's not a network request), so we can't cy.wait($intercepted_route).
249+
cy.wait(1200)
250+
})
251+
})
252+
253+
it('will return a truncated dataset rows', () => {
254+
cy.log('select larger dataset')
255+
selectBucket('defbuck2')
256+
cy.getByTestID('flux-editor').contains(`from(bucket: "defbuck2")`, {
257+
timeout: 30000,
258+
})
259+
260+
runTest(3 * 500 + 2, 5 * 500, true)
261+
})
262+
})
263+
})
145264
})
146265

147266
describe('Schema browser', () => {
@@ -287,17 +406,9 @@ describe('Script Builder', () => {
287406
})
288407

289408
describe('Schema Composition', () => {
290-
const DEFAULT_EDITOR_TEXT =
291-
'// Start by selecting data from the schema browser or typing flux here'
292-
293409
beforeEach(() => {
294410
cy.writeData(writeData, 'defbuck2')
295-
296-
window.sessionStorage.setItem(
297-
'dataExplorer.schema',
298-
JSON.parse(JSON.stringify(DEFAULT_SCHEMA))
299-
)
300-
window.sessionStorage.setItem('dataExplorer.query', DEFAULT_EDITOR_TEXT)
411+
resetSession()
301412

302413
cy.setFeatureFlags({
303414
schemaComposition: true,
@@ -337,12 +448,7 @@ describe('Script Builder', () => {
337448
cy.getByTestID('flux-sync--toggle').should('have.class', 'disabled')
338449

339450
cy.log('can still browse schema while diverged')
340-
cy.getByTestID('bucket-selector--dropdown-button').click()
341-
cy.getByTestID(`bucket-selector--dropdown--defbuck2`).click()
342-
cy.getByTestID('bucket-selector--dropdown-button').should(
343-
'contain',
344-
'defbuck2'
345-
)
451+
selectBucket('defbuck2')
346452
})
347453

348454
describe('conditions for divergence:', () => {
@@ -449,11 +555,15 @@ describe('Script Builder', () => {
449555
)
450556

451557
cy.log('click new script, and choose to delete current script')
452-
cy.getByTestID('flux-query-builder--new-script').click({force: true})
558+
cy.getByTestID('flux-query-builder--new-script')
559+
.should('be.visible')
560+
.click()
453561
cy.getByTestID('overlay--container')
454562
.should('be.visible')
455563
.within(() => {
456-
cy.getByTestID('flux-query-builder--no-save').click()
564+
cy.getByTestID('flux-query-builder--no-save')
565+
.should('be.visible')
566+
.click()
457567
})
458568

459569
cy.log('editor text is now empty')

‎src/dataExplorer/components/Results.tsx‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,31 +25,39 @@ import {FluxResult} from 'src/types/flows'
2525

2626
import './Results.scss'
2727
import {isFlagEnabled} from 'src/shared/utils/featureFlag'
28+
import {bytesFormatter} from 'src/shared/copy/notifications'
2829

2930
// simplified version migrated from src/flows/pipes/Table/view.tsx
3031
const QueryStat: FC = () => {
3132
const {result} = useContext(ResultsContext)
3233

3334
const tableColumn = result?.parsed?.table?.getColumn('table') || []
34-
const lastTableValue = tableColumn[tableColumn.length - 1] || -1
35+
const lastTableValue = tableColumn[tableColumn.length - 1]
3536

3637
let tableNum = 0
3738

3839
if (typeof lastTableValue === 'string') {
3940
tableNum = parseInt(lastTableValue) + 1
4041
} else if (typeof lastTableValue === 'boolean') {
41-
tableNum = lastTableValue ? 1 : 0
42-
} else {
43-
// number
42+
console.error('Cannot extract tableId. Check parsed csv output.')
43+
} else if (typeof lastTableValue === 'number') {
4444
tableNum = lastTableValue + 1
4545
}
4646

4747
return (
48-
<div className="query-stat">
49-
<span className="query-stat--bold">{`${tableNum} tables`}</span>
50-
<span className="query-stat--bold">{`${
51-
result?.parsed?.table?.length || 0
52-
} rows`}</span>
48+
<div className="query-stat" data-testid="query-stat">
49+
{result?.truncated ? (
50+
<span className="query-stat--bold">{`Max. display limit exceeded. Result truncated to ${bytesFormatter(
51+
result.bytes
52+
)}.`}</span>
53+
) : (
54+
<>
55+
<span className="query-stat--bold">{`${tableNum} tables`}</span>
56+
<span className="query-stat--bold">{`${
57+
result?.parsed?.table?.length || 0
58+
} rows`}</span>
59+
</>
60+
)}
5361
</div>
5462
)
5563
}

‎src/dataExplorer/components/ResultsPane.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ const ResultsPane: FC = () => {
138138
source: text,
139139
parsed: null,
140140
error: e.message,
141+
truncated: false,
142+
bytes: 0,
141143
})
142144
event('resultReceived', {status: 'error'})
143145
setStatus(RemoteDataState.Error)

‎src/flows/context/results.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ export const ResultsProvider: FC = ({children}) => {
7878
results[id] = {
7979
source: '',
8080
parsed: null,
81+
truncated: false,
82+
bytes: 0,
8183
...result,
8284
}
8385
setResults({...results})

‎src/flows/pipes/Table/view.tsx‎

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import SearchWidget from 'src/shared/components/search_widget/SearchWidget'
1414

1515
// Utilities
1616
import {View} from 'src/visualization'
17+
import {bytesFormatter} from 'src/shared/copy/notifications'
1718

1819
// Types
1920
import {RemoteDataState, SimpleTableViewProperties} from 'src/types'
@@ -36,19 +37,7 @@ const QueryStat: FC = () => {
3637
const {loading, results} = useContext(PipeContext)
3738
const queryStart = useRef(0)
3839
const [processTime, setProcessTime] = useState(0)
39-
let tableNum = 0
40-
41-
const tableColumn = results.parsed.table?.getColumn('table') || []
42-
const lastTableValue = tableColumn[tableColumn.length - 1]
43-
44-
if (typeof lastTableValue === 'string') {
45-
tableNum = parseInt(lastTableValue) + 1
46-
} else if (typeof lastTableValue === 'boolean') {
47-
tableNum = lastTableValue ? 1 : 0
48-
} else {
49-
// number
50-
tableNum = lastTableValue + 1
51-
}
40+
const [tableNum, setTableNum] = useState(0)
5241

5342
useEffect(() => {
5443
if (loading === RemoteDataState.Loading) {
@@ -71,6 +60,23 @@ const QueryStat: FC = () => {
7160
setProcessTime(0)
7261
}, [loading])
7362

63+
useEffect(() => {
64+
if (loading === RemoteDataState.Loading) {
65+
return
66+
}
67+
68+
const tableColumn = results.parsed.table?.getColumn('table') || []
69+
const lastTableValue = tableColumn[tableColumn.length - 1]
70+
71+
if (typeof lastTableValue === 'string') {
72+
setTableNum(parseInt(lastTableValue) + 1)
73+
} else if (typeof lastTableValue === 'boolean') {
74+
console.error('Cannot extract tableId. Check parsed csv output.')
75+
} else if (typeof lastTableValue === 'number') {
76+
setTableNum(lastTableValue + 1)
77+
}
78+
}, [loading, results?.parsed])
79+
7480
const queryStat = {
7581
tableNum,
7682
rowNum: results.parsed.table?.length || 0,
@@ -82,10 +88,18 @@ const QueryStat: FC = () => {
8288
}
8389

8490
return (
85-
<div className="query-stat">
86-
<span className="query-stat--bold">{`${queryStat.tableNum} tables`}</span>
87-
<span className="query-stat--bold">{`${queryStat.rowNum} rows`}</span>
88-
<span className="query-stat--normal">{`${queryStat.processTime} ms`}</span>
91+
<div className="query-stat" data-testid="query-stat">
92+
{results?.truncated ? (
93+
<span className="query-stat--bold">{`Max. display limit exceeded. Result truncated to ${bytesFormatter(
94+
results.bytes
95+
)}.`}</span>
96+
) : (
97+
<>
98+
<span className="query-stat--bold">{`${queryStat.tableNum} tables`}</span>
99+
<span className="query-stat--bold">{`${queryStat.rowNum} rows`}</span>
100+
<span className="query-stat--normal">{`${queryStat.processTime} ms`}</span>
101+
</>
102+
)}
89103
</div>
90104
)
91105
}

0 commit comments

Comments
 (0)