From 17978b2edbbded3fb98ab3792764079592bb10b6 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Wed, 22 May 2024 11:37:51 +0200 Subject: [PATCH] dash-ag-grid performance improvement --- .../dash/layouts/stepper_parts/part_one.py | 90 +- .../dash/modules/table_component/utils.py | 28 +- dev/dev_delta/dev_delta.py.ipynb | 1155 ++++++++++++++++- dev/memory_profiling/app.py | 81 ++ dev/memory_profiling/profile_app.py | 25 + 5 files changed, 1324 insertions(+), 55 deletions(-) create mode 100644 dev/memory_profiling/app.py create mode 100644 dev/memory_profiling/profile_app.py diff --git a/depictio/dash/layouts/stepper_parts/part_one.py b/depictio/dash/layouts/stepper_parts/part_one.py index 7d156fc..17de257 100644 --- a/depictio/dash/layouts/stepper_parts/part_one.py +++ b/depictio/dash/layouts/stepper_parts/part_one.py @@ -22,9 +22,10 @@ def register_callbacks_stepper_part_one(app): Input({"type": "datacollection-selection-label", "index": MATCH}, "value"), Input({"type": "btn-option", "index": MATCH, "value": ALL}, "n_clicks"), State({"type": "last-button", "index": MATCH}, "data"), + State({"type": "workflow-selection-label", "index": MATCH}, "id"), prevent_initial_call=True, ) - def update_step_1(workflow_selection, data_collection_selection, input_btn_values, component_selected): + def update_step_1(workflow_selection, data_collection_selection, input_btn_values, component_selected, id): # Use dcc.Store in store-list to get the latest button clicked using timestamps logger.info(f"CTX Triggered ID: {ctx.triggered_id}") @@ -184,7 +185,7 @@ def update_step_1(workflow_selection, data_collection_selection, input_btn_value cols = get_columns_from_data_collection(workflow_selection, data_collection_selection) logger.info(f"Columns: {cols}") columnDefs = [{"field": c, "headerTooltip": f"Type: {e['type']}"} for c, e in cols.items()] - + # if description in col sub dict, update headerTooltip for col in columnDefs: if "description" in cols[col["field"]] and cols[col["field"]]["description"] is not None: @@ -199,15 +200,23 @@ def update_step_1(workflow_selection, data_collection_selection, input_btn_value # print(df.head(20).to_dict("records")) # cellClicked, cellDoubleClicked, cellRendererData, cellValueChanged, className, columnDefs, columnSize, columnSizeOptions, columnState, csvExportParams, dangerously_allow_code, dashGridOptions, defaultColDef, deleteSelectedRows, deselectAll, detailCellRendererParams, enableEnterpriseModules, exportDataAsCsv, filterModel, getDetailRequest, getDetailResponse, getRowId, getRowStyle, getRowsRequest, getRowsResponse, id, licenseKey, masterDetail, paginationGoTo, paginationInfo, persisted_props, persistence, persistence_type, resetColumnState, rowClass, rowClassRules, rowData, rowModelType, rowStyle, rowTransaction, scrollTo, selectAll, selectedRows, style, suppressDragLeaveHidesColumns, updateColumnState, virtualRowData grid = dag.AgGrid( - id="get-started-example-basic", - # FIXME : full polars - rowData=df.head(2000).to_pandas().to_dict("records"), + id={"type": "get-started-example-basic", "index": id["index"]}, + rowModelType="infinite", columnDefs=columnDefs, dashGridOptions={ "tooltipShowDelay": 500, "pagination": True, "paginationAutoPageSize": False, "animateRows": False, + # The number of rows rendered outside the viewable area the grid renders. + "rowBuffer": 0, + # How many blocks to keep in the store. Default is no limit, so every requested block is kept. + "maxBlocksInCache": 2, + "cacheBlockSize": 100, + "cacheOverflowSize": 2, + "maxConcurrentDatasourceRequests": 2, + "infiniteInitialRowCount": 1, + "rowSelection": "multiple", }, columnSize="sizeToFit", defaultColDef={"resizable": True, "sortable": True, "filter": True}, @@ -286,3 +295,74 @@ def update_step_1(workflow_selection, data_collection_selection, input_btn_value color=component_metadata_dict[component_selected]["color"], leftSection=DashIconify(icon=component_metadata_dict[component_selected]["icon"], width=15, color=component_metadata_dict[component_selected]["color"]), ) + + @app.callback( + Output({"type": "get-started-example-basic", "index": MATCH}, "getRowsResponse"), + Input({"type": "get-started-example-basic", "index": MATCH}, "getRowsRequest"), + Input({"type": "workflow-selection-label", "index": MATCH}, "value"), + Input({"type": "datacollection-selection-label", "index": MATCH}, "value"), + prevent_initial_call=True, + ) + def infinite_scroll(request, workflow_selection, data_collection_selection): + # simulate slow callback + # time.sleep(2) + + if request is None: + return dash.no_update + + if workflow_selection is not None and data_collection_selection is not None: + + workflow_id, data_collection_id = return_mongoid(workflow_tag=workflow_selection, data_collection_tag=data_collection_selection) + + dc_specs = httpx.get( + f"{API_BASE_URL}/depictio/api/v1/datacollections/specs/{workflow_id}/{data_collection_id}", + headers={ + "Authorization": f"Bearer {TOKEN}", + }, + ).json() + + if dc_specs["config"]["type"] == "Table": + df = load_deltatable_lite(workflow_id, data_collection_id) + + partial = df[request["startRow"] : request["endRow"]] + return {"rowData": partial.to_dicts(), "rowCount": df.shape[0]} + else: + return dash.no_update + else: + return dash.no_update + + @app.callback( + Output({"type": "table-aggrid", "index": MATCH}, "getRowsResponse"), + Input({"type": "table-aggrid", "index": MATCH}, "getRowsRequest"), + Input({"type": "stored-metadata-component", "index": MATCH}, "data"), + # prevent_initial_call=True, + ) + def infinite_scroll_component(request, stored_metadata): + # simulate slow callback + # time.sleep(2) + + if request is None: + return dash.no_update + + if stored_metadata is not None: + logger.info(f"Stored metadata: {stored_metadata}") + + workflow_id = stored_metadata["wf_id"] + data_collection_id = stored_metadata["dc_id"] + + dc_specs = httpx.get( + f"{API_BASE_URL}/depictio/api/v1/datacollections/specs/{workflow_id}/{data_collection_id}", + headers={ + "Authorization": f"Bearer {TOKEN}", + }, + ).json() + + if dc_specs["config"]["type"] == "Table": + df = load_deltatable_lite(workflow_id, data_collection_id) + + partial = df[request["startRow"] : request["endRow"]] + return {"rowData": partial.to_dicts(), "rowCount": df.shape[0]} + else: + return dash.no_update + else: + return dash.no_update diff --git a/depictio/dash/modules/table_component/utils.py b/depictio/dash/modules/table_component/utils.py index b6ca459..0016df2 100644 --- a/depictio/dash/modules/table_component/utils.py +++ b/depictio/dash/modules/table_component/utils.py @@ -40,8 +40,10 @@ def build_table_frame(index, children=None): from depictio.api.v1.configs.config import logger + + def build_table(**kwargs): - logger.info("build_table") + logger.info("build_table") # def build_card(index, title, wf_id, dc_id, dc_config, column_name, column_type, aggregation, v, build_frame=False): index = kwargs.get("index") wf_id = kwargs.get("wf_id") @@ -49,11 +51,12 @@ def build_table(**kwargs): dc_config = kwargs.get("dc_config") cols = kwargs.get("cols_json") build_frame = kwargs.get("build_frame", False) + import polars as pl + df = kwargs.get("df", pl.DataFrame()) - - # Load deltatable from the selected data collection - df = load_deltatable_lite(wf_id, dc_id) + if df.is_empty(): + df = load_deltatable_lite(wf_id, dc_id) # Add dah aggrid filters to the columns for c in cols: @@ -69,17 +72,26 @@ def build_table(**kwargs): # print(cols) columnDefs = [{"field": c, "headerTooltip": f"Column type: {e['type']}", "filter": e["filter"]} for c, e in cols.items()] - # TODO: use other properties of Dash AgGrid # Prepare ag grid table table_aggrid = dag.AgGrid( id={"type": "table-aggrid", "index": str(index)}, - rowData=df.to_dict("records"), + # rowData=df.to_pandas().to_dict("records"), + rowModelType="infinite", columnDefs=columnDefs, dashGridOptions={ "tooltipShowDelay": 500, "pagination": True, - # "paginationAutoPageSize": False, - # "animateRows": False, + "paginationAutoPageSize": False, + "animateRows": False, + # The number of rows rendered outside the viewable area the grid renders. + "rowBuffer": 0, + # How many blocks to keep in the store. Default is no limit, so every requested block is kept. + "maxBlocksInCache": 2, + "cacheBlockSize": 100, + "cacheOverflowSize": 2, + "maxConcurrentDatasourceRequests": 2, + "infiniteInitialRowCount": 1, + "rowSelection": "multiple", }, # columnSize="sizeToFit", defaultColDef={"resizable": True, "sortable": True, "filter": True}, diff --git a/dev/dev_delta/dev_delta.py.ipynb b/dev/dev_delta/dev_delta.py.ipynb index bc88b5b..75732b5 100644 --- a/dev/dev_delta/dev_delta.py.ipynb +++ b/dev/dev_delta/dev_delta.py.ipynb @@ -725,49 +725,59 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (3_072, 16)
depictio_run_idsamplecellmedbinmappedsupplduplmapqread2goodpass1nb_pnb_rnb_abamdepictio_aggregation_time
strstrstri64i64i64i64i64i64i64i64f64f64f64strstr
"2019-05-10-HJL…"HG02059102""HG02059102PE20…3924357791306111756915772257830158088110.21413410.9170.1"/scratch/twebe…"2024-05-07 09:…
"2019-05-10-HJL…"HG02059102""HG02059102PE20…5435069431909166993523136880051680321510.21413415.09140.1"/scratch/twebe…"2024-05-07 09:…
"2019-05-10-HJL…"HG02059102""HG02059102PE20…3522175711330102787514813951918952103810.2141349.792670.1"/scratch/twebe…"2024-05-07 09:…
"2019-05-10-HJL…"HG02059102""HG02059102PE20…5233162171802155053222456976838577092910.21413414.48030.1"/scratch/twebe…"2024-05-07 09:…
"2019-05-10-HJL…"HG02059102""HG02059102PE20…32188637165887185010608945329645447810.2141348.574640.1"/scratch/twebe…"2024-05-07 09:…
…………………………………………
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…2728009542025182947116649440070140226310.119423.805190.1"/scratch/twebe…"2024-05-07 09:…
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…143501343027164403853612272101615210748710.1194220.01630.1"/scratch/twebe…"2024-05-07 09:…
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…2631071892636212643617209840201540400410.119423.824110.1"/scratch/twebe…"2024-05-07 09:…
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…555670859936472231017801088026210.119420.7614790.1"/scratch/twebe…"2024-05-07 09:…
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…22001802638845911276500735010900.00.00.0"/scratch/twebe…"2024-05-07 09:…
" + "shape: (6_720, 16)
depictio_run_idsamplecellmedbinmappedsupplduplmapqread2goodpass1nb_pnb_rnb_abamdepictio_aggregation_time
strstrstri64i64i64i64i64i64i64i64f64f64f64strstr
"2019-01-02-HGN…"SSC11163x02""SSC11163x02PE2…01530816312813465418540000.00.00.0"/scratch/twebe…"2024-05-17 22:…
"2019-01-02-HGN…"SSC11163x02""SSC11163x02PE2…2821008801104113224712331142148042273810.09641473.115950.1"/scratch/twebe…"2024-05-17 22:…
"2019-01-02-HGN…"SSC11163x02""SSC11163x02PE2…2821000031006114088112150541760841900310.09641473.089880.1"/scratch/twebe…"2024-05-17 22:…
"2019-01-02-HGN…"SSC11163x02""SSC11163x02PE2…97091774083773053707214701014738210.09641471.087030.1"/scratch/twebe…"2024-05-17 22:…
"2019-01-02-HGN…"SSC11163x02""SSC11163x02PE2…86405654063409043746813069713109010.09641470.964240.1"/scratch/twebe…"2024-05-17 22:…
…………………………………………
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…2728009542025182947116649440070140226310.119423.805190.1"/scratch/twebe…"2024-05-17 22:…
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…143501343027164403853612272101615210748710.1194220.01630.1"/scratch/twebe…"2024-05-17 22:…
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…2631071892636212643617209840201540400410.119423.824110.1"/scratch/twebe…"2024-05-17 22:…
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…555670859936472231017801088026210.119420.7614790.1"/scratch/twebe…"2024-05-17 22:…
"2019-12-16-HWV…"HG002x02""HG002x02PE2049…22001802638845911276500735010900.00.00.0"/scratch/twebe…"2024-05-17 22:…
" ], "text/plain": [ - "shape: (3_072, 16)\n", - "┌─────────────┬────────────┬─────────────┬────────┬───┬──────────┬──────┬─────────────┬────────────┐\n", - "│ depictio_ru ┆ sample ┆ cell ┆ medbin ┆ … ┆ nb_r ┆ nb_a ┆ bam ┆ depictio_a │\n", - "│ n_id ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ ggregation │\n", - "│ --- ┆ str ┆ str ┆ i64 ┆ ┆ f64 ┆ f64 ┆ str ┆ _time │\n", - "│ str ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ --- │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ str │\n", - "╞═════════════╪════════════╪═════════════╪════════╪═══╪══════════╪══════╪═════════════╪════════════╡\n", - "│ 2019-05-10- ┆ HG02059102 ┆ HG02059102P ┆ 39 ┆ … ┆ 10.917 ┆ 0.1 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HJLFKAFXY ┆ ┆ E20301 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "│ 2019-05-10- ┆ HG02059102 ┆ HG02059102P ┆ 54 ┆ … ┆ 15.0914 ┆ 0.1 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HJLFKAFXY ┆ ┆ E20302 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "│ 2019-05-10- ┆ HG02059102 ┆ HG02059102P ┆ 35 ┆ … ┆ 9.79267 ┆ 0.1 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HJLFKAFXY ┆ ┆ E20303 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "│ 2019-05-10- ┆ HG02059102 ┆ HG02059102P ┆ 52 ┆ … ┆ 14.4803 ┆ 0.1 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HJLFKAFXY ┆ ┆ E20304 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "│ 2019-05-10- ┆ HG02059102 ┆ HG02059102P ┆ 32 ┆ … ┆ 8.57464 ┆ 0.1 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HJLFKAFXY ┆ ┆ E20305 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 27 ┆ … ┆ 3.80519 ┆ 0.1 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HWVTJAFXY ┆ ┆ 0492 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 143 ┆ … ┆ 20.0163 ┆ 0.1 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HWVTJAFXY ┆ ┆ 0493 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 26 ┆ … ┆ 3.82411 ┆ 0.1 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HWVTJAFXY ┆ ┆ 0494 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 5 ┆ … ┆ 0.761479 ┆ 0.1 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HWVTJAFXY ┆ ┆ 0495 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 2 ┆ … ┆ 0.0 ┆ 0.0 ┆ /scratch/tw ┆ 2024-05-07 │\n", - "│ HWVTJAFXY ┆ ┆ 0496 ┆ ┆ ┆ ┆ ┆ eber/DATA/M ┆ 09:33:56 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ C_DATA/STO… ┆ │\n", - "└─────────────┴────────────┴─────────────┴────────┴───┴──────────┴──────┴─────────────┴────────────┘" + "shape: (6_720, 16)\n", + "┌─────────────┬─────────────┬─────────────┬────────┬───┬──────────┬──────┬────────────┬────────────┐\n", + "│ depictio_ru ┆ sample ┆ cell ┆ medbin ┆ … ┆ nb_r ┆ nb_a ┆ bam ┆ depictio_a │\n", + "│ n_id ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ ggregation │\n", + "│ --- ┆ str ┆ str ┆ i64 ┆ ┆ f64 ┆ f64 ┆ str ┆ _time │\n", + "│ str ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ --- │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ str │\n", + "╞═════════════╪═════════════╪═════════════╪════════╪═══╪══════════╪══════╪════════════╪════════════╡\n", + "│ 2019-01-02- ┆ SSC11163x02 ┆ SSC11163x02 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HGNWMAFXY ┆ ┆ PE20301 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "│ 2019-01-02- ┆ SSC11163x02 ┆ SSC11163x02 ┆ 28 ┆ … ┆ 3.11595 ┆ 0.1 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HGNWMAFXY ┆ ┆ PE20302 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "│ 2019-01-02- ┆ SSC11163x02 ┆ SSC11163x02 ┆ 28 ┆ … ┆ 3.08988 ┆ 0.1 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HGNWMAFXY ┆ ┆ PE20303 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "│ 2019-01-02- ┆ SSC11163x02 ┆ SSC11163x02 ┆ 9 ┆ … ┆ 1.08703 ┆ 0.1 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HGNWMAFXY ┆ ┆ PE20304 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "│ 2019-01-02- ┆ SSC11163x02 ┆ SSC11163x02 ┆ 8 ┆ … ┆ 0.96424 ┆ 0.1 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HGNWMAFXY ┆ ┆ PE20305 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 27 ┆ … ┆ 3.80519 ┆ 0.1 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HWVTJAFXY ┆ ┆ 0492 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 143 ┆ … ┆ 20.0163 ┆ 0.1 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HWVTJAFXY ┆ ┆ 0493 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 26 ┆ … ┆ 3.82411 ┆ 0.1 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HWVTJAFXY ┆ ┆ 0494 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 5 ┆ … ┆ 0.761479 ┆ 0.1 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HWVTJAFXY ┆ ┆ 0495 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "│ 2019-12-16- ┆ HG002x02 ┆ HG002x02PE2 ┆ 2 ┆ … ┆ 0.0 ┆ 0.0 ┆ /scratch/t ┆ 2024-05-17 │\n", + "│ HWVTJAFXY ┆ ┆ 0496 ┆ ┆ ┆ ┆ ┆ weber/DATA ┆ 22:21:51 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ /MC_DATA/S ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ TO… ┆ │\n", + "└─────────────┴─────────────┴─────────────┴────────┴───┴──────────┴──────┴────────────┴────────────┘" ] }, "execution_count": 2, @@ -790,7 +800,1068 @@ " \"AWS_ALLOW_HTTP\" : \"true\",\n", " \"AWS_S3_ALLOW_UNSAFE_RENAME\": \"true\"\n", "}\n", - "pl.scan_delta(\"s3://depictio-bucket/662acab447d998cd889678c1/6639f584707fcacf576dd878/6639f584707fcacf576dd879/\", storage_options=storage_options).filter\n" + "df = pl.scan_delta(\"s3://depictio-bucket/662acab447d998cd889678c1/6647d03b57d1fae8d89dce8d/6647d03b57d1fae8d89dce8e/\", storage_options=storage_options).collect()\n", + "df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'0.20.14'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pl.__version__" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "polars.series.series.Series" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "type(df[\"cell\"].unique())" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['SSC1163101PE20415',\n", + " 'HG04217x02PE20361',\n", + " 'HG00864x02PE20326',\n", + " 'BoM662Gx01PE20528',\n", + " 'GM19434x02PE20586',\n", + " 'NLEx02PE20336',\n", + " 'ckAML02x01PE20381',\n", + " 'BOM71Gx03PE20370',\n", + " 'WCHSPCCB02x01PE20586',\n", + " 'BoM69Nx01PE20480',\n", + " 'CB0003x01PE20310',\n", + " 'M56x1x02PE20446',\n", + " 'BOM65G03x03PE20412',\n", + " 'ADCr11xGFPxmCherryPE20362',\n", + " 'BoM66Gx01PE20345',\n", + " 'BoM66Gx01PE20312',\n", + " 'R8x28PE20580',\n", + " 'GM18534Bx02PE20303',\n", + " 'LFSL41midx01PE20522',\n", + " 'R8x28PE20556',\n", + " 'BoM92Nx02PE20414',\n", + " 'HG02818x02PE20369',\n", + " 'CB0003x01PE20388',\n", + " 'CAST1D1x02PE20481',\n", + " 'BoM712Gx01PE20458',\n", + " 'BoM712Gx01PE20415',\n", + " 'HG01573x02PE20387',\n", + " 'HG03065x02PE20534',\n", + " 'HG03009x02PE20366',\n", + " 'HG03065x02PE20588',\n", + " 'R8x28PE20537',\n", + " 'CB0003x01PE20391',\n", + " 'HG04217x02PE20352',\n", + " 'CAST1D1x02PE20493',\n", + " 'GM19983x02PE20455',\n", + " 'HG00171Ax02PE20470',\n", + " 'NLEx02PE20321',\n", + " 'BOMCHIP76Nx02PE20406',\n", + " 'BoM47Nx01PE20563',\n", + " 'HG01505x01PE20432',\n", + " 'BoM63Gx02PE20571',\n", + " 'GM18534Bx02PE20321',\n", + " 'BoM69Nx01PE20457',\n", + " 'BoM70Nx01PE20437',\n", + " 'HG04217x02PE20386',\n", + " 'BoM712Gx01PE20454',\n", + " 'ckAML01x01PE20377',\n", + " 'HG01596x02PE20501',\n", + " 'GM20847Bx02PE20434',\n", + " 'HG03683x01PE20427',\n", + " 'HG03371x02PE20506',\n", + " 'HG02059102PE20339',\n", + " 'BoM24HPE20446',\n", + " 'SSC11168x02PE20453',\n", + " 'BoM66Gx01PE20336',\n", + " 'R8x28PE20540',\n", + " 'SSC11163x02PE20324',\n", + " 'HG02587x02PE20395',\n", + " 'HG03683x01PE20406',\n", + " 'BoM47Nx01PE20590',\n", + " 'GM19650Ax02PE20531',\n", + " 'HG01352x02PE20432',\n", + " 'M56x1x02PE20461',\n", + " 'HG00268x01PE20455',\n", + " 'HG02018x01PE20434',\n", + " 'HG01573x02PE20395',\n", + " 'BOM65G03x03PE20435',\n", + " 'B6xMEFxP5x02PE20371',\n", + " 'AML1Nx01PE20346',\n", + " 'BoM66Gx01PE20387',\n", + " 'ckAML02x01PE20372',\n", + " 'GM20847Bx02PE20435',\n", + " 'HG03009x02PE20346',\n", + " 'ADCr11xGFPxmCherryPE20353',\n", + " 'B6CAST1F1x01PE20316',\n", + " 'HG02106x02PE20323',\n", + " 'ckAML01x01PE20366',\n", + " 'HG00864x02PE20307',\n", + " 'ADCr11xCdt1xCdc6PE20414',\n", + " 'WCHSPCS03x02PE20585',\n", + " 'GM20847Bx02PE20407',\n", + " 'WholeHSPCS03x01PE20536',\n", + " 'HG01114x02PE20339',\n", + " 'HG02106x02PE20377',\n", + " 'ckAML02x01PE20328',\n", + " 'ckAML03x01PE20421',\n", + " 'HG002x02PE20456',\n", + " 'BoM63Gx02PE20517',\n", + " 'HG01114x02PE20324',\n", + " 'HG03683x01PE20496',\n", + " 'HG02059102PE20384',\n", + " 'HG02059102PE20350',\n", + " 'BoM92Nx02PE20462',\n", + " 'HG00268x01PE20479',\n", + " 'BoM712Gx01PE20483',\n", + " 'M56x1x02PE20418',\n", + " 'WCHSPCCB02x01PE20589',\n", + " 'BoM24HPE20460',\n", + " 'BoM66Gx01PE20359',\n", + " 'HG02492x02PE20462',\n", + " 'WCHSPCCB02x01PE20538',\n", + " 'HG00268x01PE20426',\n", + " 'HG01573x02PE20308',\n", + " 'HG19036Bx02PE20582',\n", + " 'BoM63Gx02PE20554',\n", + " 'BoM47Nx01PE20566',\n", + " 'HG01596x02PE20543',\n", + " 'ckAML03x01PE20425',\n", + " 'R8x28PE20513',\n", + " 'HG02587x02PE20384',\n", + " 'GM18534Bx02PE20326',\n", + " 'BOMCHIP76Nx02PE20437',\n", + " 'HG02059102PE20332',\n", + " 'ADCr11xGFPxmCherryPE20355',\n", + " 'SSC11168x02PE20429',\n", + " 'BoM66Gx01PE20383',\n", + " 'HG00268x01PE20473',\n", + " 'HG02106x02PE20371',\n", + " 'HG00864x02PE20337',\n", + " 'HG01596x02PE20519',\n", + " 'ckAML03x01PE20475',\n", + " 'HG02587x02PE20330',\n", + " 'HG00268x01PE20435',\n", + " 'GM19983x02PE20413',\n", + " 'BOM71Gx02PE20323',\n", + " 'HG00864x02PE20392',\n", + " 'ckAML03x01PE20484',\n", + " 'BoM24HPE20432',\n", + " 'SSC1163101PE20465',\n", + " 'BoM70Nx01PE20456',\n", + " 'HG00268x01PE20409',\n", + " 'CB0003x01PE20358',\n", + " 'GM19983x02PE20406',\n", + " 'BoM662Gx01PE20543',\n", + " 'HG00268x01PE20457',\n", + " 'BOM71Gx02PE20338',\n", + " 'HG03683x01PE20482',\n", + " 'BoM69Nx01PE20449',\n", + " 'BoM69Nx01PE20447',\n", + " 'BOM71Gx03PE20322',\n", + " 'LFSL41midx01PE20595',\n", + " 'HG02818x02PE20330',\n", + " 'ADCr11xCdt1xCdc6PE20445',\n", + " 'BOMCHIP76N02x01PE20323',\n", + " 'CB0003x01PE20396',\n", + " 'SSC1163101PE20405',\n", + " 'HG01352x02PE20434',\n", + " 'WCHSPCCB02x01PE20577',\n", + " 'S14960hx04PE20458',\n", + " 'BOM71Gx03PE20356',\n", + " 'HG02492x02PE20492',\n", + " 'HG19036Bx02PE20367',\n", + " 'WholeHSPCS03x01PE20574',\n", + " 'SSC1163101PE20406',\n", + " 'BoM47Nx01PE20518',\n", + " 'ADCr11xGFPxmCherryPE20337',\n", + " 'HG01596x02PE20584',\n", + " 'BOM65G03x03PE20427',\n", + " 'BoM63Gx02PE20564',\n", + " 'WCxHSPCsPE20355',\n", + " 'B6CAST1F1x01PE20348',\n", + " 'BoM712Gx01PE20410',\n", + " 'SSC11163x02PE20303',\n", + " 'HG02587x02PE20328',\n", + " 'S149x48hx02PE20349',\n", + " 'HG19036Bx02PE20591',\n", + " 'HG01505x01PE20477',\n", + " 'BoM92Nx02PE20406',\n", + " 'B6CAST1F1x01PE20369',\n", + " 'HG02587x02PE20363',\n", + " 'HG03009x02PE20307',\n", + " 'HG03065x02PE20551',\n", + " 'WholeHSPCS03x01PE20511',\n", + " 'ckAML01x01PE20336',\n", + " 'BoM712Gx01PE20430',\n", + " 'GM19434x02PE20529',\n", + " 'B6xMEFxP5x02PE20340',\n", + " 'BoM24HPE20443',\n", + " 'SSC11163x02PE20367',\n", + " 'BOM65G03x03PE20472',\n", + " 'LFSL41midx01PE20502',\n", + " 'HG02106x02PE20331',\n", + " 'ckAML01x01PE20325',\n", + " 'WholeHSPCS03x01PE20503',\n", + " 'WCxHSPCsPE20306',\n", + " 'ckAML02x01PE20327',\n", + " 'BOM71Gx03PE20378',\n", + " 'HG19036Bx02PE20542',\n", + " 'HG03371x02PE20544',\n", + " 'BoM47Nx01PE20536',\n", + " 'ckAML03x01PE20443',\n", + " 'BoM66Gx01PE20319',\n", + " 'HG03683x01PE20488',\n", + " 'GM20847Bx02PE20423',\n", + " 'BOM71Gx02PE20318',\n", + " 'SSC11168x02PE20445',\n", + " 'HG02492x02PE20455',\n", + " 'HG01505x01PE20431',\n", + " 'BoM662Gx01PE20585',\n", + " 'WholeHSPCS03x01PE20557',\n", + " 'SSC11163x02PE20380',\n", + " 'BoM92Nx02PE20476',\n", + " 'HG02818x02PE20301',\n", + " 'HG03732x01PE20575',\n", + " 'S149x48hx02PE20390',\n", + " 'HG00096x02PE20330',\n", + " 'WholeHSPCS03x01PE20554',\n", + " 'LFSL41midx01PE20543',\n", + " 'HG03065x02PE20508',\n", + " 'HG03065x02PE20564',\n", + " 'BoM47Nx01PE20517',\n", + " 'S14960hx04PE20465',\n", + " 'GM20847Bx02PE20405',\n", + " 'BoM70Nx01PE20431',\n", + " 'HG02106x02PE20312',\n", + " 'HG01573x02PE20375',\n", + " 'HG19036Bx02PE20445',\n", + " 'HG19036Bx02PE20319',\n", + " 'BoM92Nx02PE20422',\n", + " 'HG19036Bx02PE20532',\n", + " 'GM19650Ax02PE20524',\n", + " 'R8x28PE20557',\n", + " 'HG03009x02PE20339',\n", + " 'HG00864x02PE20348',\n", + " 'HG01596x02PE20566',\n", + " 'HG00268x01PE20424',\n", + " 'BoM662Gx01PE20539',\n", + " 'HG00096x02PE20381',\n", + " 'S14960hx04PE20426',\n", + " 'HG01114x02PE20360',\n", + " 'HG03009x02PE20368',\n", + " 'WCxHSPCsPE20344',\n", + " 'GM19434x02PE20531',\n", + " 'HG01505x01PE20415',\n", + " 'ckAML02x01PE20302',\n", + " 'HG01114x02PE20379',\n", + " 'SSC1163101PE20435',\n", + " 'CAST1D1x02PE20461',\n", + " 'HG02059102PE20346',\n", + " 'HG19036Bx02PE20488',\n", + " 'HG19036Bx02PE20535',\n", + " 'WCHSPCCB02x01PE20510',\n", + " 'GM18534Bx02PE20306',\n", + " 'BoM47Nx01PE20551',\n", + " 'GM19650Ax02PE20591',\n", + " 'AML1Nx01PE20326',\n", + " 'HG02492x02PE20454',\n", + " 'HG02818x02PE20346',\n", + " 'BOMCHIP76N02x01PE20305',\n", + " 'HG19036Bx02PE20471',\n", + " 'LFSL41midx01PE20545',\n", + " 'HG00864x02PE20323',\n", + " 'BOMH12x01PE20479',\n", + " 'HG19036Bx02PE20521',\n", + " 'HG002x02PE20421',\n", + " 'BoM63Gx02PE20515',\n", + " 'BOM71Gx02PE20312',\n", + " 'SSC11163x02PE20366',\n", + " 'HG02059102PE20321',\n", + " 'SSC1163101PE20482',\n", + " 'BOMH12x01PE20449',\n", + " 'WCxHSPCsPE20357',\n", + " 'BoM70Nx01PE20422',\n", + " 'HG02106x02PE20396',\n", + " 'GM18939x02PE20468',\n", + " 'B6xMEFxP5x02PE20377',\n", + " 'ckAML01x01PE20329',\n", + " 'BoM92Nx02PE20416',\n", + " 'LFSL41midx01PE20520',\n", + " 'M56x1x02PE20432',\n", + " 'HG00268x01PE20485',\n", + " 'HG01573x02PE20323',\n", + " 'B6xMEFxP5x02PE20383',\n", + " 'HG02059102PE20315',\n", + " 'HG00171Ax02PE20485',\n", + " 'HG03732x01PE20539',\n", + " 'NLEx02PE20387',\n", + " 'HG03065x02PE20520',\n", + " 'ckAML01x01PE20367',\n", + " 'HG02587x02PE20310',\n", + " 'WCHSPCS03x02PE20556',\n", + " 'ckAML01x01PE20334',\n", + " 'ckAML01x01PE20354',\n", + " 'ckAML01x01PE20389',\n", + " 'HG03683x01PE20401',\n", + " 'GM20847Bx02PE20436',\n", + " 'WholeHSPCS03x01PE20532',\n", + " 'HG002x01PE20356',\n", + " 'LFSL41midx01PE20538',\n", + " 'BOMCHIP76N02x01PE20345',\n", + " 'HG19036Bx02PE20363',\n", + " 'HG02818x02PE20367',\n", + " 'SSC1163101PE20472',\n", + " 'HG03371x02PE20543',\n", + " 'BoM47Nx01PE20572',\n", + " 'LFSL41midx01PE20594',\n", + " 'ckAML01x01PE20385',\n", + " 'ADCr11xCdt1xCdc6PE20469',\n", + " 'HG03732x01PE20523',\n", + " 'GM18939x02PE20439',\n", + " 'ADCr11xCdt1xCdc6PE20438',\n", + " 'GM19434x02PE20547',\n", + " 'BoM24HPE20404',\n", + " 'WholeHSPCS03x01PE20569',\n", + " 'HG02106x02PE20394',\n", + " 'GM20847Bx02PE20422',\n", + " 'WCHSPCCB02x01PE20585',\n", + " 'BoM63Gx02PE20592',\n", + " 'BOMCHIP76Nx02PE20434',\n", + " 'BOM71Gx02PE20347',\n", + " 'NLEx02PE20303',\n", + " 'HG002x02PE20409',\n", + " 'BOMH12x01PE20405',\n", + " 'HG03683x01PE20437',\n", + " 'ADCr11xGFPxmCherryPE20323',\n", + " 'CB0003x01PE20351',\n", + " 'HG04217x02PE20342',\n", + " 'SSC1163101PE20470',\n", + " 'B6CAST1F1x01PE20354',\n", + " 'WCHSPCS03x02PE20581',\n", + " 'HG02059102PE20365',\n", + " 'HG02018x01PE20487',\n", + " 'HG02059102PE20320',\n", + " 'HG02106x02PE20318',\n", + " 'GM19983x02PE20457',\n", + " 'BoM47Nx01PE20502',\n", + " 'BoM63Gx02PE20555',\n", + " 'BOMCHIP76Nx02PE20454',\n", + " 'HG02492x02PE20465',\n", + " 'BoM63Gx02PE20596',\n", + " 'S14960hx04PE20421',\n", + " 'HG02587x02PE20305',\n", + " 'WCHSPCS03x02PE20531',\n", + " 'NLEx02PE20342',\n", + " 'HG002x02PE20441',\n", + " 'BOM71Gx03PE20334',\n", + " 'CB0003x01PE20333',\n", + " 'BoM63Gx02PE20593',\n", + " 'HG02018x01PE20447',\n", + " 'BOMCHIP76N02x01PE20328',\n", + " 'HG03683x01PE20403',\n", + " 'ADCr11xCdt1xCdc6PE20401',\n", + " 'BOMCHIP76N02x01PE20365',\n", + " 'WCxHSPCsPE20302',\n", + " 'NLEx02PE20378',\n", + " 'ADCr11xGFPxmCherryPE20317',\n", + " 'HG02018x01PE20411',\n", + " 'WCHSPCCB02x01PE20571',\n", + " 'BoM24HPE20403',\n", + " 'HG03732x01PE20535',\n", + " 'HG03371x02PE20542',\n", + " 'SSC1163101PE20467',\n", + " 'HG00171Ax02PE20401',\n", + " 'BoM69Nx01PE20461',\n", + " 'NLEx02PE20379',\n", + " 'HG00096x02PE20337',\n", + " 'HG03371x02PE20567',\n", + " 'BOM65G03x03PE20475',\n", + " 'BoM69Nx01PE20458',\n", + " 'HG00096x02PE20355',\n", + " 'BOMH12x01PE20470',\n", + " 'HG002x01PE20367',\n", + " 'HG002x02PE20407',\n", + " 'HG19036Bx02PE20563',\n", + " 'ckAML02x01PE20342',\n", + " 'HG01114x02PE20311',\n", + " 'BoM63Gx02PE20586',\n", + " 'HG03009x02PE20309',\n", + " 'HG02059102PE20329',\n", + " 'GM19983x02PE20435',\n", + " 'NLEx02PE20328',\n", + " 'SSC1163101PE20407',\n", + " 'CAST1D1x02PE20491',\n", + " 'BoM63Gx02PE20577',\n", + " 'BOM71Gx02PE20378',\n", + " 'ADCr11xCdt1xCdc6PE20455',\n", + " 'HG19036Bx02PE20503',\n", + " 'SSC11168x02PE20408',\n", + " 'GM19983x02PE20459',\n", + " 'GM18534Bx02PE20379',\n", + " 'GM20847Bx02PE20456',\n", + " 'HG002x01PE20385',\n", + " 'HG03371x02PE20541',\n", + " 'WCxHSPCsPE20335',\n", + " 'BoM69Nx01PE20443',\n", + " 'BoM662Gx01PE20545',\n", + " 'BoM66Gx01PE20368',\n", + " 'BoM69Nx01PE20413',\n", + " 'HG03371x02PE20540',\n", + " 'HG002x01PE20396',\n", + " 'GM19983x02PE20463',\n", + " 'LFSL41midx01PE20550',\n", + " 'S14960hx04PE20409',\n", + " 'BOM71Gx02PE20390',\n", + " 'BoM63Gx02PE20570',\n", + " 'BOMH12x01PE20402',\n", + " 'HG00171Ax02PE20454',\n", + " 'WCHSPCS03x02PE20568',\n", + " 'HG01596x02PE20529',\n", + " 'HG01596x02PE20545',\n", + " 'ckAML02x01PE20314',\n", + " 'HG02587x02PE20333',\n", + " 'GM19650Ax02PE20549',\n", + " 'HG19036Bx02PE20486',\n", + " 'HG01573x02PE20359',\n", + " 'S149x48hx02PE20367',\n", + " 'HG01352x02PE20451',\n", + " 'HG00096x02PE20328',\n", + " 'AML1Nx01PE20302',\n", + " 'ADCr11xGFPxmCherryPE20315',\n", + " 'HG03009x02PE20302',\n", + " 'HG02492x02PE20485',\n", + " 'HG03065x02PE20529',\n", + " 'HG00096x02PE20317',\n", + " 'BoM662Gx01PE20519',\n", + " 'SSC11163x02PE20360',\n", + " 'S14960hx04PE20430',\n", + " 'WCxHSPCsPE20387',\n", + " 'BoM24HPE20414',\n", + " 'B6CAST1F1x01PE20302',\n", + " 'HG01114x02PE20352',\n", + " 'GM19434x02PE20551',\n", + " 'HG03732x01PE20541',\n", + " 'HG19036Bx02PE20377',\n", + " 'BOM71Gx02PE20339',\n", + " 'HG03065x02PE20511',\n", + " 'HG03683x01PE20402',\n", + " 'SSC11163x02PE20329',\n", + " 'HG02492x02PE20467',\n", + " 'S14960hx04PE20493',\n", + " 'NLEx02PE20373',\n", + " 'S149x48hx02PE20388',\n", + " 'HG19036Bx02PE20432',\n", + " 'HG04217x02PE20379',\n", + " 'HG01505x01PE20464',\n", + " 'BOMCHIP76Nx02PE20420',\n", + " 'BOMH12x01PE20484',\n", + " 'WCHSPCS03x02PE20577',\n", + " 'GM19983x02PE20451',\n", + " 'BoM47Nx01PE20577',\n", + " 'BOMH12x01PE20417',\n", + " 'HG02492x02PE20479',\n", + " 'GM19650Ax02PE20539',\n", + " 'GM19650Ax02PE20555',\n", + " 'HG002x01PE20394',\n", + " 'BOM71Gx02PE20306',\n", + " 'HG02106x02PE20336',\n", + " 'WCxHSPCsPE20362',\n", + " 'HG04217x02PE20376',\n", + " 'HG02018x01PE20485',\n", + " 'NLEx02PE20388',\n", + " 'B6xMEFxP5x02PE20363',\n", + " 'BoM92Nx02PE20482',\n", + " 'HG01505x01PE20457',\n", + " 'HG19036Bx02PE20537',\n", + " 'ckAML03x01PE20453',\n", + " 'ckAML02x01PE20371',\n", + " 'HG02818x02PE20327',\n", + " 'HG02106x02PE20389',\n", + " 'HG19036Bx02PE20559',\n", + " 'B6xMEFxP5x02PE20349',\n", + " 'HG00864x02PE20384',\n", + " 'BOMH12x01PE20451',\n", + " 'S14960hx04PE20419',\n", + " 'HG02818x02PE20371',\n", + " 'HG02492x02PE20458',\n", + " 'HG03683x01PE20494',\n", + " 'HG00171Ax02PE20414',\n", + " 'GM19983x02PE20403',\n", + " 'GM19434x02PE20559',\n", + " 'GM19983x02PE20458',\n", + " 'HG03683x01PE20476',\n", + " 'B6CAST1F1x01PE20378',\n", + " 'HG02492x02PE20436',\n", + " 'ADCr11xGFPxmCherryPE20364',\n", + " 'BOMH12x01PE20476',\n", + " 'BoM69Nx01PE20408',\n", + " 'HG01573x02PE20378',\n", + " 'GM19650Ax02PE20595',\n", + " 'GM18939x02PE20412',\n", + " 'LFSL41midx01PE20526',\n", + " 'BOM71Gx02PE20316',\n", + " 'ckAML01x01PE20322',\n", + " 'GM18534Bx02PE20352',\n", + " 'BoM712Gx01PE20495',\n", + " 'M56x1x02PE20440',\n", + " 'B6CAST1F1x01PE20307',\n", + " 'BOMH12x01PE20412',\n", + " 'ADCr11xCdt1xCdc6PE20450',\n", + " 'BoM47Nx01PE20547',\n", + " 'GM18534Bx02PE20372',\n", + " 'HG03683x01PE20434',\n", + " 'GM20847Bx02PE20480',\n", + " 'HG00171Ax02PE20453',\n", + " 'HG02587x02PE20372',\n", + " 'SSC11168x02PE20444',\n", + " 'HG19036Bx02PE20421',\n", + " 'SSC1163101PE20447',\n", + " 'HG19036Bx02PE20389',\n", + " 'BoM662Gx01PE20583',\n", + " 'HG02059102PE20394',\n", + " 'CAST1D1x02PE20473',\n", + " 'HG03371x02PE20590',\n", + " 'ckAML03x01PE20485',\n", + " 'WCHSPCS03x02PE20514',\n", + " 'S149x48hx02PE20325',\n", + " 'HG03371x02PE20560',\n", + " 'BOM65G03x03PE20486',\n", + " 'HG19036Bx02PE20592',\n", + " 'BOMCHIP76Nx02PE20480',\n", + " 'HG002x01PE20322',\n", + " 'HG00096x02PE20314',\n", + " 'HG002x02PE20482',\n", + " 'HG04217x02PE20302',\n", + " 'BOM71Gx03PE20389',\n", + " 'BoM92Nx02PE20473',\n", + " 'HG02818x02PE20383',\n", + " 'HG19036Bx02PE20473',\n", + " 'BOMH12x01PE20419',\n", + " 'BOMCHIP76N02x01PE20337',\n", + " 'ADCr11xGFPxmCherryPE20332',\n", + " 'HG19036Bx02PE20547',\n", + " 'BOMCHIP76Nx02PE20451',\n", + " 'HG19036Bx02PE20580',\n", + " 'GM19650Ax02PE20518',\n", + " 'HG00864x02PE20386',\n", + " 'GM18534Bx02PE20328',\n", + " 'ckAML03x01PE20476',\n", + " 'HG002x02PE20450',\n", + " 'HG04217x02PE20312',\n", + " 'NLEx02PE20352',\n", + " 'BOM65G03x03PE20469',\n", + " 'BoM712Gx01PE20422',\n", + " 'HG02818x02PE20336',\n", + " 'GM18939x02PE20450',\n", + " 'HG03009x02PE20371',\n", + " 'AML1Nx01PE20370',\n", + " 'SSC11168x02PE20420',\n", + " 'LFSL41midx01PE20516',\n", + " 'HG01573x02PE20334',\n", + " 'ckAML01x01PE20391',\n", + " 'S149x48hx02PE20313',\n", + " 'CB0003x01PE20334',\n", + " 'SSC11168x02PE20430',\n", + " 'BoM24HPE20490',\n", + " 'BoM24HPE20493',\n", + " 'LFSL41midx01PE20511',\n", + " 'CAST1D1x02PE20453',\n", + " 'B6CAST1F1x01PE20319',\n", + " 'M56x1x02PE20430',\n", + " 'HG03732x01PE20531',\n", + " 'BOMH12x01PE20443',\n", + " 'SSC11163x02PE20364',\n", + " 'BoM63Gx02PE20509',\n", + " 'HG02587x02PE20352',\n", + " 'HG002x01PE20308',\n", + " 'GM19650Ax02PE20511',\n", + " 'HG01505x01PE20413',\n", + " 'BoM662Gx01PE20567',\n", + " 'HG00171Ax02PE20484',\n", + " 'HG19036Bx02PE20416',\n", + " 'WCxHSPCsPE20359',\n", + " 'BoM712Gx01PE20489',\n", + " 'AML1Nx01PE20345',\n", + " 'HG002x01PE20365',\n", + " 'SSC11168x02PE20469',\n", + " 'AML1Nx01PE20332',\n", + " 'HG03683x01PE20484',\n", + " 'HG01352x02PE20488',\n", + " 'WholeHSPCS03x01PE20575',\n", + " 'HG02492x02PE20435',\n", + " 'ckAML03x01PE20426',\n", + " 'ADCr11xGFPxmCherryPE20310',\n", + " 'AML1Nx01PE20360',\n", + " 'R8x28PE20551',\n", + " 'HG03065x02PE20594',\n", + " 'BoM69Nx01PE20407',\n", + " 'HG01596x02PE20563',\n", + " 'ckAML02x01PE20301',\n", + " 'B6xMEFxP5x02PE20335',\n", + " 'S14960hx04PE20470',\n", + " 'R8x28PE20543',\n", + " 'SSC11168x02PE20473',\n", + " 'WCxHSPCsPE20327',\n", + " 'GM20847Bx02PE20462',\n", + " 'HG02018x01PE20490',\n", + " 'GM18534Bx02PE20335',\n", + " 'HG03732x01PE20586',\n", + " 'HG00268x01PE20461',\n", + " 'GM18939x02PE20436',\n", + " 'BOM65G03x03PE20468',\n", + " 'BoM70Nx01PE20412',\n", + " 'WCHSPCS03x02PE20572',\n", + " 'ckAML01x01PE20331',\n", + " 'BOM65G03x03PE20415',\n", + " 'WCHSPCCB02x01PE20592',\n", + " 'HG00268x01PE20480',\n", + " 'S149x48hx02PE20327',\n", + " 'S149x48hx02PE20366',\n", + " 'BOM71Gx03PE20382',\n", + " 'HG19036Bx02PE20553',\n", + " 'B6xMEFxP5x02PE20384',\n", + " 'BoM24HPE20426',\n", + " 'HG01352x02PE20456',\n", + " 'HG02492x02PE20412',\n", + " 'HG00268x01PE20465',\n", + " 'HG02018x01PE20450',\n", + " 'HG19036Bx02PE20594',\n", + " 'GM19983x02PE20445',\n", + " 'BoM69Nx01PE20484',\n", + " 'SSC1163101PE20485',\n", + " 'SSC11163x02PE20346',\n", + " 'HG02059102PE20343',\n", + " 'HG04217x02PE20395',\n", + " 'GM18534Bx02PE20323',\n", + " 'GM19650Ax02PE20536',\n", + " 'GM18939x02PE20454',\n", + " 'AML1Nx01PE20368',\n", + " 'HG01352x02PE20495',\n", + " 'HG01114x02PE20331',\n", + " 'CB0003x01PE20308',\n", + " 'NLEx02PE20354',\n", + " 'HG00864x02PE20317',\n", + " 'CAST1D1x02PE20442',\n", + " 'ADCr11xGFPxmCherryPE20306',\n", + " 'HG01573x02PE20305',\n", + " 'HG00096x02PE20345',\n", + " 'HG19036Bx02PE20513',\n", + " 'HG002x01PE20391',\n", + " 'HG01114x02PE20314',\n", + " 'SSC11163x02PE20393',\n", + " 'BOM71Gx03PE20339',\n", + " 'BoM70Nx01PE20441',\n", + " 'NLEx02PE20345',\n", + " 'S14960hx04PE20402',\n", + " 'HG02059102PE20377',\n", + " 'HG01352x02PE20417',\n", + " 'HG02106x02PE20335',\n", + " 'BOM71Gx02PE20341',\n", + " 'GM19983x02PE20419',\n", + " 'SSC11163x02PE20308',\n", + " 'BOM71Gx02PE20384',\n", + " 'BoM66Gx01PE20322',\n", + " 'CAST1D1x02PE20435',\n", + " 'HG02106x02PE20387',\n", + " 'HG19036Bx02PE20304',\n", + " 'HG03009x02PE20360',\n", + " 'BOMCHIP76N02x01PE20392',\n", + " 'BOMCHIP76Nx02PE20414',\n", + " 'NLEx02PE20393',\n", + " 'M56x1x02PE20402',\n", + " 'CB0003x01PE20361',\n", + " 'HG03683x01PE20475',\n", + " 'GM18534Bx02PE20383',\n", + " 'NLEx02PE20385',\n", + " 'BoM92Nx02PE20460',\n", + " 'HG02818x02PE20338',\n", + " 'SSC1163101PE20458',\n", + " 'HG03065x02PE20549',\n", + " 'CB0003x01PE20353',\n", + " 'HG01505x01PE20487',\n", + " 'BoM24HPE20484',\n", + " 'ADCr11xCdt1xCdc6PE20444',\n", + " 'CB0003x01PE20378',\n", + " 'HG00096x02PE20350',\n", + " 'ckAML02x01PE20307',\n", + " 'ckAML02x01PE20345',\n", + " 'WCHSPCCB02x01PE20540',\n", + " 'GM20847Bx02PE20442',\n", + " 'HG01596x02PE20548',\n", + " 'BoM662Gx01PE20544',\n", + " 'ckAML03x01PE20413',\n", + " 'HG02106x02PE20370',\n", + " 'ADCr11xCdt1xCdc6PE20417',\n", + " 'LFSL41midx01PE20507',\n", + " 'HG002x01PE20324',\n", + " 'HG03371x02PE20538',\n", + " 'SSC11163x02PE20357',\n", + " 'HG03683x01PE20404',\n", + " 'HG02059102PE20386',\n", + " 'GM20847Bx02PE20491',\n", + " 'NLEx02PE20333',\n", + " 'WCHSPCCB02x01PE20506',\n", + " 'BOM71Gx02PE20307',\n", + " 'HG02106x02PE20322',\n", + " 'BoM63Gx02PE20525',\n", + " 'BoM63Gx02PE20523',\n", + " 'HG04217x02PE20334',\n", + " 'R8x28PE20571',\n", + " 'HG00171Ax02PE20492',\n", + " 'GM18534Bx02PE20358',\n", + " 'SSC11163x02PE20347',\n", + " 'BOM65G03x03PE20446',\n", + " 'WCxHSPCsPE20364',\n", + " 'M56x1x02PE20489',\n", + " 'S149x48hx02PE20351',\n", + " 'WCHSPCCB02x01PE20521',\n", + " 'M56x1x02PE20438',\n", + " 'SSC11168x02PE20433',\n", + " 'BOMCHIP76Nx02PE20433',\n", + " 'BoM92Nx02PE20458',\n", + " 'B6CAST1F1x01PE20352',\n", + " 'ADCr11xCdt1xCdc6PE20454',\n", + " 'CAST1D1x02PE20429',\n", + " 'HG00268x01PE20433',\n", + " 'SSC11168x02PE20463',\n", + " 'S149x48hx02PE20304',\n", + " 'SSC1163101PE20439',\n", + " 'HG00268x01PE20453',\n", + " 'HG02106x02PE20367',\n", + " 'HG02106x02PE20391',\n", + " 'HG03371x02PE20562',\n", + " 'WholeHSPCS03x01PE20513',\n", + " 'HG19036Bx02PE20414',\n", + " 'ckAML02x01PE20338',\n", + " 'BoM47Nx01PE20559',\n", + " 'BOMCHIP76Nx02PE20418',\n", + " 'ADCr11xCdt1xCdc6PE20405',\n", + " 'WCHSPCS03x02PE20586',\n", + " 'HG19036Bx02PE20337',\n", + " 'CB0003x01PE20302',\n", + " 'HG03683x01PE20453',\n", + " 'ckAML01x01PE20303',\n", + " 'BoM70Nx01PE20481',\n", + " 'BoM63Gx02PE20569',\n", + " 'LFSL41midx01PE20563',\n", + " 'HG01114x02PE20332',\n", + " 'HG03371x02PE20531',\n", + " 'ADCr11xCdt1xCdc6PE20477',\n", + " 'ADCr11xGFPxmCherryPE20393',\n", + " 'GM20847Bx02PE20408',\n", + " 'GM20847Bx02PE20457',\n", + " 'ckAML02x01PE20318',\n", + " 'HG19036Bx02PE20475',\n", + " 'HG03371x02PE20518',\n", + " 'HG02818x02PE20384',\n", + " 'SSC1163101PE20462',\n", + " 'BoM92Nx02PE20472',\n", + " 'BoM63Gx02PE20533',\n", + " 'M56x1x02PE20452',\n", + " 'WholeHSPCS03x01PE20573',\n", + " 'NLEx02PE20363',\n", + " 'HG01596x02PE20538',\n", + " 'HG02492x02PE20415',\n", + " 'ckAML03x01PE20417',\n", + " 'HG03009x02PE20369',\n", + " 'B6xMEFxP5x02PE20332',\n", + " 'BOM71Gx03PE20333',\n", + " 'WCxHSPCsPE20324',\n", + " 'HG00096x02PE20349',\n", + " 'SSC11163x02PE20336',\n", + " 'HG00268x01PE20417',\n", + " 'ckAML01x01PE20344',\n", + " 'HG01114x02PE20372',\n", + " 'ADCr11xCdt1xCdc6PE20418',\n", + " 'HG03732x01PE20518',\n", + " 'GM19434x02PE20571',\n", + " 'GM18534Bx02PE20340',\n", + " 'WCxHSPCsPE20365',\n", + " 'HG03371x02PE20546',\n", + " 'GM20847Bx02PE20464',\n", + " 'BoM662Gx01PE20548',\n", + " 'GM19434x02PE20575',\n", + " 'HG00268x01PE20471',\n", + " 'HG02106x02PE20340',\n", + " 'BoM92Nx02PE20483',\n", + " 'BOM71Gx03PE20380',\n", + " 'BoM662Gx01PE20578',\n", + " 'ckAML01x01PE20342',\n", + " 'WCHSPCS03x02PE20545',\n", + " 'HG03371x02PE20502',\n", + " 'WCHSPCCB02x01PE20539',\n", + " 'HG03371x02PE20534',\n", + " 'HG01573x02PE20356',\n", + " 'S14960hx04PE20445',\n", + " 'BoM24HPE20465',\n", + " 'S14960hx04PE20477',\n", + " 'R8x28PE20594',\n", + " 'AML1Nx01PE20325',\n", + " 'SSC11163x02PE20332',\n", + " 'HG01352x02PE20430',\n", + " 'HG03371x02PE20580',\n", + " 'BOMH12x01PE20433',\n", + " 'HG03371x02PE20582',\n", + " 'B6xMEFxP5x02PE20358',\n", + " 'BoM69Nx01PE20445',\n", + " 'HG00864x02PE20355',\n", + " 'HG01573x02PE20309',\n", + " 'HG00268x01PE20421',\n", + " 'B6xMEFxP5x02PE20359',\n", + " 'WholeHSPCS03x01PE20595',\n", + " 'HG00268x01PE20436',\n", + " 'HG03371x02PE20563',\n", + " 'ckAML02x01PE20388',\n", + " 'GM19650Ax02PE20576',\n", + " 'BOM65G03x03PE20430',\n", + " 'WholeHSPCS03x01PE20518',\n", + " 'S149x48hx02PE20303',\n", + " 'LFSL41midx01PE20547',\n", + " 'ADCr11xCdt1xCdc6PE20483',\n", + " 'HG00171Ax02PE20490',\n", + " 'HG03009x02PE20354',\n", + " 'HG01505x01PE20404',\n", + " 'GM19434x02PE20564',\n", + " 'BoM662Gx01PE20573',\n", + " 'HG02587x02PE20359',\n", + " 'BoM47Nx01PE20593',\n", + " 'HG002x02PE20484',\n", + " 'HG01352x02PE20433',\n", + " 'ADCr11xGFPxmCherryPE20304',\n", + " 'B6xMEFxP5x02PE20342',\n", + " 'M56x1x02PE20477',\n", + " 'LFSL41midx01PE20504',\n", + " 'LFSL41midx01PE20517',\n", + " 'HG01505x01PE20421',\n", + " 'HG04217x02PE20390',\n", + " 'SSC11168x02PE20461',\n", + " 'BOM71Gx02PE20324',\n", + " 'CB0003x01PE20338',\n", + " 'ADCr11xCdt1xCdc6PE20420',\n", + " 'BOMCHIP76Nx02PE20476',\n", + " 'HG19036Bx02PE20406',\n", + " 'HG03009x02PE20328',\n", + " 'HG03009x02PE20392',\n", + " 'WCHSPCS03x02PE20535',\n", + " 'HG01352x02PE20491',\n", + " 'BoM63Gx02PE20521',\n", + " 'HG01114x02PE20393',\n", + " 'GM18534Bx02PE20318',\n", + " 'BoM70Nx01PE20418',\n", + " 'BoM24HPE20463',\n", + " 'HG19036Bx02PE20390',\n", + " 'WCxHSPCsPE20385',\n", + " 'BOMH12x01PE20493',\n", + " 'M56x1x02PE20465',\n", + " 'R8x28PE20533',\n", + " 'HG00171Ax02PE20413',\n", + " 'HG01573x02PE20338',\n", + " 'NLEx02PE20327',\n", + " 'HG03683x01PE20408',\n", + " 'SSC1163101PE20488',\n", + " 'CAST1D1x02PE20441',\n", + " 'HG01505x01PE20468',\n", + " 'HG03732x01PE20563',\n", + " 'HG19036Bx02PE20448',\n", + " 'BOM65G03x03PE20462',\n", + " 'BOMH12x01PE20490',\n", + " 'BoM70Nx01PE20417',\n", + " 'B6xMEFxP5x02PE20362',\n", + " 'HG01352x02PE20406',\n", + " 'HG02018x01PE20451',\n", + " 'HG19036Bx02PE20315',\n", + " 'GM19650Ax02PE20569',\n", + " 'HG00864x02PE20324',\n", + " 'HG002x01PE20386',\n", + " 'GM18534Bx02PE20325',\n", + " 'HG00096x02PE20334',\n", + " 'GM19650Ax02PE20572',\n", + " 'HG02818x02PE20335',\n", + " 'HG002x01PE20377',\n", + " 'BoM92Nx02PE20423',\n", + " 'WCxHSPCsPE20308',\n", + " 'HG02059102PE20361',\n", + " 'HG19036Bx02PE20589',\n", + " 'M56x1x02PE20450',\n", + " 'NLEx02PE20390',\n", + " 'ckAML02x01PE20323',\n", + " 'ADCr11xGFPxmCherryPE20392',\n", + " 'BoM92Nx02PE20413',\n", + " 'BOM71Gx03PE20391',\n", + " 'BoM63Gx02PE20590',\n", + " 'HG00096x02PE20361',\n", + " 'BOM65G03x03PE20411',\n", + " 'HG01573x02PE20371',\n", + " 'WCxHSPCsPE20393',\n", + " 'ADCr11xCdt1xCdc6PE20478',\n", + " 'CB0003x01PE20335',\n", + " 'HG03732x01PE20562',\n", + " 'HG03371x02PE20593',\n", + " 'HG01505x01PE20411',\n", + " 'GM18939x02PE20481',\n", + " 'BOM71Gx02PE20331',\n", + " 'BOM71Gx02PE20381',\n", + " 'HG02018x01PE20496',\n", + " 'ckAML02x01PE20370',\n", + " 'HG01352x02PE20455',\n", + " 'ADCr11xGFPxmCherryPE20301',\n", + " 'B6CAST1F1x01PE20328',\n", + " 'HG02818x02PE20386',\n", + " 'WCHSPCS03x02PE20587',\n", + " 'HG02106x02PE20316',\n", + " 'HG03683x01PE20446',\n", + " 'WCHSPCCB02x01PE20556',\n", + " 'GM18534Bx02PE20351',\n", + " 'CB0003x01PE20360',\n", + " 'BOM71Gx03PE20390',\n", + " 'BoM70Nx01PE20462',\n", + " 'HG01352x02PE20469',\n", + " 'HG03371x02PE20573',\n", + " 'HG03065x02PE20509',\n", + " 'HG03065x02PE20531',\n", + " 'BoM70Nx01PE20410',\n", + " 'SSC11168x02PE20428',\n", + " 'HG19036Bx02PE20540',\n", + " 'BoM47Nx01PE20561',\n", + " 'NLEx02PE20310',\n", + " 'GM20847Bx02PE20438',\n", + " 'BoM70Nx01PE20446',\n", + " 'B6CAST1F1x01PE20377',\n", + " 'HG02018x01PE20443',\n", + " 'HG19036Bx02PE20328',\n", + " 'LFSL41midx01PE20567',\n", + " 'HG01114x02PE20320',\n", + " 'BoM70Nx01PE20423',\n", + " 'ADCr11xGFPxmCherryPE20375',\n", + " 'HG02018x01PE20470',\n", + " 'HG02059102PE20302',\n", + " 'GM19434x02PE20506',\n", + " 'BoM24HPE20406',\n", + " 'S14960hx04PE20451',\n", + " 'BoM712Gx01PE20431',\n", + " 'M56x1x02PE20492',\n", + " 'BOM65G03x03PE20464',\n", + " 'S149x48hx02PE20387',\n", + " 'HG02018x01PE20468',\n", + " 'HG00171Ax02PE20464',\n", + " 'WholeHSPCS03x01PE20561',\n", + " 'HG002x01PE20374',\n", + " 'R8x28PE20507',\n", + " 'BoM712Gx01PE20471',\n", + " 'HG02059102PE20389',\n", + " 'BoM70Nx01PE20466',\n", + " 'GM19650Ax02PE20596',\n", + " 'BoM92Nx02PE20463',\n", + " 'ADCr11xCdt1xCdc6PE20422',\n", + " 'BoM66Gx01PE20348',\n", + " 'GM19650Ax02PE20543',\n", + " 'HG01596x02PE20565',\n", + " 'HG01596x02PE20528',\n", + " 'HG01573x02PE20374',\n", + " 'CB0003x01PE20322',\n", + " 'BOM65G03x03PE20401',\n", + " 'BOM71Gx03PE20346',\n", + " 'ADCr11xGFPxmCherryPE20358',\n", + " 'BoM24HPE20424',\n", + " 'GM19983x02PE20429',\n", + " 'HG03009x02PE20334',\n", + " 'BoM662Gx01PE20532',\n", + " 'HG03009x02PE20351',\n", + " 'BoM66Gx01PE20308',\n", + " 'HG01352x02PE20474',\n", + " 'HG04217x02PE20375',\n", + " 'BOMCHIP76N02x01PE20388',\n", + " 'BOM65G03x03PE20467',\n", + " 'HG04217x02PE20322',\n", + " 'SSC11168x02PE20441',\n", + " 'HG19036Bx02PE20387',\n", + " 'BOMCHIP76N02x01PE20348',\n", + " 'HG19036Bx02PE20480',\n", + " 'HG19036Bx02PE20518',\n", + " 'HG00096x02PE20342',\n", + " 'HG002x02PE20453',\n", + " 'S149x48hx02PE20311',\n", + " 'WCHSPCS03x02PE20592',\n", + " 'HG01352x02PE20462',\n", + " 'HG00268x01PE20470',\n", + " 'LFSL41midx01PE20535',\n", + " 'HG01573x02PE20391',\n", + " 'GM19650Ax02PE20560',\n", + " 'HG002x02PE20479',\n", + " 'BOM71Gx02PE20379',\n", + " 'HG03683x01PE20415',\n", + " 'BoM24HPE20421',\n", + " 'SSC11168x02PE20451',\n", + " 'WCxHSPCsPE20352',\n", + " 'R8x28PE20546',\n", + " 'HG02492x02PE20496',\n", + " 'HG02106x02PE20386',\n", + " 'HG01573x02PE20301',\n", + " 'HG00268x01PE20452',\n", + " 'BoM24HPE20496',\n", + " 'R8x28PE20526',\n", + " 'BoM63Gx02PE20580',\n", + " 'HG03065x02PE20550',\n", + " 'HG00096x02PE20341',\n", + " 'BoM47Nx01PE20522',\n", + " 'HG03371x02PE20561',\n", + " 'HG02587x02PE20334',\n", + " 'SSC11163x02PE20389',\n", + " 'BOM71Gx03PE20352',\n", + " 'HG01352x02PE20418',\n", + " 'BoM712Gx01PE20449',\n", + " 'HG00864x02PE20316',\n", + " 'HG01114x02PE20389',\n", + " 'SSC1163101PE20426',\n", + " 'HG02587x02PE20356',\n", + " 'HG02587x02PE20367',\n", + " 'HG02587x02PE20370',\n", + " 'S14960hx04PE20447',\n", + " 'BoM712Gx01PE20456',\n", + " ...]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(df[\"cell\"].unique())" ] }, { diff --git a/dev/memory_profiling/app.py b/dev/memory_profiling/app.py new file mode 100644 index 0000000..1717262 --- /dev/null +++ b/dev/memory_profiling/app.py @@ -0,0 +1,81 @@ +import dash_ag_grid as dag +from dash import Dash, Input, Output, html, no_update, callback +import pandas as pd +import polars as pl +import time + +app = Dash(__name__) + +# df = pd.read_csv( +# "https://raw.githubusercontent.com/plotly/datasets/master/ag-grid/olympic-winners.csv" +# ) +df = pl.read_csv( + "https://raw.githubusercontent.com/plotly/datasets/master/ag-grid/olympic-winners.csv" +) + +columnDefs = [ + # this row shows the row index, doesn't use any data from the row + { + "headerName": "ID", + "maxWidth": 100, + # it is important to have node.id here, so that when the id changes (which happens + # when the row is loaded) then the cell is refreshed. + "valueGetter": {"function": "params.node.id"}, + "cellRenderer": "SpinnerCellRenderer", + }, + {"field": "athlete", "minWidth": 150}, + {"field": "country", "minWidth": 150}, + {"field": "year"}, + {"field": "sport", "minWidth": 150}, + {"field": "total"}, +] + +defaultColDef = { + "flex": 1, + "minWidth": 150, + "sortable": False, + "resizable": True, +} + +app.layout = html.Div( + [ + dag.AgGrid( + id="grid", + columnDefs=columnDefs, + defaultColDef=defaultColDef, + rowModelType="infinite", + dashGridOptions={ + # The number of rows rendered outside the viewable area the grid renders. + "rowBuffer": 0, + # How many blocks to keep in the store. Default is no limit, so every requested block is kept. + "maxBlocksInCache": 2, + "cacheBlockSize": 100, + "cacheOverflowSize": 2, + "maxConcurrentDatasourceRequests": 2, + "infiniteInitialRowCount": 1, + "rowSelection": "multiple", + "pagination": True, + "paginationAutoPageSize": True, + }, + ), + ], +) + + +@callback( + Output("grid", "getRowsResponse"), + Input("grid", "getRowsRequest"), +) +def infinite_scroll(request): + # simulate slow callback + # time.sleep(2) + + if request is None: + return no_update + # partial = df.iloc[request["startRow"]: request["endRow"]] + partial = df[request["startRow"]: request["endRow"]] + return {"rowData": partial.to_dicts(), "rowCount": df.shape[0]} + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/dev/memory_profiling/profile_app.py b/dev/memory_profiling/profile_app.py new file mode 100644 index 0000000..1ce0597 --- /dev/null +++ b/dev/memory_profiling/profile_app.py @@ -0,0 +1,25 @@ +from memory_profiler import profile, memory_usage +import app # Ensure app.py is in the same directory as profile_app.py +import logging +import threading +import time + +logging.basicConfig(level=logging.INFO) + +@profile +def run_app(): + logging.info("Starting Dash app") + app.app.run_server(debug=True) + +def log_memory_usage(): + while True: + mem_usage = memory_usage(-1, interval=1, timeout=1) + logging.info(f"Memory usage: {mem_usage[0]} MB") + time.sleep(5) # Log memory usage every 5 seconds + +if __name__ == "__main__": + # Start a separate thread to log memory usage + threading.Thread(target=log_memory_usage, daemon=True).start() + + # Run the Dash app + run_app()