Skip to content

Commit

Permalink
Add support for Modin and Snowpark Pandas (#8506)
Browse files Browse the repository at this point in the history
## Describe your changes

Adds support for Modin Dataframe & Series and Snowpark Pandas Dataframe
& Series by only showing the first 10k rows instead of loading all the
data into memory.

This also does a big refactoring of the testing logic for unevaluated
data objects. Instead of testing uneveluated objects for every
individual command, we focus on testing it on the bases of
`convert_anything_to_df` and make sure that all the relevant command use
this method. This also removes all of the e2e tests related to
unevaluated dataframes since there isn't really any value to test those
mock implementation visually.

---

**Contribution License Agreement**

By submitting this pull request you agree that all contributions to this
project are made under the Apache 2.0 license.

---------

Co-authored-by: Jonathan Shi <149419494+sfc-gh-joshi@users.noreply.github.com>
  • Loading branch information
LukasMasuch and sfc-gh-joshi committed Apr 26, 2024
1 parent 7b9f046 commit 43e7a7c
Show file tree
Hide file tree
Showing 58 changed files with 989 additions and 1,014 deletions.
28 changes: 0 additions & 28 deletions e2e/scripts/st_arrow_unevaluated_pyspark_dataframe.py

This file was deleted.

21 changes: 0 additions & 21 deletions e2e/scripts/st_map.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,6 @@
import pandas as pd

import streamlit as st
from tests.streamlit import pyspark_mocks
from tests.streamlit.snowpark_mocks import DataFrame as MockedSnowparkDataFrame
from tests.streamlit.snowpark_mocks import Table as MockedSnowparkTable

"""
### Empty map
Expand All @@ -29,24 +26,6 @@
st.map()


"""
### st.map with pyspark.sql.DataFrame
"""

st.map(pyspark_mocks.DataFrame(is_map=True))

"""
### st.map with unevaluated Snowpark Table
"""

st.map(MockedSnowparkTable(is_map=True, num_of_rows=50000))

"""
### st.map with unevaluated Snowpark DataFrame
"""

st.map(MockedSnowparkDataFrame(is_map=True, num_of_rows=50000))

"""
### Simple map
"""
Expand Down
78 changes: 0 additions & 78 deletions e2e/specs/st_arrow_unevaluated_pyspark_dataframe.spec.js

This file was deleted.

78 changes: 0 additions & 78 deletions e2e/specs/st_arrow_unevaluated_snowpark_dataframe.spec.js

This file was deleted.

61 changes: 31 additions & 30 deletions e2e/specs/st_map.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,43 +19,44 @@ describe("st.map", () => {
cy.loadApp("http://localhost:3000/");

// Wait for all the maps to be loaded.
cy.get(".element-container .stDeckGlJsonChart").should("have.length", 7);
cy.get(".element-container .stDeckGlJsonChart").should("have.length", 4);
});

it("displays 7 maps", () => {
cy.get(".element-container .stDeckGlJsonChart").should("have.length", 7)
it("displays 4 maps", () => {
cy.get(".element-container .stDeckGlJsonChart").should("have.length", 4);
});

it("displays 7 zoom buttons", () => {
cy.get(".element-container .zoomButton").should("have.length", 7)
})

it("displays warning about data being capped", () => {
cy.get("div [data-testid='stCaptionContainer']")
.should("have.length", 2)

// Check warning body.
cy.getIndexed("div [data-testid='stCaptionContainer']", 0)
.should("contain", "⚠️ Showing only 10k rows. Call collect() on the dataframe to show more.")
cy.getIndexed("div [data-testid='stCaptionContainer']", 1)
.should("contain", "⚠️ Showing only 10k rows. Call collect() on the dataframe to show more.")
})
it("displays 4 zoom buttons", () => {
cy.get(".element-container .zoomButton").should("have.length", 4);
});

it("displays basic and complex maps correctly", () => {
cy.get(".mapboxgl-canvas")
cy.get(".mapboxgl-canvas");
// Adding a sufficient wait to ensure the map fully loads before taking the snapshot
cy.wait(10000)
cy.get(".element-container:has(.stDeckGlJsonChart)", { waitForAnimations: true })
.eq(5).matchImageSnapshot("stDeckGlJsonChart")
cy.get(".element-container:has(.stDeckGlJsonChart)", { waitForAnimations: true })
.eq(6).matchImageSnapshot("stDeckGlJsonChart-complexMap")
cy.wait(10000);
cy.get(".element-container:has(.stDeckGlJsonChart)", {
waitForAnimations: true,
})
.eq(2)
.matchImageSnapshot("stDeckGlJsonChart");
cy.get(".element-container:has(.stDeckGlJsonChart)", {
waitForAnimations: true,
})
.eq(3)
.matchImageSnapshot("stDeckGlJsonChart-complexMap");

// Need to manually change theme vs. matchThemedSnapshot to be able to add wait in right sequence
cy.changeTheme("Dark")
cy.wait(10000)
cy.get(".element-container:has(.stDeckGlJsonChart)", { waitForAnimations: true })
.eq(5).matchImageSnapshot("stDeckGlJsonChart-dark")
cy.get(".element-container:has(.stDeckGlJsonChart)", { waitForAnimations: true })
.eq(6).matchImageSnapshot("stDeckGlJsonChart-complexMap-dark")
})
cy.changeTheme("Dark");
cy.wait(10000);
cy.get(".element-container:has(.stDeckGlJsonChart)", {
waitForAnimations: true,
})
.eq(2)
.matchImageSnapshot("stDeckGlJsonChart-dark");
cy.get(".element-container:has(.stDeckGlJsonChart)", {
waitForAnimations: true,
})
.eq(3)
.matchImageSnapshot("stDeckGlJsonChart-complexMap-dark");
});
});
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import streamlit as st
from tests.streamlit.snowpark_mocks import DataFrame

snowpark_dataframe = DataFrame(num_of_rows=50000, num_of_cols=4)

st.dataframe(snowpark_dataframe)

st.line_chart(snowpark_dataframe)

st.bar_chart(snowpark_dataframe)

st.area_chart(snowpark_dataframe)

st.table(snowpark_dataframe)
Loading

0 comments on commit 43e7a7c

Please sign in to comment.