Add integration tests with selenium (#1528)

.github/workflows/ci-frontend.yml

@@ -30,6 +30,6 @@ jobs:
           pip install pytest
           pip install pytest-random-order
           pip install --no-cache-dir .
-      - name: Test flask app
+      - name: Test flask web app
         run: |
           pytest --random-order asreview/webapp/tests

asreview/search.py

@@ -40,7 +40,7 @@ def _create_inverted_index(match_strings):
     return index
 
 
-def _match_best(keywords, index, match_strings, threshold=0.75):
+def _match_best(keywords, index, match_strings, threshold=0.9):
     n_match = len(match_strings)
     word = re.compile(r"['\w]+")
     key_list = word.findall(keywords.lower())

asreview/webapp/src/Components/ProfilePopper.js

@@ -149,7 +149,7 @@ const ProfilePopper = (props) => {
       <ClickAwayListener onClickAway={handleClickAway}>
         <Box>
           <Tooltip title="Profile">
-            <ButtonBase onClick={handleClick}>
+            <ButtonBase id="profile-popper" onClick={handleClick}>
               <Avatar
                 alt="user"
                 src={ElasAvatar}
@@ -233,7 +233,7 @@ const ProfilePopper = (props) => {
                   </MenuItem>
                 )}
 
-                <MenuItem onClick={handleSignOut}>
+                <MenuItem id="signout" onClick={handleSignOut}>
                   <ListItemIcon>
                     <Logout fontSize="small" />
                   </ListItemIcon>

asreview/webapp/src/Components/SignInForm.js

@@ -61,7 +61,7 @@ const SignInForm = (props) => {
       onError: (data) => {
         console.error("Signin error", data);
       },
-    },
+    }
   );
 
   const handleSubmit = (e) => {
@@ -100,6 +100,7 @@ const SignInForm = (props) => {
     <>
       <Stack spacing={3}>
         <TextField
+          id="email"
           label="Email"
           name="email"
           type="email"
@@ -111,6 +112,7 @@ const SignInForm = (props) => {
         />
         <FormControl>
           <TextField
+            id="password"
             label="Password"
             value={password}
             onChange={handlePasswordChange}
@@ -122,6 +124,7 @@ const SignInForm = (props) => {
           <FormControlLabel
             control={
               <Checkbox
+                id="show-password"
                 checked={showPassword}
                 onChange={toggleShowPassword}
                 value="showPassword"
@@ -135,18 +138,27 @@ const SignInForm = (props) => {
       {isError && <InlineErrorHandler message={error.message} />}
       <Stack className={classes.button} direction="row">
         {allowAccountCreation && (
-          <Button onClick={handleSignUp} sx={{ textTransform: "none" }}>
+          <Button
+            id="create-profile"
+            onClick={handleSignUp}
+            sx={{ textTransform: "none" }}
+          >
             Create profile
           </Button>
         )}
 
         {hasEmailConfig && (
-          <Button onClick={handleForgotPassword} sx={{ textTransform: "none" }}>
+          <Button
+            id="forgot-password"
+            onClick={handleForgotPassword}
+            sx={{ textTransform: "none" }}
+          >
             Forgot password
           </Button>
         )}
 
         <LoadingButton
+          id="sign-in"
           loading={isLoading}
           variant="contained"
           color="primary"

asreview/webapp/src/Components/SignUpForm.js

@@ -257,13 +257,15 @@ const SignUpForm = (props) => {
 
                   <Stack className={classes.button} direction="row">
                     <Button
+                      id="sign-in"
                       onClick={handleSignIn}
                       sx={{ textTransform: "none" }}
                     >
                       Sign In instead
                     </Button>
                     <LoadingButton
                       //loading={isLoading}
+                      id="create-profile"
                       variant="contained"
                       color="primary"
                       onClick={handleSubmit}

asreview/webapp/src/HomeComponents/DashboardComponents/ProjectTable.js

@@ -603,7 +603,12 @@ const ProjectTable = (props) => {
               <Typography sx={{ color: "text.secondary", marginTop: "64px" }}>
                 Your projects will show up here
               </Typography>
-              <Button onClick={props.toggleProjectSetup}>Get Started</Button>
+              <Button
+                id="get-started"
+                onClick={props.toggleProjectSetup}
+              >
+                Get Started
+              </Button>
               <img
                 src={ElasArrowRightAhead}
                 alt="ElasArrowRightAhead"

asreview/webapp/src/HomeComponents/DashboardComponents/ProjectsOverview.js

@@ -45,6 +45,7 @@ const ProjectsOverview = (props) => {
         </Stack>
       </Box>
       <Fab
+        id="create-project"
         className="main-page-fab"
         color="primary"
         onClick={props.toggleProjectSetup}

asreview/webapp/src/ProjectComponents/ReviewComponents/DecisionButton.js

@@ -46,6 +46,7 @@ const DecisionButton = (props) => {
       >
         <Box>
           <Fab
+            id="irrelevant"
             disabled={props.disableButton()}
             onClick={() => props.makeDecision(0)}
             size={props.mobileScreen ? "small" : "large"}
@@ -57,6 +58,7 @@ const DecisionButton = (props) => {
         </Box>
         <Box>
           <Fab
+            id="relevant"
             onClick={() => props.makeDecision(1)}
             color="primary"
             disabled={props.disableButton()}

asreview/webapp/src/ProjectComponents/SetupComponents/DataComponents/DataFormCard.js

@@ -116,6 +116,7 @@ const DataFormCard = (props) => {
           <Stack direction="row" sx={{ alignItems: "center" }}>
             {props.added && <Check color="success" sx={{ mr: 1 }} />}
             <Button
+              id={(props.primaryDefault || "add").toLowerCase().replace(/\s+/g, "-")}
               disabled={props.datasetAdded !== undefined && !props.datasetAdded}
               onClick={props.toggleAddCard}
             >

asreview/webapp/src/ProjectComponents/SetupComponents/DataComponents/PriorSearch.js

@@ -113,14 +113,18 @@ const PriorSearch = (props) => {
               </StyledIconButton>
             </Tooltip>
             <InputBase
+              id="search-input"
               autoFocus
               fullWidth
               onChange={onChangeKeyword}
               onKeyDown={onKeyDown}
               placeholder="Search"
               sx={{ ml: 1 }}
             />
-            <StyledIconButton onClick={onClickSearch}>
+            <StyledIconButton
+              id="search"
+              onClick={onClickSearch}
+            >
               <Search />
             </StyledIconButton>
           </Stack>
@@ -145,7 +149,7 @@ const PriorSearch = (props) => {
               !data?.result.filter((record) => record?.included === -1)
                 .length) && (
               <Box className={classes.empty}>
-                <Typography variant="body2" sx={{ color: "text.secondary" }}>
+                <Typography id="no-search-result" variant="body2" sx={{ color: "text.secondary" }}>
                   Your search results will show up here
                 </Typography>
               </Box>

asreview/webapp/src/ProjectComponents/SetupComponents/DataComponents/PriorUnlabeled.js

@@ -126,7 +126,7 @@ const PriorUnlabeled = (props) => {
   };
 
   return (
-    <Root>
+    <Root className="search-result">
       {isError && (
         <Box sx={{ pt: 8 }}>
           <InlineErrorHandler
@@ -176,6 +176,7 @@ const PriorUnlabeled = (props) => {
             </Typography>
             <Box>
               <Button
+                id="relevant"
                 onClick={() => {
                   mutate({
                     project_id: props.project_id,
@@ -191,6 +192,7 @@ const PriorUnlabeled = (props) => {
                 Yes
               </Button>
               <Button
+                id="irrelevant"
                 onClick={() => {
                   mutate({
                     project_id: props.project_id,

asreview/webapp/src/ProjectComponents/SetupComponents/FinishSetup.js

@@ -47,15 +47,15 @@ const FinishSetup = (props) => {
         props.handleBack();
         queryClient.resetQueries("fetchProjectStatus");
       },
-    },
+    }
   );
 
   const onClickCloseSetup = async () => {
     props.toggleProjectSetup();
     console.log("Opening existing project " + props.project_id);
     await queryClient.prefetchQuery(
       ["fetchInfo", { project_id: props.project_id }],
-      ProjectAPI.fetchInfo,
+      ProjectAPI.fetchInfo
     );
     if (props.mode !== projectModes.SIMULATION) {
       navigate(`/projects/${props.project_id}/review`);
@@ -145,7 +145,9 @@ const FinishSetup = (props) => {
                   <TypographySubtitle1Medium>
                     AI is ready to assist you
                   </TypographySubtitle1Medium>
-                  <Button onClick={onClickCloseSetup}>Start Reviewing</Button>
+                  <Button id="start-reviewing" onClick={onClickCloseSetup}>
+                    Start Reviewing
+                  </Button>
                 </Stack>
               </Fade>
             )}

asreview/webapp/src/ProjectComponents/SetupComponents/SetupDialog.js

@@ -611,6 +611,7 @@ const SetupDialog = (props) => {
               </Button>
             )}
             <Button
+              id="next"
               disabled={disableNextButton()}
               variant="contained"
               onClick={handleNext}

asreview/webapp/src/StyledComponents/StyledAlert.js

@@ -10,7 +10,11 @@ export function ExplorationModeRecordAlert(props) {
     >
       Labeled as{" "}
       {
-        <Box sx={{ textDecoration: "underline" }} display="inline">
+        <Box
+          className="labeled-as"
+          sx={{ textDecoration: "underline" }}
+          display="inline"
+        >
           {props.label}
         </Box>
       }{" "}

asreview/webapp/src/api/ProjectAPI.js

@@ -548,7 +548,6 @@ class ProjectAPI {
     if (variables.is_prior === 1) {
       body.set("is_prior", 1);
     }
-
     const url =
       api_url + `projects/${variables.project_id}/record/${variables.doc_id}`;
     return new Promise((resolve, reject) => {

asreview/webapp/src/index.js

@@ -18,7 +18,7 @@ render(
           </Routes>
         </AuthProvider>
       </BrowserRouter>
-    </Provider>
+    </Provider>,
   </React.StrictMode>,
   document.getElementById("root"),
 );

asreview/webapp/tests/README.md

@@ -6,7 +6,7 @@ This folder contains the test suite of the ASReview app. It is organized in the
 
 - **data**: Contains project data used in tests.
 
-- **integration_tests**: Forthcoming.
+- **integration_tests**: Contains integration tests. See README.md in this folder.
 
 - **test_api**: Contains API related tests. These tests are independent, unit-like tests.
 
@@ -35,13 +35,13 @@ Ideally a test function tests one particular feature and can be executed indepen
 
 ## Running the tests
 
-**Important**: if you run the entire test stuite, please make sure you have compiles the app's assets:
+**Important**: if you run the entire test suite, please make sure you have compiled the app's assets:
 
 ```
 python setup.py compile_assets
 ```
 
-Please run your tests with the `--random-order` option to ensure test independency. With Pytest you can run all tests within a particular module. For example (running from the asreview root-folder):
+Please run your tests __from the root directory__ with the `--random-order` option to ensure test independency. With Pytest you can run all tests within a particular module. For example:
 
 ```
 pytest --random-order -s -v ./asreview/webapp/tests/test_api/test_projects.py

asreview/webapp/tests/integration_tests/README.md

@@ -0,0 +1,45 @@
+# Integration tests (in development)
+
+In a comprehensive testing strategy, both unit tests and integration tests play important roles. Unit tests are valuable for isolating and verifying small units of code, while integration tests help ensure that the ASReview web application functions correctly as a whole, considering the collaboration between different components.
+
+The integration tests interact with the frontend of the application. They test the user interface but also verify the internal state of the application by, for example, checking the file system (not implemented yet) or the state of the database in an authorized version of the app.
+
+Running integration tests comes with challenges because they interact with an asynchronous frontend. The tests are slower than unit tests and are potentially brittle: they break when timeouts are reached. With a robust `utils` module we try to overcome this brittleness.
+
+Despite the slower nature of this type of testing it provides valuable information about the interaction between the application's many components. Note that these tests can also be used to quickly setup a project state and populate the database in development mode by simply running a number of them. 
+
+## Software
+
+For the integration tests [pytest-selenium](https://pytest-selenium.readthedocs.io/) is used.
+
+## Using and running the integration tests
+
+The structure of integration tests differs from unit tests. Instead of a number of short unit tests, every integration test module describes a "user story". A user story contains a number of actions in the user interface and a number of assert statements that test whether the actions are reflected in the application's state.
+
+Before any integration test can be executed, an instance of the ASReview must be running. That might be on Docker, a development instance, even a production instance will work. But note that the tests will have a severe impact on the running application: projects will be created, and in an authorized version user accounts are going to be created as well. __The current modules even clear the database completely before starting__. 
+
+To avoid data loss, it is recommended to run the integration tests against a dedicated testing instance of the ASReview app. A Docker instance may seem a great solution, but has one drawback: it's hard to reach the filesystem from outside the Docker environment and that filesystem will eventually also be part of the integration-test suite. 
+
+To run a module/test with `pytest` one must provide 3 mandatory arguments:
+* --driver: a browser driver
+* --url: a URL where selenium can find the frontend
+* --database-uri: a URI where pytest can find the database when an authorized version of ASReview is tested
+
+There is another optional argument `--reading-time` which expects an integer value N that forces the test to wait N seconds before it labels a record as relevant or irrelevant. It simulates the reading time it takes for an end user to make a decision about the relevance of a record.
+
+Use the following command to execute one or more modules:
+```
+$ pytest -v -s <path to specific module or integration-test folder> \
+    --driver <browser> \
+    --url <URL> \
+    --database-uri <database URI>
+```
+
+Here's a more concrete example:
+```
+$ pytest -v -s asreview/webapp/tests/integration_tests/signup_signin_create_project_test.py \
+    --driver firefox \
+    --url http://localhost:3000 \
+    --database-uri postgresql+psycopg2://username:password@127.0.0.1:5432/asreview
+```
+In this concrete example, the test is executed against the frontend of ASReview that runs on a local server on port 3000, using FireFox as the test browser, and the authorization PostgreSQL database can also be found under local host, port 5432.