Merge pull request atlassian#101 from atlassian/release-1.2.1-beta

Release 1.2.1 beta

Author: ometelytsia

README.md

@@ -90,9 +90,12 @@ venv\Scripts\activate
 pip install -r requirements.txt
 ```
 
-## Upgrading
-If you see "There is newer version of Taurus available, consider upgrading." warning message during your 
-test execution, please use the next command to apply the upgrade in your virtual environment.
+## Upgrading the toolkit
+Get latest codebase from master branch:
+```
+git pull
+```
+Activate virtual env for the toolkit and install latest versions of libraries:
 ```
 pip install -r requirements.txt
 ```

app/confluence.yml

@@ -89,4 +89,4 @@ modules:
       version: "79.0.3945.36" # Supports Chrome version 79. You can refer to http://chromedriver.chromium.org/downloads
 reporting:
 - data-source: sample-labels
-  module: junit-xml
+  module: junit-xml

app/jira.yml

@@ -91,4 +91,4 @@ modules:
       version: "79.0.3945.36" # Supports Chrome version 79. You can refer to http://chromedriver.chromium.org/downloads
 reporting:
 - data-source: sample-labels
-  module: junit-xml
+  module: junit-xml

app/selenium_ui/confluence/modules.py

@@ -112,7 +112,7 @@ def measure(webdriver, interaction):
 def create_page(webdriver, datasets):
     @print_timing
     def measure(webdriver, interaction):
-        webdriver.find_element(By.ID, "quick-create-page-button").click()
+        _wait_until(webdriver, ec.element_to_be_clickable((By.ID, 'quick-create-page-button')), interaction).click()
         _dismiss_popup(webdriver, "#closeDisDialog")
         _wait_until(webdriver, ec.element_to_be_clickable((By.ID, 'rte-button-publish')), interaction)
 

app/selenium_ui/jira/modules.py

@@ -240,9 +240,11 @@ def browse_project(webdriver, datasets):
     @print_timing
     def measure(webdriver, interaction):
         page_size = 25
-        pages = len(datasets['project_keys']) // page_size
-        webdriver.get(APPLICATION_URL +
-                      f'/secure/BrowseProjects.jspa?selectedCategory=all&selectedProjectType=all&page={random.randint(1, pages)}')
+        projects_count = len(datasets['project_keys'])
+        pages = projects_count // page_size if projects_count % page_size == 0 else projects_count // page_size + 1
+        webdriver.get(
+            APPLICATION_URL +
+            f'/secure/BrowseProjects.jspa?selectedCategory=all&selectedProjectType=all&page={random.randint(1, pages)}')
         _wait_until(webdriver, AnyEc(ec.presence_of_element_located((By.CSS_SELECTOR, "tbody.projects-list")),
                                      ec.presence_of_element_located((By.CLASS_NAME, "none-panel"))
                                      ), interaction)

app/util/conf.py

@@ -2,7 +2,7 @@
 
 from util.project_paths import JIRA_YML, CONFLUENCE_YML
 
-TOOLKIT_VERSION = '1.2.0'
+TOOLKIT_VERSION = '1.2.1'
 
 
 def read_yml_file(file):

app/util/confluence/populate_db.sh

@@ -109,7 +109,8 @@ if [[ ${FREE_SPACE_GB} -lt ${REQUIRED_SPACE_GB} ]]; then
    echo "Required space: ${REQUIRED_SPACE_GB} GB"
    exit 1
 fi;
-time wget ${DB_DUMP_URL}
+# use computer style progress bar
+time wget --progress=dot:giga ${DB_DUMP_URL}
 if [[ $? -ne 0 ]]; then
   echo "DB dump download failed! Pls check available disk space."
   exit 1
@@ -159,5 +160,8 @@ echo "Step6: Start Confluence"
 sudo systemctl start confluence
 rm -rf ${DB_DUMP_NAME}
 
+echo "Finished"
+echo  # move to a new line
+
 echo "Important: new admin user credentials are admin/admin"
 echo "Wait a couple of minutes until Confluence is started."

app/util/confluence/upload_attachments.sh

@@ -81,11 +81,11 @@ if [[ ${FREE_SPACE_GB} -lt ${REQUIRED_SPACE_GB} ]]; then
    echo "Required space: ${REQUIRED_SPACE_GB} GB"
    exit 1
 fi;
-sudo su confluence -c "time wget ${ATTACHMENTS_TAR_URL}"
+sudo su confluence -c "time wget --progress=dot:giga ${ATTACHMENTS_TAR_URL}"
 
 echo "Step3: Untar attachments to tmp folder"
 sudo su -c "rm -rf ${ATTACHMENTS_DIR}"
-sudo su confluence -c "time tar -xzvf ${ATTACHMENTS_TAR}"
+sudo su confluence -c "time tar -xzf ${ATTACHMENTS_TAR} --checkpoint=.10000"
 if [[ $? -ne 0 ]]; then
   echo "Untar failed!"
   exit 1
@@ -97,4 +97,7 @@ sudo su -c "rm -rf ${ATTACHMENTS_TAR}"
 
 echo "Step4: Copy attachments to EFS"
 sudo su confluence -c "time ./msrsync -P -p 100 -f 3000 ${ATTACHMENTS_DIR} ${EFS_DIR}"
-sudo su -c "rm -rf ${ATTACHMENTS_DIR}"
+sudo su -c "rm -rf ${ATTACHMENTS_DIR}"
+
+echo "Finished"
+echo  # move to a new line

app/util/data_preparation/jira/prepare-data.py

@@ -12,6 +12,7 @@
 ISSUES = "issues"
 JQLS = "jqls"
 PROJECT_KEYS = "project_keys"
+PROJECTS_COUNT_LIMIT = 1000
 
 DEFAULT_USER_PASSWORD = 'password'
 DEFAULT_USER_PREFIX = 'performance_'
@@ -78,7 +79,7 @@ def __write_to_file(file_path, items):
 def __create_data_set(jira_api):
     dataset = dict()
     dataset[USERS] = __get_users(jira_api)
-    software_project_keys = __get_software_project_keys(jira_api)
+    software_project_keys = __get_software_project_keys(jira_api, PROJECTS_COUNT_LIMIT)
     dataset[PROJECT_KEYS] = software_project_keys
     dataset[ISSUES] = __get_issues(jira_api, software_project_keys)
     dataset[SCRUM_BOARDS] = __get_boards(jira_api, 'scrum')
@@ -115,13 +116,13 @@ def __get_users(jira_api):
     return users
 
 
-def __get_software_project_keys(jira_api):
+def __get_software_project_keys(jira_api, max_projects_count):
     all_projects = jira_api.get_all_projects()
     software_project_keys = [project['key'] for project in all_projects if 'software' == project.get('projectTypeKey')]
     if not software_project_keys:
         raise SystemExit("There is no software project in Jira")
-
-    return software_project_keys
+    # Limit number of projects to avoid "Request header is too large" for further requests.
+    return software_project_keys[:max_projects_count]
 
 
 def main():

app/util/environment_checker.py

@@ -1,9 +1,15 @@
 from sys import version_info
 
-SUPPORTED_PYTHON_VERSION = (3, 6)
+from util.conf import TOOLKIT_VERSION
 
-python_version = version_info[0:2]
-if python_version < SUPPORTED_PYTHON_VERSION:
+MIN_SUPPORTED_PYTHON_VERSION = (3, 6, 0)
+
+
+print("Data Center App Performance Toolkit version: {}".format(TOOLKIT_VERSION))
+
+python_version = version_info[0:3]
+print("Python version: {}".format(python_version))
+if python_version < MIN_SUPPORTED_PYTHON_VERSION:
     raise Exception(
         "Python version {} is not supported. "
-        "Please use Python version {} or higher.".format(python_version, SUPPORTED_PYTHON_VERSION))
+        "Please use Python version {} or higher.".format(python_version, MIN_SUPPORTED_PYTHON_VERSION))

app/util/jira/populate_db.sh

@@ -145,7 +145,8 @@ if [[ ${FREE_SPACE_GB} -lt ${REQUIRED_SPACE_GB} ]]; then
    echo "Required space: ${REQUIRED_SPACE_GB} GB"
    exit 1
 fi;
-time wget ${DB_DUMP_URL}
+# use computer style progress bar
+time wget --progress=dot:giga ${DB_DUMP_URL}
 if [[ $? -ne 0 ]]; then
   echo "DB dump download failed! Pls check available disk space."
   exit 1
@@ -195,5 +196,8 @@ echo "Step6: Start Jira"
 sudo su jira -c "${START_JIRA}"
 rm -rf ${DB_DUMP_NAME}
 
+echo "Finished"
+echo  # move to a new line
+
 echo "Important: new admin user credentials are admin/admin"
 echo "Wait a couple of minutes until Jira is started."

app/util/jira/upload_attachments.sh

@@ -81,11 +81,11 @@ if [[ ${FREE_SPACE_GB} -lt ${REQUIRED_SPACE_GB} ]]; then
    echo "Required space: ${REQUIRED_SPACE_GB} GB"
    exit 1
 fi;
-sudo su jira -c "time wget ${ATTACHMENTS_TAR_URL}"
+sudo su jira -c "time wget --progress=dot:giga ${ATTACHMENTS_TAR_URL}"
 
 echo "Step3: Untar attachments to tmp folder"
 sudo su -c "rm -rf ${ATTACHMENTS_DIR}"
-sudo su jira -c "tar -xzvf ${ATTACHMENTS_TAR}"
+sudo su jira -c "tar -xzf ${ATTACHMENTS_TAR} --checkpoint=.10000"
 if [[ $? -ne 0 ]]; then
   echo "Untar failed!"
   exit 1
@@ -97,4 +97,7 @@ sudo su -c "rm -rf ${ATTACHMENTS_TAR}"
 
 echo "Step4: Copy attachments to EFS"
 sudo su jira -c "time ./msrsync -P -p 100 -f 3000 ${ATTACHMENTS_DIR} ${EFS_DIR}"
-sudo su -c "rm -rf ${ATTACHMENTS_DIR}"
+sudo su -c "rm -rf ${ATTACHMENTS_DIR}"
+
+echo "Finished"
+echo  # move to a new line

requirements.txt

@@ -1,4 +1,4 @@
-matplotlib==3.1.1
-pandas==0.25.0
-importlib-metadata==0.20
-bzt==1.13.9
+matplotlib==3.1.2
+pandas==0.25.3
+importlib-metadata==1.4.0
+bzt==1.13.9