Fix quotas test issue in upgrade pipeline

1. Change way for quotas verification in upgrade pipeline, prepare specific size of image, then it's an known value for verifcation;
2. Add notary key rotate test;
3. For issue brought by docker 20, clean containerd cache is the only effective way, so both dockerd and containerd should be cache cleard and restarted;
4. Upgrade E2E Dockerfile for importing readable file size package, and other issues;
5. Uncomment project level robot account test in nightly.
6. Get DNS from local setting, and set it into docker deamon config file;

Signed-off-by: danfengliu <danfengl@vmware.com>

tests/apitests/python/library/base.py

@@ -92,6 +92,32 @@ def _get_string_from_unicode(udata):
         result = result + tmp.strip('\n\r\t')
     return result
 
+def restart_process(process):
+    if process == "dockerd":
+        full_process_name = process
+    elif process == "containerd":
+        full_process_name = "/usr/local/bin/containerd"
+    else:
+        raise Exception("Please input dockerd or containerd for process retarting.")
+    run_command_with_popen("ps aux |grep " + full_process_name)
+    for i in range(10):
+        pid = run_command_with_popen(["pidof " + full_process_name])
+        if pid in [None, ""]:
+            break
+        run_command_with_popen(["kill " + str(pid)])
+        time.sleep(3)
+
+    run_command_with_popen("ps aux |grep " + full_process_name)
+    run_command_with_popen("rm -rf /var/lib/" + process + "/*")
+    run_command_with_popen(full_process_name + " > ./daemon-local.log 2>&1 &")
+    time.sleep(3)
+    pid = run_command_with_popen(["pidof " + full_process_name])
+    if pid in [None, ""]:
+        raise Exception("Failed to start process {}.".format(full_process_name))
+    run_command_with_popen("ps aux |grep " + full_process_name)
+
+
+
 def run_command_with_popen(command):
     print("Command: ", subprocess.list2cmdline(command))
 
@@ -100,11 +126,14 @@ def run_command_with_popen(command):
                             stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
         output, errors = proc.communicate()
     except Exception as e:
-        print("Error:", e)
+        print("Run command caught exception:", e)
+        output = None
     else:
         print(proc.returncode, errors, output)
     finally:
         proc.stdout.close()
+        print("output: ", output)
+        return output
 
 def run_command(command, expected_error_message = None):
     print("Command: ", subprocess.list2cmdline(command))

tests/apitests/python/test_project_level_policy_content_trust.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 
 import unittest
+import time
 
 from testutils import ADMIN_CLIENT, suppress_urllib3_warning
 from testutils import harbor_server
@@ -12,6 +13,8 @@
 from library.repository import push_self_build_image_to_project
 from library.repository import pull_harbor_image
 from library.docker_api import docker_image_clean_all
+from library.base import  restart_process
+
 class TestProjects(unittest.TestCase):
     @suppress_urllib3_warning
     def setUp(self):
@@ -78,6 +81,9 @@ def testProjectLevelPolicyContentTrust(self):
 
         #7. Pull image(IA) failed and the reason is "The image is not signed in Notary".
         docker_image_clean_all()
+        restart_process("containerd")
+        restart_process("dockerd")
+        time.sleep(30)
         pull_harbor_image(harbor_server, ADMIN_CLIENT["username"], ADMIN_CLIENT["password"], TestProjects.repo_name, tag, expected_error_message = "The image is not signed in Notary")
 
 if __name__ == '__main__':

tests/apitests/python/test_retention.py

@@ -44,12 +44,12 @@ def setUp(self):
 
     @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
     def tearDown(self):
-        # TODO delete_repository will fail when no tags left anymore
-        #     resp=self.repo.list_repositories(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT)
-        #     for repo in resp:
-        #         self.repo.delete_repository(repo.name, **TestProjects.USER_RA_CLIENT)
-        #     self.project.delete_project(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT)
-        #     self.user.delete_user(TestProjects.user_ra_id, **ADMIN_CLIENT)
+        #TODO delete_repository will fail when no tags left anymore
+        resp=self.repo.list_repositories(TestProjects.project_src_repo_name, **TestProjects.USER_RA_CLIENT)
+        for repo in resp:
+            self.repo.delete_repository(TestProjects.project_src_repo_name, repo.name.split('/')[1], **TestProjects.USER_RA_CLIENT)
+        self.project.delete_project(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT)
+        self.user.delete_user(TestProjects.user_ra_id, **ADMIN_CLIENT)
         print("Case completed")
 
     def testTagRetention(self):

tests/ci/api_common_install.sh

@@ -18,6 +18,33 @@ cat /proc/version
 sudo -H pip install --ignore-installed urllib3 chardet requests --upgrade
 python --version
 
+#---------------Set DNS for docker v20-------------------#
+# In docker v20, it fixed an issue named  "Wrong resolv.conf
+# used on Ubuntu 19", this fix caused DNS solve problem 
+# in container. So the current work round is read DNS server
+# from system and set the value in /etc/docker/daemon.json.
+
+ip addr
+dns_ip=$(netplan ip leases eth0 | grep -i dns | awk -F = '{print $2}')
+dns_ip_list=$(echo $dns_ip | tr " " "\n")
+dns_cfg=""
+for ip in $dns_ip_list
+do
+    dns_cfg="$dns_cfg,\"$ip\""
+done
+
+cat /etc/docker/daemon.json
+
+if [ $(cat /etc/docker/daemon.json |grep \"dns\" |wc -l) -eq 0 ];then
+    sudo sed "s/}/,\n   \"dns\": [${dns_cfg:1}]\n}/" -i /etc/docker/daemon.json
+fi
+
+cat /etc/docker/daemon.json
+sudo systemctl daemon-reload
+sudo systemctl restart docker
+sudo systemctl status docker
+#--------------------------------------------------------#
+
 sudo ./tests/hostcfg.sh
 
 if [ "$2" = 'LDAP' ]; then

tests/ci/api_run.sh

@@ -18,7 +18,7 @@ harbor_logs_bucket="harbor-ci-logs"
 #echo "content_language = en" >> $botofile
 #echo "default_project_id = $GS_PROJECT_ID" >> $botofile
 DIR="$(cd "$(dirname "$0")" && pwd)"
-E2E_IMAGE="goharbor/harbor-e2e-engine:2.6.2"
+E2E_IMAGE="goharbor/harbor-e2e-engine:2.6.3"
 
 # GS util
 function uploader {

tests/e2e-image/Dockerfile

@@ -1,11 +1,14 @@
 FROM ubuntu:18.04
 ENV LANG C.UTF-8
 # V 2.0
-# V 2.0.1: upgrade docker to version 19.03.12.
+# V 2.0.1: Upgrade docker to version 19.03.12.
 # V 2.5 Add support for e2e py-test (especially containerd).
-# V 2.6 docker 19.03.12.
-# V 2.6.1 upgrade containerd(ctr) to v1.4.3, docker 20.10.3.
-# V 2.6.2 package busybox into E2E image.
+# V 2.6 Upgrade docker 19.03.12.
+# V 2.6.1 Upgrade containerd(ctr) to v1.4.3, docker 20.10.3.
+# V 2.6.2 Package busybox into E2E image.
+# V 2.6.3 a. Swith python version from 3.7 to 3.6;
+#         b. Upgrade and fix cnab-to-oci build issue;
+#         c. Install hurry.filesize tool in python.
 
 RUN apt-get update && apt-get install -y --no-install-recommends wget curl gnupg2
 RUN apt-get install libseccomp2
@@ -66,8 +69,8 @@ RUN apt-get update && apt-get install -y software-properties-common && \
 RUN apt-get update && \
     apt-get install -y  golang-go
 
-RUN apt-get update -y ; apt-get install -y zbar-tools libzbar-dev python-zbar python3.7
-RUN rm /usr/bin/python ; ln -s /usr/bin/python3.7 /usr/bin/python ; apt-get install -y python3-pip
+RUN apt-get update -y ; apt-get install -y zbar-tools libzbar-dev python-zbar python3.6
+RUN rm /usr/bin/python ; ln -s /usr/bin/python3.6 /usr/bin/python ; apt-get install -y python3-pip
 RUN python -m pip install --upgrade pip
 
 RUN wget -N http://chromedriver.storage.googleapis.com/2.40/chromedriver_linux64.zip && \
@@ -82,7 +85,7 @@ RUN apt-get update && apt install libnss3-tools && \
     echo Harbor12345 > password.ca && \
     certutil -d sql:$HOME/.pki/nssdb -N -f password.ca
 
-RUN pip3 install pyasn1 google-apitools==0.5.31 gsutil robotframework==3.2.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot robotframework-JSONLibrary --upgrade
+RUN pip3 install pyasn1 google-apitools==0.5.31 gsutil robotframework==3.2.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot robotframework-JSONLibrary hurry.filesize --upgrade
 
 ENV CONTAINERD_VERSION 1.4.3
 RUN wget https://github.com/containerd/containerd/releases/download/v1.4.3/containerd-$CONTAINERD_VERSION-linux-amd64.tar.gz && \
@@ -147,8 +150,8 @@ RUN wget http://nchc.dl.sourceforge.net/sourceforge/tcl/tcl8.4.11-src.tar.gz &&
     cd /ixdba.net/bin && \
     mv  expect /usr/local/bin/expect
 
-RUN CNAB_PATH=$(go env GOPATH)/src/github.com/docker && mkdir -p $CNAB_PATH && cd $CNAB_PATH && git clone https://github.com/cnabio/cnab-to-oci.git && \
-    cd cnab-to-oci && git checkout v0.3.0-beta4 && \
+RUN CNAB_PATH=$(go env GOPATH)/src/github.com/cnabio && mkdir -p $CNAB_PATH && cd $CNAB_PATH && git clone https://github.com/cnabio/cnab-to-oci.git && \
+    cd cnab-to-oci && git checkout v0.3.1-beta1 && \
     go list && \
     make build && \
     mv bin/cnab-to-oci /usr/local/bin

tests/resources/Docker-Util.robot

@@ -27,7 +27,7 @@ Pull image
     Log To Console  \nRunning docker pull ${image}...
     ${image_with_tag}=  Set Variable If  '${tag}'=='${null}'  ${image}  ${image}:${tag}
     Run Keyword If  ${is_robot}==${false}  Wait Unitl Command Success  docker login -u ${user} -p ${pwd} ${ip}
-    ...  ELSE  Wait Unitl Command Success  docker login -u robot\\\$${user} -p ${pwd} ${ip}
+    ...  ELSE  Wait Unitl Command Success  docker login -u robot\\\$${project}+${user} -p ${pwd} ${ip}
     ${output}=  Docker Pull  ${ip}/${project}/${image_with_tag}
     Log  ${output}
     Log To Console  ${output}
@@ -48,7 +48,7 @@ Push image
     ${image_in_use}=   Set Variable If  ${need_pull_first}==${true}  ${image_in_use}  ${image_with_or_without_tag}
     Run Keyword If  ${need_pull_first}==${true}   Docker Pull  ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image_in_use}
     Run Keyword If  ${is_robot}==${false}  Wait Unitl Command Success  docker login -u ${user} -p ${pwd} ${ip}
-    ...  ELSE  Wait Unitl Command Success  docker login -u robot\\\$${user} -p ${pwd} ${ip}
+    ...  ELSE  Wait Unitl Command Success  docker login -u robot\\\$${project}+${user} -p ${pwd} ${ip}
     Run Keyword If  ${need_pull_first}==${true}  Wait Unitl Command Success  docker tag ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image_in_use} ${ip}/${project}/${image_in_use_with_tag}
     ...  ELSE  Wait Unitl Command Success  docker tag ${image_in_use} ${ip}/${project}/${image_in_use_with_tag}
     Wait Unitl Command Success  docker push ${ip}/${project}/${image_in_use_with_tag}
@@ -70,33 +70,31 @@ Push Image With Tag
     Clean All Local Images
 
 Clean All Local Images
-    Wait Unitl Command Success  docker rmi -f $(docker images -a -q)
-    Wait Unitl Command Success  docker system prune -a -f
+    ${rc}  ${out}=  Run Keyword And Ignore Error  Run  docker rmi -f $(docker images -a -q)
+    Log All  ${out}
+    ${rc}  ${out}=  Run Keyword And Ignore Error  Run  docker system prune -a -f
+    Log All  ${out}
 
 Cannot Docker Login Harbor
     [Arguments]  ${ip}  ${user}  ${pwd}
     Command Should be Failed  docker login -u ${user} -p ${pwd} ${ip}
 
 Cannot Pull Image
     [Arguments]  ${ip}  ${user}  ${pwd}  ${project}  ${image}  ${tag}=${null}  ${err_msg}=${null}
+    Restart Process Locally  containerd
+    Restart Process Locally  dockerd
     ${image_with_tag}=  Set Variable If  '${tag}'=='${null}'  ${image}  ${image}:${tag}
     Wait Unitl Command Success  docker login -u ${user} -p ${pwd} ${ip}
     FOR  ${idx}  IN RANGE  0  30
         ${out}  Run Keyword And Ignore Error  Command Should be Failed  docker pull ${ip}/${project}/${image_with_tag}
         Exit For Loop If  '${out[0]}'=='PASS'
         Sleep  3
     END
+    Clean All Local Images
     Log To Console  Cannot Pull Image - Pull Log: ${out[1]}
     Should Be Equal As Strings  '${out[0]}'  'PASS'
     Run Keyword If  '${err_msg}' != '${null}'  Should Contain  ${out[1]}  ${err_msg}
 
-Cannot Pull Unsigned Image
-    [Arguments]  ${ip}  ${user}  ${pass}  ${proj}  ${imagewithtag}
-    Wait Unitl Command Success  docker login -u ${user} -p ${pass} ${ip}
-    ${output}=  Command Should be Failed  docker pull ${ip}/${proj}/${imagewithtag}
-    Log To Console  ${output}
-    Should Contain  ${output}  The image is not signed in Notary
-
 Cannot Push image
     [Arguments]  ${ip}  ${user}  ${pwd}  ${project}  ${image}  ${err_msg}=${null}  ${err_msg_2}=${null}
     Log To Console  \nRunning docker push ${image}...
@@ -108,6 +106,7 @@ Cannot Push image
     Run Keyword If  '${err_msg}' != '${null}'  Should Contain  ${output}  ${err_msg}
     Run Keyword If  '${err_msg_2}' != '${null}'  Should Contain  ${output}  ${err_msg_2}
     Wait Unitl Command Success  docker logout ${ip}
+    Clean All Local Images
 
 Wait Until Container Stops
     [Arguments]  ${container}
@@ -149,37 +148,50 @@ Start Docker Daemon Locally
     [Return]  ${handle}
 
 Start Containerd Daemon Locally
-    ${handle}=  Start Process  containerd > ./daemon-local.log 2>&1 &  shell=True
+    ${handle}=  Start Process  /usr/local/bin/containerd > ./daemon-local.log 2>&1 &  shell=True
     FOR  ${IDX}  IN RANGE  5
-        ${pid}=  Run  pidof containerd
+        ${pid}=  Run  pidof /usr/local/bin/containerd
         Log To Console  pid: ${pid}
         Exit For Loop If  '${pid}' != '${EMPTY}'
         Sleep  2s
     END
     Sleep  2s
     [Return]  ${handle}
 
-Restart Docker Daemon Locally
+Restart Process Locally
+    [Arguments]  ${process}
+    ${full_process}=  Set Variable If
+    ...  '${process}'=='containerd'  /usr/local/bin/containerd  dockerd
+    ${start_process_cmd}=  Set Variable If
+    ...  '${process}'=='dockerd'  /usr/local/bin/dockerd-entrypoint.sh dockerd>./daemon-local.log 2>&1
+    ...  '${process}'=='containerd'  ${full_process} > ./daemon-local.log 2>&1 &
+    Should Be True  '${start_process_cmd}' != '${EMPTY}'
+    Run Keyword If  '${process}'=='dockerd'  OperatingSystem.File Should Exist  /usr/local/bin/dockerd-entrypoint.sh
+
     FOR  ${IDX}  IN RANGE  5
-        ${pid}=  Run  pidof dockerd
+        ${pid}=  Run  pidof ${full_process}
         Exit For Loop If  '${pid}' == '${EMPTY}'
-        ${result}=  Run Process  kill ${pid}  shell=True
+        ${result}=  Run  kill ${pid}
         Log To Console  Kill docker process: ${result}
         Sleep  2s
     END
-    ${pid}=  Run  pidof dockerd
+    ${pid}=  Run  pidof ${full_process}
     Should Be Equal As Strings  '${pid}'  '${EMPTY}'
-    OperatingSystem.File Should Exist  /usr/local/bin/dockerd-entrypoint.sh
-    ${result}=  Run Process  rm -rf /var/lib/docker/*  shell=True
-    Log To Console  Clear /var/lib/docker: ${result}
-    ${handle}=  Start Process  /usr/local/bin/dockerd-entrypoint.sh dockerd>./daemon-local.log 2>&1  shell=True
-    Process Should Be Running  ${handle}
+
+    ${result}=  Run  rm -rf /var/lib/${process}/*
+    Log All  Clear /var/lib/${process}: ${result}
+    ${handle}=  Start Process  ${start_process_cmd}  shell=True
+    Log All  handle : ${handle}
     FOR  ${IDX}  IN RANGE  5
-        ${pid}=  Run  pidof dockerd
+        ${pid}=  Run  pidof ${full_process}
+        Log All  pid : ${pid}
         Exit For Loop If  '${pid}' != '${EMPTY}'
         Sleep  2s
     END
     Sleep  2s
+    #Process Should Be Running  ${handle}
+    ${result}=  Run  ps aux |grep ${full_process}
+    Log All  result : ${result}
     [Return]  ${handle}
 
 Prepare Docker Cert
@@ -217,7 +229,7 @@ Docker Login
 
 Docker Pull
     [Arguments]  ${image}
-    ${output}=  Retry Keyword N Times When Error  2  Wait Unitl Command Success  docker pull ${image}
+    ${output}=  Retry Keyword N Times When Error  6  Wait Unitl Command Success  docker pull ${image}
     Log All  Docker Pull: ${output}
     [Return]  ${output}
 
@@ -245,6 +257,7 @@ Docker Image Can Not Be Pulled
         Log To Console  Docker pull return value is ${out}
         Sleep  3
     END
+    Clean All Local Images
     Log To Console  Cannot Pull Image From Docker - Pull Log: ${out[1]}
     Should Be Equal As Strings  '${out[0]}'  'PASS'
 
@@ -259,5 +272,6 @@ Docker Image Can Be Pulled
         Exit For Loop If  '${out[0]}'=='PASS'
         Sleep  5
     END
+    Clean All Local Images
     Run Keyword If  '${out[0]}'=='FAIL'  Capture Page Screenshot
-    Should Be Equal As Strings  '${out[0]}'  'PASS'
+    Should Be Equal As Strings  '${out[0]}'  'PASS'

tests/resources/Harbor-Pages/Project.robot

@@ -90,8 +90,9 @@ Switch To Project Tab Overflow
 
 Navigate To Projects
     Reload Page
+    Sleep  3
     Retry Element Click  xpath=${projects_xpath}
-    Sleep  2
+    Sleep  1
 
 Project Should Display
     [Arguments]  ${projectname}
@@ -235,12 +236,12 @@ Go Into Repo
     Retry Wait Until Page Not Contains Element  ${repo_list_spinner}
     ${repo_name_element}=  Set Variable  xpath=//clr-dg-cell[contains(.,'${repoName}')]/a
     FOR  ${n}  IN RANGE  1  3
-        Reload Page
         Retry Element Click  ${repo_search_icon}
         Retry Clear Element Text  ${repo_search_input}
         Retry Text Input  ${repo_search_input}  ${repoName}
         ${out}  Run Keyword And Ignore Error  Retry Wait Until Page Contains Element  ${repo_name_element}
         Sleep  2
+        Run Keyword If  '${out[0]}'=='FAIL'  Reload Page
         Continue For Loop If  '${out[0]}'=='FAIL'
         Retry Click Repo Name  ${repo_name_element}
         Sleep  2
@@ -330,7 +331,7 @@ Get Statics
 Retry Get Statics
     [Arguments]  ${locator}
     @{param}  Create List  ${locator}
-    ${ret}=  Retry Keyword N Times When Error  3  Get Statics  @{param}
+    ${ret}=  Retry Keyword N Times When Error  5  Get Statics  @{param}
     [Return]  ${ret}
 
 Get Statics Private Repo

tests/resources/Harbor-Pages/Project_Robot_Account.robot

@@ -16,7 +16,8 @@ Create A Robot Account And Return Token
     Switch To Project Robot Account
     Retry Element Click    ${project_robot_account_create_btn}
     Retry Text Input    ${project_robot_account_create_name_input}    ${robot_account_name}
-    Retry Element Click    ${project_robot_account_never_expired_chkbox}
+    Retry Element Click  xpath=//select[@id='expiration-type']
+    Retry Element Click  xpath=//select[@id='expiration-type']//option[@value='never']
     Retry Double Keywords When Error    Retry Element Click    ${project_robot_account_create_save_btn}    Retry Wait Until Page Not Contains Element    ${project_robot_account_create_save_btn}
     ${token}=    Get Value    ${project_robot_account_token_input}
     [Return]    ${token}

tests/resources/Harbor-Pages/Project_Robot_Account_Elements.robot

@@ -5,6 +5,5 @@ Documentation  This resource provides any keywords related to the Harbor robot-a
 ${project_robot_account_tabpage}  xpath=//project-detail//a[contains(.,'Robot Accounts')]
 ${project_robot_account_create_btn}  xpath=//project-detail/app-robot-account//button
 ${project_robot_account_token_input}  xpath=//app-robot-account//hbr-copy-input//input
-${project_robot_account_never_expired_chkbox}  xpath=//add-robot//clr-checkbox-wrapper/label[contains(.,'Never Expired')]
-${project_robot_account_create_name_input}  //input[@id='robot_name']
-${project_robot_account_create_save_btn}  //add-robot//button[contains(.,'SAVE')]
+${project_robot_account_create_name_input}  //input[@id='name']
+${project_robot_account_create_save_btn}  //button[@id='system-robot-save']