Added code to handle 3495 timer for both default and dedicated bearer while mme restarts

.circleci/config.yml

@@ -282,6 +282,7 @@ commands:
 
             mkdir -p versions
             cp *_version versions || true
+          no_output_timeout: 20m
       - store_artifacts:
           path: /tmp/logs
       - store_test_results:
@@ -879,6 +880,9 @@ jobs:
   docusaurus_build_and_deploy:
     docker:
       - image: circleci/node:8.11.1
+    environment:
+      DOCUSAURUS_URL: 'https://magma.github.io'
+      DOCUSAURUS_BASE_URL: '/magma/'
     steps:
       - checkout
       - run:

CODEOWNERS

@@ -8,6 +8,7 @@ ci-scripts/ @rdefosse @tmdzk
 orc8r/ @xjtian @mpgermano @hcgatewood @emakeev
 
 feg/ @themarwhal @mpgermano @uri200 @emakeev
+cwf/ @themarwhal @mpgermano @uri200 @emakeev
 
 openwrt/ @emakeev @uri200
 

README.md

@@ -25,7 +25,7 @@ Magma has three major components:
 ![Magma architecture diagram](docs/readmes/assets/magma_overview.png?raw=true "Magma Architecture")
 
 ## Usage Docs
-The documentation for developing and using Magma is available at: [https://magma.github.io/magma](https://magma.github.io/magma)
+The documentation for developing and using Magma is available at: [https://docs.magmacore.org/docs/basics/introduction.html](https://docs.magmacore.org)
 
 ## Join the Magma Community
 

ci-scripts/JenkinsFile-GitLab

@@ -28,15 +28,18 @@ def OAI_GIT_BRANCH = "master"
 // Location of the executor node
 def nodeExecutor = "libvirt"
 
+def GITHUB_USER = "magmabot"
 def slack_channel = "#magma-ci-bot"
-// lock mechanism
-def cn_ci_resource = params.MagmaVmDockerResources
 
 
 pipeline {
   agent {
     label "libvirt"
   }
+  parameters {
+    booleanParam(name: 'REGRESSION_TEST', defaultValue: false, description: 'Test master branch for regressions and submit a Github issue')
+  }
+
   options {
     timestamps()
     ansiColor('xterm')
@@ -57,11 +60,17 @@ pipeline {
     stage ("Retrieve and Prepare Source Code") {
       steps {
         script {
+          def branch
+          if (params.REGRESSION_TEST) {
+            branch = 'master'
+          } else {
+            branch = sha1
+          }
           checkout(
             changelog: false,
             poll: false,
             scm: [$class: 'GitSCM',
-                  branches: [[name: '$sha1']],
+                  branches: [[name: "$branch"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [],
                   submoduleCfg: [],
@@ -294,15 +303,15 @@ pipeline {
                 echo "Disabling TCP checksumming on all VMs"
                 sh('cd lte/gateway && vagrant ssh magma -c "sudo ethtool --offload eth1 rx off tx off && sudo ethtool --offload eth2 rx off tx off"')
                 sh('cd lte/gateway && vagrant ssh magma_test -c "sudo ethtool --offload eth1 rx off tx off && sudo ethtool --offload eth2 rx off tx off"')
+                // Adding capture on the S1 interface
+                sh('cd lte/gateway && vagrant ssh magma -c "nohup sudo tcpdump -i eth1 port 36412 -w ~/magma/archives/magma_run_s1ap_tester.pcap > /dev/null & sleep 1"')
 
                 // Making sure the Traffic server is up and running
                 sh "sleep 20"
 
                 echo "Starting the integration Tests - S1AP Tester"
                 // We have removed the traffic testcases from mandatory suite.
 
-          // FIXME!!!! set 110 MINUTES instead of 30
-
                 timeout (time: 110, unit: 'MINUTES') {
                   myShCmdWithLog('cd lte/gateway && vagrant ssh magma_test -c "cd magma/lte/gateway/python/integ_tests/ && source ~/build/python/bin/activate && make integ_test"', 'archives/magma_run_s1ap_tester.log')
                 }
@@ -333,6 +342,8 @@ pipeline {
           post {
             always {
               script {
+                // Stopping capture
+                sh('cd lte/gateway && vagrant ssh magma -c "sudo pkill tcpdump"')
                 def retrieveOAIcovFiles = true
                 try {
                   myShCmdWithLog('cd lte/gateway && vagrant ssh magma -c "cd magma/lte/gateway && make coverage_oai"', 'archives/magma_vagrant_make_coverage_oai.log')
@@ -429,8 +440,10 @@ stage ("Deploy SPGW-CUPS") {
 stage ("Test-AGW1-w-S11") {
       steps {
         script {
+          // Adding capture on the S1 and S11 interfaces
+          sh('cd lte/gateway && vagrant ssh magma -c "nohup sudo tcpdump -i any port 36412 or port 2123 -w ~/magma/archives/magma_run_s1ap_tester_s11.pcap > /dev/null & sleep 1"')
           // making sure the TRF server is up
-          echo "Remove unnecessary "
+          echo "Remove unnecessary route"
           sh('cd lte/gateway && vagrant ssh magma_trfserver -c "sudo ip route del 192.168.128.0/24 via 192.168.129.1 dev eth2"')
           sh('cd lte/gateway && vagrant reload magma_test')
           // making sure the TRF server is up
@@ -471,6 +484,8 @@ stage ("Test-AGW1-w-S11") {
         always {
           script {
             sh('cd lte/gateway && vagrant ssh magma -c "cd magma/lte/gateway && make stop"')
+            // Stopping capture
+            sh('cd lte/gateway && vagrant ssh magma -c "sudo pkill tcpdump"')
             // Retrieving the sys logs and mme log for more debugging.
             sh('cd lte/gateway && vagrant ssh magma -c "sudo cat /var/log/syslog" > ${WORKSPACE}/archives/magma_dev_syslog_s11.log')
             try {
@@ -558,6 +573,9 @@ stage ("Test-AGW1-w-S11") {
           def message = "MAGMA " + JOB_NAME + " build (" + BUILD_ID + "): failed (" + BUILD_URL + ")"
           echo message
           sendSocialMediaMessage(slack_channel,color, message)
+          if (params.REGRESSION_TEST) {
+            createOrUpdateGithubIssue(GIT_URL, GITHUB_USER, message)
+          }
       }
     }
   }
@@ -576,6 +594,67 @@ def myShCmdWithLogAppend(cmd, logFile) {
         ${cmd} 2>&1 | tee -a $WORKSPACE/${logFile}
   """
 }
+
+def createOrUpdateGithubIssue(git_url, github_user, message) {
+  issueTitle = "[CI] Regression tests failed"
+  githubProject = git_url.split('/')[1] + "/" + git_url.split('/')[2]
+  issueId = getIssueByTitle(githubProject, github_user, issueTitle)
+  if (issueId != false) {
+    updateGitHubIssue(githubProject, github_user, issueId, message)
+    println("GitHub issue #${issueId} updated")
+  } else {
+    createGitHubIssue(githubProject, github_user, issueTitle, message)
+    println("GitHub issue created")
+  }
+}
+
+def getIssueByTitle(githubProject, github_user, title) {
+  withCredentials([string(credentialsId: 'magma_bot_github_api_token', variable: 'TOKEN')]) {
+    try {
+      id = sh(returnStdout: true, script: """curl -G -u "$github_user:$TOKEN" \
+        "https://api.github.com/search/issues" \
+        -H "Accept: application/vnd.github.v3+json" \
+        --data-urlencode "q=repo:${githubProject} author:$github_user state:open in:title ${title}" \
+        | jq .items[0].number""") .trim()
+    } catch (Exception e) {
+      println("Failed looking up github issue")
+      return false
+    }
+    if (id && id != "null") {
+      println("Found matching github issue $id")
+      return id
+    } else {
+      return false
+    }
+  }
+}
+
+def updateGitHubIssue(githubProject, github_user, issueId, message) {
+  message = message.replace('\n', '\\n')
+  withCredentials([string(credentialsId: 'magma_bot_github_api_token', variable: 'TOKEN')]) {
+    sh(returnStdout: true, script: """curl -X "POST" -u "$github_user:$TOKEN" \
+       "https://api.github.com/repos/${githubProject}/issues/${issueId}/comments" \
+       -H "Accept: application/vnd.github.v3+json" \
+       -d '{"body": "${message}"}' """)
+  }
+}
+
+def createGitHubIssue(githubProject, github_user, title, message) {
+  message = message.replace('\n', '\\n')
+  withCredentials([string(credentialsId: 'magma_bot_github_api_token', variable: 'TOKEN')]) {
+    sh(returnStdout: true, script: """curl -X "POST" -u "$github_user:$TOKEN" \
+       "https://api.github.com/repos/magma/magma/issues" \
+       -H "Accept: application/vnd.github.v3+json" \
+       -d '{
+           "title": "${title}",
+           "body": "${message}",
+           "labels": [
+           "type: bug"
+           ]
+       }' """)
+  }
+}
+
 //-------------------------------------------------------------------------------
 // Abstraction function to send social media messages:
 // like on Slack or Mattermost

cwf/gateway/configs/ctraced.yml

@@ -11,3 +11,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 log_level: INFO
+
+# Directory for temporary storage of packet captures
+trace_directory: /var/opt/magma/tmp/trace
+
+# Interface to capture on
+trace_interfaces:
+  - eth0
+
+# Options available:
+#   - tshark
+#   - tcpdump
+# tshark has more capabilities - see command_builder.py
+trace_tool: tshark

cwf/gateway/configs/state.yml

@@ -13,6 +13,8 @@
 
 log_level: INFO
 
+sync_interval: 60
+
 #state_protos:
 #  - proto_file:  - file to load proto from
 #    proto_msg:   - msg to load from proto file

cwf/gateway/docker/docker-compose.override.yml

@@ -33,7 +33,7 @@ services:
 
   sessiond:
     environment:
-      MAGMA_PRINT_GRPC_PAYLOAD: 0
+      MAGMA_PRINT_GRPC_PAYLOAD: 1
     build:
       context: ${BUILD_CONTEXT}
       dockerfile: cwf/gateway/docker/c/Dockerfile

cwf/gateway/fabfile.py

@@ -49,7 +49,7 @@ def integ_test(gateway_host=None, test_host=None, trf_host=None,
                gateway_vm="cwag", gateway_ansible_file="cwag_dev.yml",
                transfer_images=False, destroy_vm=False, no_build=False,
                tests_to_run="all", skip_unit_tests=False, test_re=None,
-               test_result_xml=None, run_tests=True):
+               test_result_xml=None, run_tests=True, count="1"):
     """
     Run the integration tests. This defaults to running on local vagrant
     machines, but can also be pointed to an arbitrary host (e.g. amazon) by
@@ -129,8 +129,8 @@ def integ_test(gateway_host=None, test_host=None, trf_host=None,
     execute(_set_cwag_networking, cwag_test_br_mac)
 
     # check if docker services are alive except for OCS2 and PCRF2
-    ignoreList = ["ocs2", "pcrf2"]
-    execute(_check_docker_services, ignoreList)
+    ignore_list = ["ocs2", "pcrf2"]
+    execute(_check_docker_services, ignore_list)
 
     _switch_to_vm_no_destroy(gateway_host, "cwag_test", "cwag_test.yml")
     execute(_start_ue_simulator)
@@ -141,14 +141,15 @@ def integ_test(gateway_host=None, test_host=None, trf_host=None,
         print("run_test was set to false. Test will not be run\n"
               "You can now run the tests manually from cwag_test")
         sys.exit(0)
-
+    
     # HSSLESS tests are to be executed from gateway_host VM
     if tests_to_run.value == SubTests.HSSLESS.value:
         _switch_to_vm_no_destroy(gateway_host, gateway_vm, gateway_ansible_file)
-        execute(_run_integ_tests, gateway_host, trf_host, tests_to_run, test_re)
+        execute(_run_integ_tests, gateway_host, trf_host,
+                tests_to_run, test_re, count, test_result_xml)
     else:
         execute(_run_integ_tests, test_host, trf_host,
-                tests_to_run, test_re, test_result_xml)
+                tests_to_run, test_re, count, test_result_xml)
 
     # If we got here means everything work well!!
     if not test_host and not trf_host:
@@ -340,17 +341,17 @@ def _stop_docker_services(services):
         )
 
 
-def _check_docker_services(ignoreList):
+def _check_docker_services(ignore_list):
     with cd(CWAG_ROOT + "/docker"), settings(warn_only=True), hide("warnings"):
 
-        grepIgnore = "| grep --invert-match '" + \
-                     '\|'.join(ignoreList) + "'" if ignoreList else ""
+        grep_ignore = "| grep --invert-match '" + \
+                     '\|'.join(ignore_list) + "'" if ignore_list else ""
         count = 0
         while (count < 5):
             # force wait to make sure docker logs are up
             time.sleep(1)
             result = run(" docker ps --format \"{{.Names}}\t{{.Status}}\" | "
-                         "grep Restarting" + grepIgnore )
+                         "grep Restarting" + grep_ignore )
 
             if result.return_code == 1:
                 # grep returns code 1 when empty string
@@ -387,7 +388,7 @@ def _add_docker_host_remote_network_envvar():
 
 
 def _run_integ_tests(test_host, trf_host, tests_to_run: SubTests,
-                     test_re=None, test_result_xml=None):
+                     test_re=None, count="1", test_result_xml=None):
     """ Run the integration tests """
     # add docker host environment as well
     shell_env_vars = {
@@ -401,7 +402,7 @@ def _run_integ_tests(test_host, trf_host, tests_to_run: SubTests,
     go_test_cmd = "gotestsum --format=standard-verbose "
     if test_result_xml: # generate test result XML in cwf/gateway directory
         go_test_cmd += "--junitfile ../" + test_result_xml + " "
-    go_test_cmd += " -- -test.short -timeout 50m" # go test args
+    go_test_cmd += " -- -test.short -timeout 50m -count " + count # go test args
     go_test_cmd += " -tags=" + tests_to_run.value
     if test_re:
         go_test_cmd += " -run=" + test_re

cwf/gateway/go.mod

@@ -49,7 +49,7 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/vishvananda/netlink v1.1.0
 	golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897
-	golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0
+	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
 	google.golang.org/appengine v1.6.5 // indirect
 	google.golang.org/grpc v1.33.2
 	magma/cwf/cloud/go v0.0.0-00010101000000-000000000000