diff --git a/.circleci/config.yml b/.circleci/config.yml index 64bbeab604c0..667ab173540f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -282,6 +282,7 @@ commands: mkdir -p versions cp *_version versions || true + no_output_timeout: 20m - store_artifacts: path: /tmp/logs - store_test_results: @@ -879,6 +880,9 @@ jobs: docusaurus_build_and_deploy: docker: - image: circleci/node:8.11.1 + environment: + DOCUSAURUS_URL: 'https://magma.github.io' + DOCUSAURUS_BASE_URL: '/magma/' steps: - checkout - run: diff --git a/CODEOWNERS b/CODEOWNERS index 77b98332ac96..3c3a9945b815 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -8,6 +8,7 @@ ci-scripts/ @rdefosse @tmdzk orc8r/ @xjtian @mpgermano @hcgatewood @emakeev feg/ @themarwhal @mpgermano @uri200 @emakeev +cwf/ @themarwhal @mpgermano @uri200 @emakeev openwrt/ @emakeev @uri200 diff --git a/README.md b/README.md index 0beb58a61382..d01a9a7ee391 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Magma has three major components: ![Magma architecture diagram](docs/readmes/assets/magma_overview.png?raw=true "Magma Architecture") ## Usage Docs -The documentation for developing and using Magma is available at: [https://magma.github.io/magma](https://magma.github.io/magma) +The documentation for developing and using Magma is available at: [https://docs.magmacore.org/docs/basics/introduction.html](https://docs.magmacore.org) ## Join the Magma Community diff --git a/ci-scripts/JenkinsFile-GitLab b/ci-scripts/JenkinsFile-GitLab index 56097e9e5675..08784b3fc679 100644 --- a/ci-scripts/JenkinsFile-GitLab +++ b/ci-scripts/JenkinsFile-GitLab @@ -28,15 +28,18 @@ def OAI_GIT_BRANCH = "master" // Location of the executor node def nodeExecutor = "libvirt" +def GITHUB_USER = "magmabot" def slack_channel = "#magma-ci-bot" -// lock mechanism -def cn_ci_resource = params.MagmaVmDockerResources pipeline { agent { label "libvirt" } + parameters { + booleanParam(name: 'REGRESSION_TEST', defaultValue: false, description: 'Test master branch for regressions and submit a Github issue') + } + options { timestamps() ansiColor('xterm') @@ -57,11 +60,17 @@ pipeline { stage ("Retrieve and Prepare Source Code") { steps { script { + def branch + if (params.REGRESSION_TEST) { + branch = 'master' + } else { + branch = sha1 + } checkout( changelog: false, poll: false, scm: [$class: 'GitSCM', - branches: [[name: '$sha1']], + branches: [[name: "$branch"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], @@ -294,6 +303,8 @@ pipeline { echo "Disabling TCP checksumming on all VMs" sh('cd lte/gateway && vagrant ssh magma -c "sudo ethtool --offload eth1 rx off tx off && sudo ethtool --offload eth2 rx off tx off"') sh('cd lte/gateway && vagrant ssh magma_test -c "sudo ethtool --offload eth1 rx off tx off && sudo ethtool --offload eth2 rx off tx off"') + // Adding capture on the S1 interface + sh('cd lte/gateway && vagrant ssh magma -c "nohup sudo tcpdump -i eth1 port 36412 -w ~/magma/archives/magma_run_s1ap_tester.pcap > /dev/null & sleep 1"') // Making sure the Traffic server is up and running sh "sleep 20" @@ -301,8 +312,6 @@ pipeline { echo "Starting the integration Tests - S1AP Tester" // We have removed the traffic testcases from mandatory suite. - // FIXME!!!! set 110 MINUTES instead of 30 - timeout (time: 110, unit: 'MINUTES') { myShCmdWithLog('cd lte/gateway && vagrant ssh magma_test -c "cd magma/lte/gateway/python/integ_tests/ && source ~/build/python/bin/activate && make integ_test"', 'archives/magma_run_s1ap_tester.log') } @@ -333,6 +342,8 @@ pipeline { post { always { script { + // Stopping capture + sh('cd lte/gateway && vagrant ssh magma -c "sudo pkill tcpdump"') def retrieveOAIcovFiles = true try { myShCmdWithLog('cd lte/gateway && vagrant ssh magma -c "cd magma/lte/gateway && make coverage_oai"', 'archives/magma_vagrant_make_coverage_oai.log') @@ -429,8 +440,10 @@ stage ("Deploy SPGW-CUPS") { stage ("Test-AGW1-w-S11") { steps { script { + // Adding capture on the S1 and S11 interfaces + sh('cd lte/gateway && vagrant ssh magma -c "nohup sudo tcpdump -i any port 36412 or port 2123 -w ~/magma/archives/magma_run_s1ap_tester_s11.pcap > /dev/null & sleep 1"') // making sure the TRF server is up - echo "Remove unnecessary " + echo "Remove unnecessary route" sh('cd lte/gateway && vagrant ssh magma_trfserver -c "sudo ip route del 192.168.128.0/24 via 192.168.129.1 dev eth2"') sh('cd lte/gateway && vagrant reload magma_test') // making sure the TRF server is up @@ -471,6 +484,8 @@ stage ("Test-AGW1-w-S11") { always { script { sh('cd lte/gateway && vagrant ssh magma -c "cd magma/lte/gateway && make stop"') + // Stopping capture + sh('cd lte/gateway && vagrant ssh magma -c "sudo pkill tcpdump"') // Retrieving the sys logs and mme log for more debugging. sh('cd lte/gateway && vagrant ssh magma -c "sudo cat /var/log/syslog" > ${WORKSPACE}/archives/magma_dev_syslog_s11.log') try { @@ -558,6 +573,9 @@ stage ("Test-AGW1-w-S11") { def message = "MAGMA " + JOB_NAME + " build (" + BUILD_ID + "): failed (" + BUILD_URL + ")" echo message sendSocialMediaMessage(slack_channel,color, message) + if (params.REGRESSION_TEST) { + createOrUpdateGithubIssue(GIT_URL, GITHUB_USER, message) + } } } } @@ -576,6 +594,67 @@ def myShCmdWithLogAppend(cmd, logFile) { ${cmd} 2>&1 | tee -a $WORKSPACE/${logFile} """ } + +def createOrUpdateGithubIssue(git_url, github_user, message) { + issueTitle = "[CI] Regression tests failed" + githubProject = git_url.split('/')[1] + "/" + git_url.split('/')[2] + issueId = getIssueByTitle(githubProject, github_user, issueTitle) + if (issueId != false) { + updateGitHubIssue(githubProject, github_user, issueId, message) + println("GitHub issue #${issueId} updated") + } else { + createGitHubIssue(githubProject, github_user, issueTitle, message) + println("GitHub issue created") + } +} + +def getIssueByTitle(githubProject, github_user, title) { + withCredentials([string(credentialsId: 'magma_bot_github_api_token', variable: 'TOKEN')]) { + try { + id = sh(returnStdout: true, script: """curl -G -u "$github_user:$TOKEN" \ + "https://api.github.com/search/issues" \ + -H "Accept: application/vnd.github.v3+json" \ + --data-urlencode "q=repo:${githubProject} author:$github_user state:open in:title ${title}" \ + | jq .items[0].number""") .trim() + } catch (Exception e) { + println("Failed looking up github issue") + return false + } + if (id && id != "null") { + println("Found matching github issue $id") + return id + } else { + return false + } + } +} + +def updateGitHubIssue(githubProject, github_user, issueId, message) { + message = message.replace('\n', '\\n') + withCredentials([string(credentialsId: 'magma_bot_github_api_token', variable: 'TOKEN')]) { + sh(returnStdout: true, script: """curl -X "POST" -u "$github_user:$TOKEN" \ + "https://api.github.com/repos/${githubProject}/issues/${issueId}/comments" \ + -H "Accept: application/vnd.github.v3+json" \ + -d '{"body": "${message}"}' """) + } +} + +def createGitHubIssue(githubProject, github_user, title, message) { + message = message.replace('\n', '\\n') + withCredentials([string(credentialsId: 'magma_bot_github_api_token', variable: 'TOKEN')]) { + sh(returnStdout: true, script: """curl -X "POST" -u "$github_user:$TOKEN" \ + "https://api.github.com/repos/magma/magma/issues" \ + -H "Accept: application/vnd.github.v3+json" \ + -d '{ + "title": "${title}", + "body": "${message}", + "labels": [ + "type: bug" + ] + }' """) + } +} + //------------------------------------------------------------------------------- // Abstraction function to send social media messages: // like on Slack or Mattermost diff --git a/cwf/gateway/configs/ctraced.yml b/cwf/gateway/configs/ctraced.yml index e4ef28aba475..79d383fbd944 100644 --- a/cwf/gateway/configs/ctraced.yml +++ b/cwf/gateway/configs/ctraced.yml @@ -11,3 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. log_level: INFO + +# Directory for temporary storage of packet captures +trace_directory: /var/opt/magma/tmp/trace + +# Interface to capture on +trace_interfaces: + - eth0 + +# Options available: +# - tshark +# - tcpdump +# tshark has more capabilities - see command_builder.py +trace_tool: tshark diff --git a/cwf/gateway/configs/state.yml b/cwf/gateway/configs/state.yml index cee266b1ebed..314e69f2666c 100644 --- a/cwf/gateway/configs/state.yml +++ b/cwf/gateway/configs/state.yml @@ -13,6 +13,8 @@ log_level: INFO +sync_interval: 60 + #state_protos: # - proto_file: - file to load proto from # proto_msg: - msg to load from proto file diff --git a/cwf/gateway/docker/docker-compose.override.yml b/cwf/gateway/docker/docker-compose.override.yml index 5c32849f72ca..8811e16f695e 100644 --- a/cwf/gateway/docker/docker-compose.override.yml +++ b/cwf/gateway/docker/docker-compose.override.yml @@ -33,7 +33,7 @@ services: sessiond: environment: - MAGMA_PRINT_GRPC_PAYLOAD: 0 + MAGMA_PRINT_GRPC_PAYLOAD: 1 build: context: ${BUILD_CONTEXT} dockerfile: cwf/gateway/docker/c/Dockerfile diff --git a/cwf/gateway/fabfile.py b/cwf/gateway/fabfile.py index a1e950c291b8..c807be66dd55 100644 --- a/cwf/gateway/fabfile.py +++ b/cwf/gateway/fabfile.py @@ -49,7 +49,7 @@ def integ_test(gateway_host=None, test_host=None, trf_host=None, gateway_vm="cwag", gateway_ansible_file="cwag_dev.yml", transfer_images=False, destroy_vm=False, no_build=False, tests_to_run="all", skip_unit_tests=False, test_re=None, - test_result_xml=None, run_tests=True): + test_result_xml=None, run_tests=True, count="1"): """ Run the integration tests. This defaults to running on local vagrant machines, but can also be pointed to an arbitrary host (e.g. amazon) by @@ -129,8 +129,8 @@ def integ_test(gateway_host=None, test_host=None, trf_host=None, execute(_set_cwag_networking, cwag_test_br_mac) # check if docker services are alive except for OCS2 and PCRF2 - ignoreList = ["ocs2", "pcrf2"] - execute(_check_docker_services, ignoreList) + ignore_list = ["ocs2", "pcrf2"] + execute(_check_docker_services, ignore_list) _switch_to_vm_no_destroy(gateway_host, "cwag_test", "cwag_test.yml") execute(_start_ue_simulator) @@ -141,14 +141,15 @@ def integ_test(gateway_host=None, test_host=None, trf_host=None, print("run_test was set to false. Test will not be run\n" "You can now run the tests manually from cwag_test") sys.exit(0) - + # HSSLESS tests are to be executed from gateway_host VM if tests_to_run.value == SubTests.HSSLESS.value: _switch_to_vm_no_destroy(gateway_host, gateway_vm, gateway_ansible_file) - execute(_run_integ_tests, gateway_host, trf_host, tests_to_run, test_re) + execute(_run_integ_tests, gateway_host, trf_host, + tests_to_run, test_re, count, test_result_xml) else: execute(_run_integ_tests, test_host, trf_host, - tests_to_run, test_re, test_result_xml) + tests_to_run, test_re, count, test_result_xml) # If we got here means everything work well!! if not test_host and not trf_host: @@ -340,17 +341,17 @@ def _stop_docker_services(services): ) -def _check_docker_services(ignoreList): +def _check_docker_services(ignore_list): with cd(CWAG_ROOT + "/docker"), settings(warn_only=True), hide("warnings"): - grepIgnore = "| grep --invert-match '" + \ - '\|'.join(ignoreList) + "'" if ignoreList else "" + grep_ignore = "| grep --invert-match '" + \ + '\|'.join(ignore_list) + "'" if ignore_list else "" count = 0 while (count < 5): # force wait to make sure docker logs are up time.sleep(1) result = run(" docker ps --format \"{{.Names}}\t{{.Status}}\" | " - "grep Restarting" + grepIgnore ) + "grep Restarting" + grep_ignore ) if result.return_code == 1: # grep returns code 1 when empty string @@ -387,7 +388,7 @@ def _add_docker_host_remote_network_envvar(): def _run_integ_tests(test_host, trf_host, tests_to_run: SubTests, - test_re=None, test_result_xml=None): + test_re=None, count="1", test_result_xml=None): """ Run the integration tests """ # add docker host environment as well shell_env_vars = { @@ -401,7 +402,7 @@ def _run_integ_tests(test_host, trf_host, tests_to_run: SubTests, go_test_cmd = "gotestsum --format=standard-verbose " if test_result_xml: # generate test result XML in cwf/gateway directory go_test_cmd += "--junitfile ../" + test_result_xml + " " - go_test_cmd += " -- -test.short -timeout 50m" # go test args + go_test_cmd += " -- -test.short -timeout 50m -count " + count # go test args go_test_cmd += " -tags=" + tests_to_run.value if test_re: go_test_cmd += " -run=" + test_re diff --git a/cwf/gateway/go.mod b/cwf/gateway/go.mod index afd5e4ca3f94..2dca54b2982f 100644 --- a/cwf/gateway/go.mod +++ b/cwf/gateway/go.mod @@ -49,7 +49,7 @@ require ( github.com/stretchr/testify v1.6.1 github.com/vishvananda/netlink v1.1.0 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 - golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 + golang.org/x/net v0.0.0-20201110031124-69a78807bb2b google.golang.org/appengine v1.6.5 // indirect google.golang.org/grpc v1.33.2 magma/cwf/cloud/go v0.0.0-00010101000000-000000000000 diff --git a/cwf/gateway/go.sum b/cwf/gateway/go.sum index cd80b21c4692..5d1acaeacd72 100644 --- a/cwf/gateway/go.sum +++ b/cwf/gateway/go.sum @@ -244,6 +244,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -256,6 +257,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -265,6 +267,7 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -425,6 +428,7 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -479,6 +483,7 @@ github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSg github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rlmcpherson/s3gof3r v0.5.0/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs= github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= @@ -550,6 +555,7 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/warthog618/sms v0.3.0/go.mod h1:+bYZGeBxu003sxD5xhzsrIPBAjPBzTABsRTwSpd7ld4= +github.com/wmnsk/go-gtp v0.7.15/go.mod h1:v1psjZ7skpPSDegH23Amg9rNufs0BoXNM+GBtW5t58I= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -618,6 +624,8 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -704,6 +712,7 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -717,6 +726,7 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -727,6 +737,7 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= diff --git a/cwf/gateway/integ_tests/gateway.mconfig b/cwf/gateway/integ_tests/gateway.mconfig index 59ed4c1df64d..affafe288621 100644 --- a/cwf/gateway/integ_tests/gateway.mconfig +++ b/cwf/gateway/integ_tests/gateway.mconfig @@ -57,7 +57,7 @@ "relayEnabled": true, "gxGyRelayEnabled": true, "walletExhaustDetection": { - "terminateOnExhaust": true + "terminateOnExhaust": false } }, "aaa_server": { diff --git a/cwf/gateway/integ_tests/gx_enforcement_test.go b/cwf/gateway/integ_tests/gx_enforcement_test.go index 7e08a33079a2..e576d2817833 100644 --- a/cwf/gateway/integ_tests/gx_enforcement_test.go +++ b/cwf/gateway/integ_tests/gx_enforcement_test.go @@ -168,7 +168,9 @@ func TestGxMidSessionRuleRemovalWithCCA_U(t *testing.T) { assert.NoError(t, setPCRFExpectations(expectations, defaultUpdateAnswer)) tr.AuthenticateAndAssertSuccess(imsi) - tr.WaitForEnforcementStatsToSync() + + assert.Eventually(t, tr.WaitForEnforcementStatsForRule(imsi, "static-pass-all-1", "static-pass-all-3"), time.Minute, 2*time.Second) + req := &cwfprotos.GenTrafficRequest{Imsi: imsi, Volume: &wrappers.StringValue{Value: "250K"}} _, err = tr.GenULTraffic(req) assert.NoError(t, err) @@ -177,16 +179,6 @@ func TestGxMidSessionRuleRemovalWithCCA_U(t *testing.T) { // Since static-pass-all-1 has higher precedence, it will get hit. tr.WaitForEnforcementStatsToSync() - // Assert that enforcement_stats rules are properly installed and the right - // amount of data was passed through - recordsBySubID, err := tr.GetPolicyUsage() - assert.NoError(t, err) - record1 := recordsBySubID[prependIMSIPrefix(imsi)]["static-pass-all-1"] - if record1 != nil { - assert.True(t, record1.BytesTx > uint64(0), fmt.Sprintf("%s did not pass any data", record1.RuleId)) - } - assert.NotNil(t, record1, fmt.Sprintf("No policy usage record for imsi: %v rule=static-pass-all-1", imsi)) - // Assert that a CCR-I was sent up to the PCRF tr.AssertAllGxExpectationsMetNoError() @@ -206,7 +198,8 @@ func TestGxMidSessionRuleRemovalWithCCA_U(t *testing.T) { // Generate traffic to trigger the CCR-U so that the rule removal/install happens _, err = tr.GenULTraffic(req) assert.NoError(t, err) - tr.WaitForEnforcementStatsToSync() + + assert.Eventually(t, tr.WaitForEnforcementStatsForRule(imsi, "static-pass-all-2"), time.Minute, 2*time.Second) fmt.Println("Generating traffic again to put data through static-pass-all-2") _, err = tr.GenULTraffic(req) @@ -216,9 +209,7 @@ func TestGxMidSessionRuleRemovalWithCCA_U(t *testing.T) { // Assert that we sent back a CCA-Update with RuleRemovals tr.AssertAllGxExpectationsMetNoError() - recordsBySubID, err = tr.GetPolicyUsage() - assert.NoError(t, err) - assert.NotNil(t, recordsBySubID[prependIMSIPrefix(imsi)]["static-pass-all-2"], fmt.Sprintf("No policy usage record for imsi: %v rule=static-pass-all-2", imsi)) + assert.Eventually(t, tr.WaitForEnforcementStatsForRuleGreaterThan(imsi, "static-pass-all-2", 0), time.Minute, 2*time.Second) tr.DisconnectAndAssertSuccess(imsi) fmt.Println("wait for flows to get deactivated") diff --git a/cwf/gateway/integ_tests/gx_reauth_test.go b/cwf/gateway/integ_tests/gx_reauth_test.go index 33c983211c47..59f0bfc0c327 100644 --- a/cwf/gateway/integ_tests/gx_reauth_test.go +++ b/cwf/gateway/integ_tests/gx_reauth_test.go @@ -372,7 +372,7 @@ func TestGxReAuthWithMidSessionPolicyInstallAndRemoval(t *testing.T) { // and a rule base "base-raa1" // - Generate traffic and assert that there's > 0 data usage for the rule with the // highest priority. -// - Send a PCRF ReAuth request to refill quoto for a session +// - Send a PCRF ReAuth request to refill quota for a session // - Assert that the response is successful // - Generate traffic and assert that there's > 0 data usage for the newly installed // rule. @@ -399,7 +399,7 @@ func TestGxReAuthQuotaRefill(t *testing.T) { } _, err := tr.GenULTraffic(req) assert.NoError(t, err) - tr.WaitForEnforcementStatsToSync() + assert.Eventually(t, tr.WaitForEnforcementStatsForRule(imsi, "static-pass-all-raa1"), time.Minute, 2*time.Second) // Check that enforcement flow is installed and traffic is less than the quota tr.AssertPolicyUsage(imsi, "static-pass-all-raa1", 0, 500*KiloBytes+Buffer) diff --git a/cwf/gateway/integ_tests/gy_enforcement_test.go b/cwf/gateway/integ_tests/gy_enforcement_test.go index dabbcbc4af99..163434c5f4e5 100644 --- a/cwf/gateway/integ_tests/gy_enforcement_test.go +++ b/cwf/gateway/integ_tests/gy_enforcement_test.go @@ -348,7 +348,7 @@ func TestGyCreditExhaustionWithoutCRRU(t *testing.T) { // - Set an expectation for a CCR-I to be sent up to OCS, to which it will // NOT respond with any answer. -// - Asset that authentication fails and that no rules were insalled +// - Asset that authentication fails and that no rules were installed func TestGyLinksFailureOCStoFEG(t *testing.T) { fmt.Println("\nRunning TestGyLinksFailureOCStoFEG...") @@ -392,8 +392,9 @@ func TestGyLinksFailureOCStoFEG(t *testing.T) { // - Assert that CCR-U was is generated // - Generate 2M traffic and assert that UE flows are NOT deleted and data was passed. // - Expect a CCR-T, trigger a UE disconnect, and assert the CCR-T is received. -// NOTE : the test is only verifying that session was not terminated. Improvment is needed to validate -// that ovs rule is well added and traffic is being redirected. +// NOTE : the test is only verifying that session was not terminated. +// Improvement is needed to validate that ovs rule is well added and +// traffic is being redirected. func TestGyCreditExhaustionRedirect(t *testing.T) { fmt.Println("\nRunning TestGyCreditExhaustionRedirect...") @@ -453,7 +454,7 @@ func TestGyCreditExhaustionRedirect(t *testing.T) { assert.NoError(t, err) tr.WaitForEnforcementStatsToSync() - // Check that UE mac flow was not removed and data was passed + // Check that enforcement stats flow was not removed and data was passed tr.AssertPolicyUsage(imsi, "static-pass-all-ocs2", 0, 5*MegaBytes+Buffer) // Wait for service deactivation @@ -481,7 +482,7 @@ func TestGyCreditExhaustionRedirect(t *testing.T) { assert.NoError(t, err) tr.WaitForEnforcementStatsToSync() - // Check that UE mac flow was not removed and data was passed + // Check that enforcement stats flow was not removed and data was passed tr.AssertPolicyUsage(imsi, "static-pass-all-ocs2", 0, 7*MegaBytes+Buffer) // When we initiate a UE disconnect, we expect a terminate request to go up @@ -624,7 +625,7 @@ func TestGyAbortSessionRequest(t *testing.T) { assert.NoError(t, err) tr.WaitForEnforcementStatsToSync() - // Check that UE mac flow is installed and traffic is less than the quota + // Check that enforcement stats flow is installed and traffic is less than the quota tr.AssertPolicyUsage(imsi, "static-pass-all-ocs2", 0, 5*MegaBytes+Buffer) asa, err := sendChargingAbortSession( @@ -643,13 +644,7 @@ func TestGyAbortSessionRequest(t *testing.T) { assert.Equal(t, uint32(diam.LimitedSuccess), asa.ResultCode) // check if all session related info is cleaned up - checkSessionAborted := func() bool { - recordsBySubID, err := tr.GetPolicyUsage() - assert.NoError(t, err) - return recordsBySubID["IMSI"+imsi]["static-pass-all-ocs2"] == nil - } - assert.Eventually(t, checkSessionAborted, 2*time.Minute, 5*time.Second, - "request not terminated as expected") + assert.Eventually(t, tr.WaitForNoEnforcementStatsForRule(imsi, "static-pass-all-ocs2"), 2*time.Minute, 5*time.Second) // trigger disconnection tr.DisconnectAndAssertSuccess(imsi) @@ -741,7 +736,7 @@ func TestGyCreditExhaustionRestrict(t *testing.T) { assert.NoError(t, err) tr.WaitForEnforcementStatsToSync() - // Check that UE mac flow was not removed and flow data hit restrict rule + // Check that enforcement stats flow was not removed and flow data hit restrict rule tr.AssertPolicyUsage(imsi, "restrict-pass-user", uint64(math.Round(1.8*MegaBytes)), 3*MegaBytes+Buffer) // Send ReAuth Request to update quota @@ -769,7 +764,7 @@ func TestGyCreditExhaustionRestrict(t *testing.T) { assert.NoError(t, err) tr.WaitForEnforcementStatsToSync() - // Check that UE mac flow was not removed and data passed + // Check that enforcement stats flow was not removed and data passed tr.AssertPolicyUsage(imsi, "static-pass-all-ocs2", uint64(math.Round(1.8*MegaBytes)), 3*MegaBytes+Buffer) // trigger disconnection @@ -788,7 +783,7 @@ func TestGyCreditExhaustionRestrict(t *testing.T) { // - Generate 2M traffic and assert that UE flows are reinstalled for RG // and traffic goes through them. func TestGyCreditTransientErrorRestrict(t *testing.T) { - fmt.Println("\nRunning TestGyCreditExhaustionRestrict...") + fmt.Println("\nRunning TestGyCreditTransientErrorRestrict...") tr, ruleManager, ue := ocsTestSetupSingleRule(t) imsi := ue.GetImsi() @@ -849,12 +844,7 @@ func TestGyCreditTransientErrorRestrict(t *testing.T) { tr.AuthenticateAndAssertSuccess(imsi) // by this point we should be already redirected since credit was suspended - - // Update directoryd record to include client IP - err := updateDirectorydRecord("IMSI"+imsi, "ipv4_addr", TrafficCltIP) - assert.NoError(t, err) - - tr.WaitForEnforcementStatsToSync() + assert.Eventually(t, tr.WaitForEnforcementStatsForRule(imsi, "restrict-pass-user"), time.Minute, 2*time.Second) // Wait for service deactivation time.Sleep(3 * time.Second) @@ -865,15 +855,14 @@ func TestGyCreditTransientErrorRestrict(t *testing.T) { Volume: &wrappers.StringValue{Value: "2M"}, Timeout: 20, } - _, err = tr.GenULTraffic(req) + _, err := tr.GenULTraffic(req) assert.NoError(t, err) tr.WaitForEnforcementStatsToSync() - // Check that UE mac flow was not removed and flow data hit restrict rule + // Check that enforcement stats flow was not removed and flow data hit restrict rule tr.AssertPolicyUsage(imsi, "restrict-pass-user", uint64(math.Round(1.5*MegaBytes)), 3*MegaBytes+Buffer) // check static rule is gone - policyUsage, err := tr.GetPolicyUsage() - assert.Nil(t, policyUsage["IMSI"+imsi]["static-pass-all-ocs1"], fmt.Sprintf("Policy usage record2 for imsi: %v was NOT removed", imsi)) + assert.Eventually(t, tr.WaitForNoEnforcementStatsForRule(imsi, "static-pass-all-ocs1"), time.Minute, 2*time.Second) // Send ReAuth Request to update quota raa, err := sendChargingReAuthRequestEntireSession(imsi) @@ -898,8 +887,8 @@ func TestGyCreditTransientErrorRestrict(t *testing.T) { assert.NoError(t, err) tr.WaitForEnforcementStatsToSync() - // TODO: uncoment once we fix passing the ip to pipelined for cwf - // Check that UE mac flow was not removed and data passed + // TODO: uncomment once we fix passing the ip to pipelined for cwf + // Check that enforcement stats flow was not removed and data passed //tr.AssertPolicyUsage(imsi, "static-pass-all-ocs1", uint64(math.Round(1.5*MegaBytes)), 3*MegaBytes+Buffer) //assert.Nil(t, policyUsage["IMSI"+imsi]["restrict-pass-user"], fmt.Sprintf("Policy usage restrict-pass-user for imsi: %v was NOT removed", imsi)) @@ -918,7 +907,7 @@ func TestGyCreditTransientErrorRestrict(t *testing.T) { // - Assert that UE flows for one rule are delete. // - Assert that UE flows for the other rule are still valid func TestGyWithTransientErrorCode(t *testing.T) { - fmt.Println("\nRunning TestGyWithErrorCode...") + fmt.Println("\nRunning TestGyWithTransientErrorCode...") tr, ruleManager, ue := ocsTestSetup(t) imsi := ue.GetImsi() @@ -969,20 +958,9 @@ func TestGyWithTransientErrorCode(t *testing.T) { } _, err := tr.GenULTraffic(req) assert.NoError(t, err) - tr.WaitForEnforcementStatsToSync() - // Wait for flow deletion due to quota exhaustion - tr.WaitForEnforcementStatsToSync() - - // Check that one of the flows is removed but session is not terminated - preSuspension_recordsBySubID, err := tr.GetPolicyUsage() - assert.NoError(t, err) - preSuspensionRecord1 := preSuspension_recordsBySubID["IMSI"+imsi]["static-pass-all-ocs1"] - assert.NotNil(t, preSuspensionRecord1, fmt.Sprintf("Policy usage record1 for imsi: %v was removed", imsi)) - - // TODO: uncoment once we fix passing the ip to pipelined for cwf - //preSuspensionRecord2 := preSuspension_recordsBySubID["IMSI"+imsi]["static-pass-all-ocs2"] - //assert.Nil(t, preSuspensionRecord2, fmt.Sprintf("Policy usage record2 for imsi: %v was NOT removed", imsi)) + fmt.Println("RG 1 should now be suspended") + assert.Eventually(t, tr.WaitForNoEnforcementStatsForRule(imsi, "static-pass-all-ocs2"), time.Minute, 2*time.Second) // Assert that we saw a Terminate request tr.AssertAllGyExpectationsMetNoError() @@ -1000,8 +978,8 @@ func TestGyWithTransientErrorCode(t *testing.T) { // - Send an CCA-U with a 5xxx code which should trigger termination // - Assert that UE flows are deleted. // - Expect a CCR-T, trigger a UE disconnect, and assert the CCR-T is received. -func TestGyWithPermanetErrorCode(t *testing.T) { - fmt.Println("\nRunning TestGyWithErrorCode...") +func TestGyWithPermanentErrorCode(t *testing.T) { + fmt.Println("\nRunning TestGyWithPermanentErrorCode...") tr, ruleManager, ue := ocsTestSetup(t) imsi := ue.GetImsi() @@ -1057,16 +1035,9 @@ func TestGyWithPermanetErrorCode(t *testing.T) { } _, err := tr.GenULTraffic(req) assert.NoError(t, err) - tr.WaitForEnforcementStatsToSync() - - // Wait for flow deletion due to quota exhaustion - tr.WaitForEnforcementStatsToSync() - // Check that UE mac flow is removed - recordsBySubID, err := tr.GetPolicyUsage() - assert.NoError(t, err) - record := recordsBySubID["IMSI"+imsi]["static-pass-all-ocs2"] - assert.Nil(t, record, fmt.Sprintf("Policy usage record for imsi: %v was not removed", imsi)) + // Check that enforcement stats flow is removed + assert.Eventually(t, tr.WaitForNoEnforcementStatsForRule(imsi, "static-pass-all-ocs2"), time.Minute, 2*time.Second) // Assert that we saw a Terminate request tr.AssertAllGyExpectationsMetNoError() diff --git a/cwf/gateway/integ_tests/omni_rules_test.go b/cwf/gateway/integ_tests/omni_rules_test.go index 2cf22328473e..da615b427064 100644 --- a/cwf/gateway/integ_tests/omni_rules_test.go +++ b/cwf/gateway/integ_tests/omni_rules_test.go @@ -23,6 +23,7 @@ import ( cwfprotos "magma/cwf/cloud/go/protos" "magma/feg/cloud/go/protos" fegprotos "magma/feg/cloud/go/protos" + "magma/feg/gateway/diameter" "magma/lte/cloud/go/services/policydb/obsidian/models" "github.com/fiorix/go-diameter/v4/diam" @@ -42,8 +43,7 @@ import ( // - Assert that the traffic goes through. This means the network wide rules // gets installed properly. // - Trigger a Gx RAR with a rule removal for the block all rule. Assert the -// answer is successful. Since the only rule with a usage monitor is removed, -// the session will terminate. Assert that policy usage is empty. +// answer is successful. func TestOmnipresentRules(t *testing.T) { fmt.Println("\nRunning TestOmnipresentRules...") tr := NewTestRunner(t) @@ -140,13 +140,7 @@ func TestOmnipresentRules(t *testing.T) { assert.Eventually(t, tr.WaitForPolicyReAuthToProcess(raa, imsi), time.Minute, 2*time.Second) // Check ReAuth success - fmt.Printf("RAA result code=%v, should be=%v\n", int(raa.ResultCode), diam.Success) - //assert.Equal(t, diam.Success, int(raa.ResultCode)) - - // With all monitored rules gone, the session should terminate - recordsBySubID, err = tr.GetPolicyUsage() - assert.NoError(t, err) - assert.Empty(t, recordsBySubID[prependIMSIPrefix(imsi)]) + assert.Equal(t, int(raa.ResultCode), diameter.SuccessCode) // trigger disconnection tr.DisconnectAndAssertSuccess(imsi) @@ -155,7 +149,8 @@ func TestOmnipresentRules(t *testing.T) { } // TODO: test disabled for now. Need to modify mconfig to enable/disable Gx -func testGxDisabledOmnipresentRules(t *testing.T) { +func TestGxDisabledOmnipresentRules(t *testing.T) { + t.Skip() fmt.Println("\nRunning TestOmnipresentRulesGxDisabled...") tr := NewTestRunner(t) ruleManager, err := NewRuleManager() diff --git a/cwf/gateway/integ_tests/pipelined.yml b/cwf/gateway/integ_tests/pipelined.yml index bd9d094fbc57..9d9daf233f8c 100644 --- a/cwf/gateway/integ_tests/pipelined.yml +++ b/cwf/gateway/integ_tests/pipelined.yml @@ -126,7 +126,7 @@ monitored_ifaces: ['cwag_br0', ] # Whether pipelined should cleanup flows on restarts -clean_restart: true +clean_restart: false # Information for cwf check quota servers quota_check_ip: '192.0.0.1' @@ -138,7 +138,7 @@ li_local_iface: li_port li_mirror_all: false #li_dst_iface: eth4 -redis_enabled: false +redis_enabled: true # Logs grpc payload content magma_print_grpc_payload: true diff --git a/cwf/gateway/integ_tests/rule_manager.go b/cwf/gateway/integ_tests/rule_manager.go index ef168c330250..e6e9edd3a442 100644 --- a/cwf/gateway/integ_tests/rule_manager.go +++ b/cwf/gateway/integ_tests/rule_manager.go @@ -86,7 +86,8 @@ func (manager *RuleManager) AddStaticPassAllToDBAndPCRFforIMSIs( // AddStaticPassAllToDB adds a static rule that passes all traffic to policyDB // storage func (manager *RuleManager) AddStaticPassAllToDB(ruleID string, monitoringKey string, ratingGroup uint32, trackingType string, priority uint32) error { - fmt.Printf("************************* Adding a Pass-All static rule: %s\n", ruleID) + fmt.Printf("************************* Adding a Pass-All static rule: %s, mkey: %s, rg: %d, trackingType: %s\n", + ruleID, monitoringKey, ratingGroup, trackingType) staticPassAll := getStaticPassAll(ruleID, monitoringKey, ratingGroup, trackingType, priority, nil) return manager.insertStaticRuleIntoRedis(staticPassAll) } diff --git a/cwf/gateway/integ_tests/test_runner.go b/cwf/gateway/integ_tests/test_runner.go index 9f9fcebd0214..8952904f7176 100644 --- a/cwf/gateway/integ_tests/test_runner.go +++ b/cwf/gateway/integ_tests/test_runner.go @@ -299,6 +299,74 @@ func (tr *TestRunner) WaitForPoliciesToSync() { time.Sleep(4 * ruleUpdatePeriod) } +func (tr *TestRunner) WaitForEnforcementStatsForRule(imsi string, ruleIDs ...string) func() bool { + // Wait until the ruleIDs show up for the IMSI + return func() bool { + fmt.Printf("Waiting until %s, %v shows up in enforcement stats...\n", imsi, ruleIDs) + records, err := tr.GetPolicyUsage() + if err != nil { + return false + } + if records[prependIMSIPrefix(imsi)] == nil { + return false + } + for _, ruleID := range ruleIDs { + if records[prependIMSIPrefix(imsi)][ruleID] == nil { + return false + } + } + fmt.Printf("%s, %v are now in enforcement stats!\n", imsi, ruleIDs) + return true + } +} + +func (tr *TestRunner) WaitForNoEnforcementStatsForRule(imsi string, ruleIDs ...string) func() bool { + // Wait until the ruleIDs disappear for the IMSI + return func() bool { + fmt.Printf("Waiting until %s, %v disappear from enforcement stats...\n", imsi, ruleIDs) + records, err := tr.GetPolicyUsage() + if err != nil { + return false + } + if records[prependIMSIPrefix(imsi)] == nil { + fmt.Printf("%s are no longer in enforcement stats!\n", imsi) + return true + } + for _, ruleID := range ruleIDs { + if records[prependIMSIPrefix(imsi)][ruleID] != nil { + return false + } + } + fmt.Printf("%s, %v are no longer in enforcement stats!\n", imsi, ruleIDs) + return true + } +} + +func (tr *TestRunner) WaitForEnforcementStatsForRuleGreaterThan(imsi, ruleID string, min uint64) func() bool { + // Todo figure out the best way to figure out when RAR is processed + return func() bool { + fmt.Printf("Waiting until %s, %s has more than %d bytes in enforcement stats...\n", imsi, ruleID, min) + records, err := tr.GetPolicyUsage() + imsi = prependIMSIPrefix(imsi) + if err != nil { + return false + } + if records[imsi] == nil { + return false + } + record := records[imsi][ruleID] + if record == nil { + return false + } + txBytes := record.BytesTx + if record.BytesTx <= min { + return false + } + fmt.Printf("%s, %s now passed %d > %d in enforcement stats!\n", imsi, ruleID, txBytes, min) + return true + } +} + //WaitForPolicyReAuthToProcess returns a method which checks for reauth answer and // if it has sessionID which contains the IMSI func (tr *TestRunner) WaitForPolicyReAuthToProcess(raa *fegprotos.PolicyReAuthAnswer, imsi string) func() bool { diff --git a/docs/docusaurus/sidebars.json b/docs/docusaurus/sidebars.json index 9ad4bbd207d8..ce89d3847188 100644 --- a/docs/docusaurus/sidebars.json +++ b/docs/docusaurus/sidebars.json @@ -62,7 +62,7 @@ "ids": ["lte/tr069", "lte/s1ap_tests", "lte/dev_notes"] } ], - "Feature Guides": ["howtos/ue_metering", "howtos/config_agw_bridged"], + "Feature Guides": ["howtos/ue_metering", "howtos/config_agw_bridged", "howtos/thanos"], "Federation Gateway": [ { "type": "subcategory", diff --git a/docs/docusaurus/siteConfig.js b/docs/docusaurus/siteConfig.js index fe1e9161cbb7..bd74bd3f2090 100644 --- a/docs/docusaurus/siteConfig.js +++ b/docs/docusaurus/siteConfig.js @@ -26,16 +26,19 @@ const users = [ }, ]; +const url = process.env.DOCUSAURUS_URL || 'https://magmacore.org' +const baseUrl = process.env.DOCUSAURUS_BASE_URL || '/' + const siteConfig = { title: 'Magma Documentation', // Title for your website. - disableTitleTagline: true, + disableTitleTagline: true, tagline: 'Bring more people online by enabling operators with open, flexible, and extensible network solutions', // Used for publishing and more projectName: 'magma', - organizationName: 'magma', - url: 'https://magmacore.org', // Your website URL - baseUrl: '/', // Base URL for your project */ + organizationName: 'magma', + url: url, // Your website URL + baseUrl: baseUrl, // Base URL for your project */ // For github.io type URLs, you would set the url and baseUrl like: // url: 'https://facebook.github.io', // baseUrl: '/test-site/', diff --git a/docs/readmes/faq/magma_faq.md b/docs/readmes/faq/magma_faq.md index b07374aef810..dcbdfee67363 100644 --- a/docs/readmes/faq/magma_faq.md +++ b/docs/readmes/faq/magma_faq.md @@ -83,6 +83,11 @@ This section lists some of the commonly asked questions related to Magma operati - Then click on API trigger action button e.g. **GET**, **PUT**, **DELETE** etc. - Click on **Try it out** button on right hand side. - Put in the required inputs and click **Execute**. + +### How can I check the services running in Orchestrator? + - List the running pods with `kubectl -norc8r get pods` + - Grab the name of orc8r-controller pods, they are in the format `orc8r-controller-xxxxxxxxxx-yyyyy` + - Collect the state of services on each pod: `kubectl -norc8r exec orc8r-controller-xxxxxxxxxx-yyyyy bash -- supervisorctl status` ## NMS ### What is an NMS (Network Management System)? diff --git a/docs/readmes/howtos/thanos.md b/docs/readmes/howtos/thanos.md new file mode 100644 index 000000000000..26d06c362480 --- /dev/null +++ b/docs/readmes/howtos/thanos.md @@ -0,0 +1,97 @@ +--- +id: thanos +title: Thanos +hide_title: true +--- + +# Scaled Metrics with Thanos + +[Thanos](https://thanos.io/) is an open-sourced scaled prometheus solution used +to provide features like global query view, unlimited retention, and downsampling +and compaction. Orchestrator provides large deployments the option to deploy +with Thanos in order to have a more robust metrics pipeline. + +## Deploying Orc8r with Thanos + +The terraform module makes deploying Thanos very easy. In your `main.tf` you +just need to set the following values: + +``` +module orc8r { + thanos_enabled = true +} + +module orc8r-app { + thanos_enabled = true + thanos_object_store_bucket_name = "" +} +``` + +Choose a value for `thanos_object_store_bucket_name` that will be globally +(across all of AWS) unique, but other than that the value doesn't matter. + +That's all you need to do to deploy with Thanos! All interacting components +will be adjusted accordingly, so the NMS/Grafana will work the same as before. +If you don't care about the internals you can stop reading here. + +If you run `kubectl -n orc8r get pods` it should now look like this: +``` +NAME READY STATUS RESTARTS AGE +fluentd-6fb9f57dff-ljmfw 1/1 Running 0 24h +fluentd-6fb9f57dff-p54p9 1/1 Running 0 24h +nms-magmalte-f4bbf4cfb-tqblm 1/1 Running 0 24h +nms-nginx-proxy-57b8585d6-4ml6s 1/1 Running 0 24h +orc8r-alertmanager-84d79f774b-4svrs 1/1 Running 0 24h +orc8r-alertmanager-configurer-68d6c55c9c-6q9xg 1/1 Running 0 24h +orc8r-controller-7494c96646-4w4jp 1/1 Running 0 20h +orc8r-controller-7494c96646-7fcg5 1/1 Running 0 20h +orc8r-nginx-5f9d7f4bcc-cz5ld 1/1 Running 0 20h +orc8r-nginx-5f9d7f4bcc-rszpr 1/1 Running 0 20h +orc8r-prometheus-5bdd644fd8-mm8gb 2/2 Running 0 24h +orc8r-prometheus-cache-f84884575-7vw8d 1/1 Running 0 24h +orc8r-prometheus-configurer-69df67988-w9dc6 2/2 Running 0 20h +orc8r-thanos-compact-66dd4d974b-jwzjk 1/1 Running 0 20h +orc8r-thanos-query-5d5cb888bd-vm9t8 1/1 Running 0 114m +orc8r-thanos-store-0-7479bf59f6-97wbp 1/1 Running 0 114m +orc8r-user-grafana-bc644b4fc-28nmf 1/1 Running 0 24h +``` + +Notice that the prometheus pod now has another container running, this is the +[thanos sidecar](https://thanos.io/v0.17/components/sidecar.md/). There is another +sidecar that runs with `prometheus-configurer`, and then three more components +that run independently: compact, query, and store. + +## Advanced configuration options + +The default infrastructure setup deploys an additional node for Thanos, since +there is one component that requires significant on-node ephemeral storage. +However, you may want to deploy more nodes if you want to make sure thanos +components run on different nodes than the rest of orc8r. To do that you can +override the default value for `thanos_worker_groups` in the `orc8r` module. +The default value is: +``` +[ + { + name = "thanos-1" + instance_type = "m5d.xlarge" + asg_desired_capacity = 1 + asg_min_size = 1 + asg_max_size = 1 + autoscaling_enabled = false + kubelet_extra_args = "--node-labels=compute-type=thanos" + }, + ] +``` + +To add more workers, either adjust the `asg_...` values in that object, or add +another entry to that array of worker groups. To specify thanos components to +run on specific nodes, just set the following variables in the `orc8r-app` module: +``` +thanos_query_node_selector = "thanos" +thanos_store_node_selector = "thanos" +``` +> Note: set the value to the same value you used for `--node-labels=compute-type=` +> in order to run on that worker group + +These are advanced configuration options, and we don't expect them to be necessary, +but are available to give more fine-grained control over your deployment. diff --git a/feg/cloud/configs/service_registry.yml b/feg/cloud/configs/service_registry.yml index 110b06cade5a..10c846e63046 100644 --- a/feg/cloud/configs/service_registry.yml +++ b/feg/cloud/configs/service_registry.yml @@ -30,7 +30,7 @@ services: proxy_type: "clientcert" proxy_aliases: s6a_proxy: - port: 9079 + port: 9103 session_proxy: port: 9079 swx_proxy: diff --git a/feg/gateway/configs/ctraced.yml b/feg/gateway/configs/ctraced.yml index e4ef28aba475..79d383fbd944 100644 --- a/feg/gateway/configs/ctraced.yml +++ b/feg/gateway/configs/ctraced.yml @@ -11,3 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. log_level: INFO + +# Directory for temporary storage of packet captures +trace_directory: /var/opt/magma/tmp/trace + +# Interface to capture on +trace_interfaces: + - eth0 + +# Options available: +# - tshark +# - tcpdump +# tshark has more capabilities - see command_builder.py +trace_tool: tshark diff --git a/feg/gateway/gtp/gtp_client.go b/feg/gateway/gtp/gtp_client.go index fd2e2301c1c1..ab2997b24db0 100644 --- a/feg/gateway/gtp/gtp_client.go +++ b/feg/gateway/gtp/gtp_client.go @@ -11,6 +11,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +// gtp module wraps gp-gtp v2 client providing the client with some custom functions +// to ease its instantiation + package gtp import ( @@ -33,17 +36,18 @@ type Client struct { remoteAddr *net.UDPAddr } -// NewConnectedAutoClient creates a GTP-C client finding out automatically the local Address that needs to -// be used to reach the remote IP. It checks remote address using echo. And It also runs the GTP-C -// server waiting for incoming calls. -func NewConnectedAutoClient(ctx context.Context, remoteAddrStr string, connType uint8) (*Client, error) { - remoteAddr, err := net.ResolveUDPAddr("udp", remoteAddrStr) +// NewConnectedAutoClient creates a GTP client finding out automatically the local IP Address to +// be used to reach the remote IP. +// It checks if remote end is alive using echo. +// It runs the GTP-C server to serve incoming calls and responses. +func NewConnectedAutoClient(ctx context.Context, remoteIPAndPortStr string, connType uint8) (*Client, error) { + remoteAddr, err := net.ResolveUDPAddr("udp", remoteIPAndPortStr) if err != nil { - return nil, fmt.Errorf("could not resolve remote address: %s", err) + return nil, fmt.Errorf("could not resolve remote address %s: %s", remoteIPAndPortStr, err) } localAddrIp, err := GetOutboundIP(remoteAddr) if err != nil { - return nil, fmt.Errorf("Could not find automatically local address: %s", err) + return nil, fmt.Errorf("could not find local address automatically: %s", err) } localAddr := &net.UDPAddr{IP: localAddrIp, Port: GTPC_PORT, Zone: ""} @@ -57,7 +61,7 @@ func NewConnectedClient(ctx context.Context, localAddr, remoteAddr *net.UDPAddr, c := NewClient(localAddr, remoteAddr, connType) c.Conn, err = gtpv2.Dial(ctx, localAddr, remoteAddr, connType, 0) if err != nil { - return nil, fmt.Errorf("Could not connect to GTP-C %s server: %s", remoteAddr.String(), err) + return nil, fmt.Errorf("could not connect to GTP-C %s server: %s", remoteAddr.String(), err) } return c, nil } @@ -111,10 +115,9 @@ func (c *Client) GetLocalAddress() *net.UDPAddr { // Get preferred outbound ip of this machine func GetOutboundIP(testIp *net.UDPAddr) (net.IP, error) { - //Conn, err := net.Dial("udp", "8.8.8.8:80") connection, err := net.Dial("udp", testIp.String()) if err != nil { - return nil, fmt.Errorf("Couldn't local io running in this server: %s", err) + return nil, err } defer connection.Close() localAddr := connection.LocalAddr().(*net.UDPAddr) diff --git a/feg/radius/lib/go/oc/go.mod b/feg/radius/lib/go/oc/go.mod index ce6070b3d724..594f8fdd8714 100644 --- a/feg/radius/lib/go/oc/go.mod +++ b/feg/radius/lib/go/oc/go.mod @@ -15,7 +15,6 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.1.0 fbc/lib/go/http v0.0.0 fbc/lib/go/oc/helpers v0.0.0 - github.com/gogo/protobuf v1.2.0 // indirect github.com/jessevdk/go-flags v1.4.1-0.20181221193153-c0795c8afcf4 github.com/kelseyhightower/envconfig v1.3.0 github.com/pkg/errors v0.8.1 diff --git a/lte/cloud/configs/service_registry.yml b/lte/cloud/configs/service_registry.yml index 3cb5980b378f..88f99e0e60fb 100644 --- a/lte/cloud/configs/service_registry.yml +++ b/lte/cloud/configs/service_registry.yml @@ -10,6 +10,11 @@ # See the License for the specific language governing permissions and # limitations under the License. services: + ha: + host: "localhost" + port: 9119 + proxy_type: "clientcert" + lte: host: "localhost" port: 9113 diff --git a/lte/cloud/go/go.mod b/lte/cloud/go/go.mod index 31e3278dc237..ede88ce983f7 100644 --- a/lte/cloud/go/go.mod +++ b/lte/cloud/go/go.mod @@ -32,6 +32,7 @@ require ( github.com/lib/pq v1.2.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.5.1 + github.com/prometheus/common v0.9.1 github.com/stretchr/testify v1.5.1 github.com/thoas/go-funk v0.7.0 github.com/warthog618/sms v0.3.0 diff --git a/lte/cloud/go/go.sum b/lte/cloud/go/go.sum index 97319ec8b162..e0a09947cd7c 100644 --- a/lte/cloud/go/go.sum +++ b/lte/cloud/go/go.sum @@ -29,6 +29,7 @@ github.com/aeden/traceroute v0.0.0-20181124220833-147686d9cb0f/go.mod h1:WwE/rUG github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= @@ -240,6 +241,7 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -476,9 +478,11 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= +github.com/prometheus/prometheus v0.0.0-20190607092147-e23fa22233cf h1:FXtC3S+q2e1o8wS2eOASih8ijeJDv2EZ+3dPuJQIrcY= github.com/prometheus/prometheus v0.0.0-20190607092147-e23fa22233cf/go.mod h1:oYrT4Vs22/NcnoVYXt5m4cIHP+znvgyusahVpyETKTw= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.8.0/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= +github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rlmcpherson/s3gof3r v0.5.0/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs= @@ -626,6 +630,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/lte/cloud/go/protos/had_orc8r.pb.go b/lte/cloud/go/protos/ha_orc8r.pb.go similarity index 65% rename from lte/cloud/go/protos/had_orc8r.pb.go rename to lte/cloud/go/protos/ha_orc8r.pb.go index 104b9251430d..567824e98bea 100644 --- a/lte/cloud/go/protos/had_orc8r.pb.go +++ b/lte/cloud/go/protos/ha_orc8r.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: lte/protos/had_orc8r.proto +// source: lte/protos/ha_orc8r.proto package protos @@ -49,7 +49,7 @@ func (x GetEnodebOffloadStateResponse_EnodebOffloadState) String() string { } func (GetEnodebOffloadStateResponse_EnodebOffloadState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_b4cc90157bd02bee, []int{1, 0} + return fileDescriptor_b1d9bca7830728b9, []int{1, 0} } type GetEnodebOffloadStateRequest struct { @@ -62,7 +62,7 @@ func (m *GetEnodebOffloadStateRequest) Reset() { *m = GetEnodebOffloadSt func (m *GetEnodebOffloadStateRequest) String() string { return proto.CompactTextString(m) } func (*GetEnodebOffloadStateRequest) ProtoMessage() {} func (*GetEnodebOffloadStateRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_b4cc90157bd02bee, []int{0} + return fileDescriptor_b1d9bca7830728b9, []int{0} } func (m *GetEnodebOffloadStateRequest) XXX_Unmarshal(b []byte) error { @@ -95,7 +95,7 @@ func (m *GetEnodebOffloadStateResponse) Reset() { *m = GetEnodebOffloadS func (m *GetEnodebOffloadStateResponse) String() string { return proto.CompactTextString(m) } func (*GetEnodebOffloadStateResponse) ProtoMessage() {} func (*GetEnodebOffloadStateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_b4cc90157bd02bee, []int{1} + return fileDescriptor_b1d9bca7830728b9, []int{1} } func (m *GetEnodebOffloadStateResponse) XXX_Unmarshal(b []byte) error { @@ -130,30 +130,30 @@ func init() { proto.RegisterMapType((map[uint32]GetEnodebOffloadStateResponse_EnodebOffloadState)(nil), "magma.lte.GetEnodebOffloadStateResponse.EnodebOffloadStatesEntry") } -func init() { proto.RegisterFile("lte/protos/had_orc8r.proto", fileDescriptor_b4cc90157bd02bee) } - -var fileDescriptor_b4cc90157bd02bee = []byte{ - // 315 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0xca, 0x29, 0x49, 0xd5, - 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x2f, 0xd6, 0xcf, 0x48, 0x4c, 0x89, 0xcf, 0x2f, 0x4a, 0xb6, 0x28, - 0xd2, 0x03, 0x0b, 0x08, 0x71, 0xe6, 0x26, 0xa6, 0xe7, 0x26, 0xea, 0xe5, 0x94, 0xa4, 0x2a, 0xc9, - 0x71, 0xc9, 0xb8, 0xa7, 0x96, 0xb8, 0xe6, 0xe5, 0xa7, 0xa4, 0x26, 0xf9, 0xa7, 0xa5, 0xe5, 0xe4, - 0x27, 0xa6, 0x04, 0x97, 0x24, 0x96, 0xa4, 0x06, 0xa5, 0x16, 0x96, 0xa6, 0x16, 0x97, 0x28, 0xfd, - 0x66, 0xe2, 0x92, 0xc5, 0xa1, 0xa0, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xa8, 0x94, 0x4b, 0x34, - 0x15, 0x2c, 0x1b, 0x9f, 0x0f, 0x91, 0x8e, 0x2f, 0x06, 0xc9, 0x17, 0x4b, 0x30, 0x2a, 0x30, 0x6b, - 0x70, 0x1b, 0x39, 0xea, 0xc1, 0x2d, 0xd3, 0xc3, 0x6b, 0x90, 0x1e, 0xa6, 0x54, 0xb1, 0x6b, 0x5e, - 0x49, 0x51, 0x65, 0x90, 0x70, 0x2a, 0xa6, 0x8c, 0x54, 0x33, 0x23, 0x97, 0x04, 0x2e, 0x1d, 0x42, - 0x02, 0x5c, 0xcc, 0xd9, 0xa9, 0x95, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0xbc, 0x41, 0x20, 0xa6, 0x50, - 0x20, 0x17, 0x6b, 0x59, 0x62, 0x4e, 0x69, 0xaa, 0x04, 0x93, 0x02, 0xa3, 0x06, 0x9f, 0x91, 0x35, - 0x05, 0xae, 0x0a, 0x82, 0x98, 0x64, 0xc5, 0x64, 0xc1, 0xa8, 0x14, 0xcb, 0x25, 0x84, 0xa9, 0x40, - 0x88, 0x93, 0x8b, 0xd5, 0xcf, 0x3f, 0xde, 0x3f, 0x40, 0x80, 0x41, 0x48, 0x94, 0x4b, 0x30, 0x20, - 0xc8, 0xd3, 0xd7, 0x31, 0x28, 0x32, 0xde, 0xd9, 0xdf, 0xcf, 0xcf, 0xd5, 0x39, 0xc4, 0xd5, 0x45, - 0x80, 0x51, 0x48, 0x95, 0x4b, 0x11, 0x43, 0x38, 0xde, 0xd1, 0xcf, 0x25, 0x3e, 0xd8, 0x35, 0x28, - 0xcc, 0xd3, 0xcf, 0x3d, 0x3e, 0xd4, 0x35, 0x58, 0x80, 0xc9, 0xa8, 0x98, 0x8b, 0xd9, 0x23, 0xd1, - 0x45, 0x28, 0x87, 0x4b, 0x14, 0xab, 0x23, 0x85, 0xd4, 0x09, 0x7b, 0x03, 0x1c, 0x8d, 0x52, 0x1a, - 0xc4, 0xfa, 0x57, 0x89, 0xc1, 0x49, 0x3a, 0x4a, 0x12, 0xac, 0x58, 0x1f, 0x94, 0x82, 0x92, 0x73, - 0xf2, 0x4b, 0x53, 0xf4, 0xd3, 0xf3, 0xa1, 0x49, 0x29, 0x89, 0x0d, 0x4c, 0x1b, 0x03, 0x02, 0x00, - 0x00, 0xff, 0xff, 0xfe, 0x0f, 0x14, 0x7e, 0x5f, 0x02, 0x00, 0x00, +func init() { proto.RegisterFile("lte/protos/ha_orc8r.proto", fileDescriptor_b1d9bca7830728b9) } + +var fileDescriptor_b1d9bca7830728b9 = []byte{ + // 312 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x92, 0xcd, 0x4b, 0xc3, 0x40, + 0x10, 0xc5, 0xbb, 0x29, 0x15, 0x3a, 0xa2, 0xc4, 0x95, 0x42, 0x5a, 0x3f, 0xa8, 0x01, 0x31, 0xa7, + 0x04, 0xea, 0xa5, 0xe8, 0xa9, 0xb6, 0xa1, 0xf6, 0x60, 0x52, 0xb7, 0x2a, 0x28, 0xc8, 0xb2, 0x6d, + 0xa7, 0x15, 0x4c, 0xbb, 0x35, 0xbb, 0x11, 0x7a, 0xf6, 0xcf, 0xf6, 0x22, 0x49, 0xc5, 0x4b, 0xac, + 0x0a, 0x9e, 0xf6, 0xe3, 0xf7, 0x76, 0x76, 0x1e, 0xf3, 0xa0, 0x1a, 0x69, 0xf4, 0x16, 0xb1, 0xd4, + 0x52, 0x79, 0x4f, 0x82, 0xcb, 0x78, 0xd4, 0x8c, 0xdd, 0xec, 0x4c, 0xcb, 0x33, 0x31, 0x9d, 0x09, + 0x37, 0xd2, 0x68, 0x1f, 0xc2, 0x7e, 0x17, 0xb5, 0x3f, 0x97, 0x63, 0x1c, 0x86, 0x93, 0x49, 0x24, + 0xc5, 0x78, 0xa0, 0x85, 0x46, 0x86, 0x2f, 0x09, 0x2a, 0x6d, 0xbf, 0x1b, 0x70, 0xb0, 0x46, 0xa0, + 0x16, 0x72, 0xae, 0x90, 0x26, 0x50, 0xc1, 0x8c, 0x72, 0xb9, 0xc2, 0x5c, 0xa5, 0x5c, 0x59, 0xa4, + 0x5e, 0x74, 0x36, 0x1b, 0x2d, 0xf7, 0xeb, 0x33, 0xf7, 0xc7, 0x42, 0x6e, 0x1e, 0x29, 0x7f, 0xae, + 0xe3, 0x25, 0xdb, 0xc5, 0x3c, 0xa9, 0xbd, 0x11, 0xb0, 0xd6, 0xbd, 0xa0, 0x26, 0x14, 0x9f, 0x71, + 0x69, 0x91, 0x3a, 0x71, 0xb6, 0x58, 0xba, 0xa5, 0xd7, 0x50, 0x7a, 0x15, 0x51, 0x82, 0x96, 0x51, + 0x27, 0xce, 0x76, 0xe3, 0xfc, 0x1f, 0x5d, 0xb1, 0x55, 0xa5, 0x33, 0xa3, 0x49, 0xec, 0x47, 0xa0, + 0x79, 0x01, 0x2d, 0x43, 0x29, 0x08, 0x79, 0xd8, 0x37, 0x0b, 0xb4, 0x02, 0x3b, 0x7d, 0xd6, 0xbb, + 0x6a, 0xb1, 0x7b, 0xde, 0x0e, 0x83, 0xc0, 0x6f, 0xdf, 0xf8, 0x1d, 0x93, 0xd0, 0x63, 0x38, 0xca, + 0x5d, 0xf3, 0x56, 0xd0, 0xe1, 0x03, 0x9f, 0xdd, 0xf5, 0x82, 0x2e, 0xbf, 0xf5, 0x07, 0xa6, 0xd1, + 0x88, 0xc1, 0xb8, 0x14, 0x34, 0x82, 0xca, 0xb7, 0x3d, 0xd2, 0x93, 0xdf, 0x5d, 0x64, 0x53, 0xac, + 0x39, 0x7f, 0xb5, 0x6b, 0x17, 0x2e, 0xf6, 0x1e, 0xaa, 0x99, 0xd8, 0x4b, 0xf3, 0x33, 0x8a, 0x64, + 0x32, 0xf6, 0xa6, 0xf2, 0x33, 0x48, 0xc3, 0x8d, 0x6c, 0x3d, 0xfd, 0x08, 0x00, 0x00, 0xff, 0xff, + 0x2d, 0x32, 0xa2, 0x92, 0x5d, 0x02, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -164,76 +164,76 @@ var _ grpc.ClientConnInterface // is compatible with the grpc package it is being compiled against. const _ = grpc.SupportPackageIsVersion6 -// HaDClient is the client API for HaD service. +// HaClient is the client API for Ha service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type HaDClient interface { +type HaClient interface { // Fetch all ENB state for ENBs served within the pools of the calling AGW GetEnodebOffloadState(ctx context.Context, in *GetEnodebOffloadStateRequest, opts ...grpc.CallOption) (*GetEnodebOffloadStateResponse, error) } -type haDClient struct { +type haClient struct { cc grpc.ClientConnInterface } -func NewHaDClient(cc grpc.ClientConnInterface) HaDClient { - return &haDClient{cc} +func NewHaClient(cc grpc.ClientConnInterface) HaClient { + return &haClient{cc} } -func (c *haDClient) GetEnodebOffloadState(ctx context.Context, in *GetEnodebOffloadStateRequest, opts ...grpc.CallOption) (*GetEnodebOffloadStateResponse, error) { +func (c *haClient) GetEnodebOffloadState(ctx context.Context, in *GetEnodebOffloadStateRequest, opts ...grpc.CallOption) (*GetEnodebOffloadStateResponse, error) { out := new(GetEnodebOffloadStateResponse) - err := c.cc.Invoke(ctx, "/magma.lte.HaD/GetEnodebOffloadState", in, out, opts...) + err := c.cc.Invoke(ctx, "/magma.lte.Ha/GetEnodebOffloadState", in, out, opts...) if err != nil { return nil, err } return out, nil } -// HaDServer is the server API for HaD service. -type HaDServer interface { +// HaServer is the server API for Ha service. +type HaServer interface { // Fetch all ENB state for ENBs served within the pools of the calling AGW GetEnodebOffloadState(context.Context, *GetEnodebOffloadStateRequest) (*GetEnodebOffloadStateResponse, error) } -// UnimplementedHaDServer can be embedded to have forward compatible implementations. -type UnimplementedHaDServer struct { +// UnimplementedHaServer can be embedded to have forward compatible implementations. +type UnimplementedHaServer struct { } -func (*UnimplementedHaDServer) GetEnodebOffloadState(ctx context.Context, req *GetEnodebOffloadStateRequest) (*GetEnodebOffloadStateResponse, error) { +func (*UnimplementedHaServer) GetEnodebOffloadState(ctx context.Context, req *GetEnodebOffloadStateRequest) (*GetEnodebOffloadStateResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetEnodebOffloadState not implemented") } -func RegisterHaDServer(s *grpc.Server, srv HaDServer) { - s.RegisterService(&_HaD_serviceDesc, srv) +func RegisterHaServer(s *grpc.Server, srv HaServer) { + s.RegisterService(&_Ha_serviceDesc, srv) } -func _HaD_GetEnodebOffloadState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _Ha_GetEnodebOffloadState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetEnodebOffloadStateRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(HaDServer).GetEnodebOffloadState(ctx, in) + return srv.(HaServer).GetEnodebOffloadState(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/magma.lte.HaD/GetEnodebOffloadState", + FullMethod: "/magma.lte.Ha/GetEnodebOffloadState", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(HaDServer).GetEnodebOffloadState(ctx, req.(*GetEnodebOffloadStateRequest)) + return srv.(HaServer).GetEnodebOffloadState(ctx, req.(*GetEnodebOffloadStateRequest)) } return interceptor(ctx, in, info, handler) } -var _HaD_serviceDesc = grpc.ServiceDesc{ - ServiceName: "magma.lte.HaD", - HandlerType: (*HaDServer)(nil), +var _Ha_serviceDesc = grpc.ServiceDesc{ + ServiceName: "magma.lte.Ha", + HandlerType: (*HaServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "GetEnodebOffloadState", - Handler: _HaD_GetEnodebOffloadState_Handler, + Handler: _Ha_GetEnodebOffloadState_Handler, }, }, Streams: []grpc.StreamDesc{}, - Metadata: "lte/protos/had_orc8r.proto", + Metadata: "lte/protos/ha_orc8r.proto", } diff --git a/lte/cloud/go/protos/mconfig/mconfigs.pb.go b/lte/cloud/go/protos/mconfig/mconfigs.pb.go index dfdc4df060e4..6026221115c5 100644 --- a/lte/cloud/go/protos/mconfig/mconfigs.pb.go +++ b/lte/cloud/go/protos/mconfig/mconfigs.pb.go @@ -1346,6 +1346,8 @@ type MME struct { // Overrides field 4 if this is not empty. Field 4 is in the process of // being deprecated AttachedEnodebTacs []int32 `protobuf:"varint,15,rep,packed,name=attached_enodeb_tacs,json=attachedEnodebTacs,proto3" json:"attached_enodeb_tacs,omitempty"` + // MME relative capacity - capacity within an MME group. 8-bit + MmeRelativeCapacity int32 `protobuf:"varint,16,opt,name=mme_relative_capacity,json=mmeRelativeCapacity,proto3" json:"mme_relative_capacity,omitempty"` // Primary DNS server DnsPrimary string `protobuf:"bytes,20,opt,name=dns_primary,json=dnsPrimary,proto3" json:"dns_primary,omitempty"` // Secondary DNS server @@ -1490,6 +1492,13 @@ func (m *MME) GetAttachedEnodebTacs() []int32 { return nil } +func (m *MME) GetMmeRelativeCapacity() int32 { + if m != nil { + return m.MmeRelativeCapacity + } + return 0 +} + func (m *MME) GetDnsPrimary() string { if m != nil { return m.DnsPrimary @@ -2054,168 +2063,170 @@ func init() { func init() { proto.RegisterFile("lte/protos/mconfig/mconfigs.proto", fileDescriptor_cb46bfd77f2ecf71) } var fileDescriptor_cb46bfd77f2ecf71 = []byte{ - // 2607 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x59, 0xdd, 0x72, 0x1b, 0xc7, - 0xb1, 0x16, 0x08, 0x90, 0x04, 0x1a, 0x20, 0xb9, 0x1a, 0x52, 0xe2, 0x0a, 0xf2, 0x91, 0x29, 0xe8, - 0xc8, 0xa6, 0xff, 0x20, 0x1f, 0xd8, 0x52, 0xc9, 0x2e, 0x5b, 0x2e, 0x10, 0x58, 0x91, 0xf0, 0x21, - 0x48, 0xd4, 0x82, 0x76, 0x5c, 0xae, 0x4a, 0xa6, 0x06, 0xbb, 0x03, 0x60, 0xa2, 0xfd, 0xab, 0x9d, - 0x01, 0x49, 0xf8, 0x2e, 0x8f, 0x90, 0x54, 0xe5, 0x11, 0xf2, 0x10, 0x79, 0x88, 0xbc, 0x45, 0xee, - 0x73, 0x91, 0xe4, 0x2e, 0x17, 0xa9, 0x99, 0x9d, 0x5d, 0x02, 0x20, 0x28, 0xbb, 0x58, 0x95, 0xaa, - 0x5c, 0x61, 0xa7, 0xfb, 0xeb, 0x9e, 0x99, 0x9e, 0xee, 0x9e, 0xee, 0x01, 0x3c, 0xf6, 0x04, 0x7d, - 0x16, 0xc5, 0xa1, 0x08, 0xf9, 0x33, 0xdf, 0x09, 0x83, 0x21, 0x1b, 0xa5, 0xbf, 0xbc, 0xae, 0xe8, - 0x68, 0xc3, 0x27, 0x23, 0x9f, 0xd4, 0x35, 0xb5, 0xfa, 0x20, 0x8c, 0x9d, 0x97, 0x71, 0x2a, 0xe3, - 0x84, 0xbe, 0x1f, 0x06, 0x09, 0xb2, 0xf6, 0x4f, 0x80, 0x75, 0x2b, 0x08, 0x5d, 0x3a, 0x68, 0xa3, - 0x06, 0x94, 0xbc, 0x70, 0x84, 0x3d, 0x7a, 0x4e, 0x3d, 0x33, 0xb7, 0x97, 0xdb, 0xdf, 0x6c, 0xdc, - 0xab, 0x27, 0x9a, 0x94, 0x82, 0xfa, 0x71, 0x38, 0x3a, 0x96, 0x4c, 0xbb, 0xe8, 0xe9, 0x2f, 0x64, - 0x40, 0x3e, 0x72, 0x98, 0xb9, 0xb2, 0x97, 0xdb, 0x5f, 0xb5, 0xe5, 0x27, 0xaa, 0x42, 0x91, 0x92, - 0x78, 0xe8, 0x04, 0xae, 0x67, 0xe6, 0x15, 0x39, 0x1b, 0xa3, 0x27, 0xb0, 0x31, 0x20, 0x81, 0x7b, - 0xc1, 0x5c, 0x31, 0xc6, 0xfe, 0xf8, 0x27, 0xb3, 0xa0, 0x00, 0x95, 0x8c, 0xd8, 0x1d, 0xff, 0x84, - 0xde, 0x85, 0x72, 0xe4, 0xf9, 0x01, 0x73, 0xb1, 0xc7, 0xb8, 0x30, 0x57, 0xf7, 0x72, 0xfb, 0x25, - 0x1b, 0x12, 0xd2, 0x31, 0xe3, 0x02, 0x3d, 0x83, 0x6d, 0x3e, 0x19, 0x0c, 0x63, 0xe2, 0x53, 0x4c, - 0x38, 0x67, 0xa3, 0xc0, 0xa7, 0x81, 0x30, 0xd7, 0x94, 0x2e, 0x94, 0xb2, 0x9a, 0x19, 0x07, 0xbd, - 0x04, 0x93, 0x47, 0xd4, 0x61, 0xc4, 0xc3, 0x99, 0x60, 0x44, 0x84, 0xa0, 0x71, 0x60, 0xae, 0x2b, - 0xa9, 0xfb, 0x9a, 0xdf, 0xd7, 0xec, 0x5e, 0xc2, 0x45, 0x0d, 0xb8, 0x47, 0x3c, 0x2f, 0xbc, 0xc0, - 0x54, 0xd9, 0x08, 0x8b, 0x98, 0x04, 0xdc, 0x67, 0xc2, 0x2c, 0xee, 0xe5, 0xf6, 0x8b, 0xf6, 0xb6, - 0x62, 0x26, 0xf6, 0x3b, 0xd3, 0x2c, 0x69, 0x12, 0x41, 0x1c, 0xb3, 0x94, 0x98, 0x44, 0x10, 0x07, - 0x7d, 0x01, 0x45, 0x87, 0x0f, 0x07, 0x38, 0x26, 0xc2, 0x04, 0x65, 0xd7, 0x47, 0xf5, 0xb9, 0x13, - 0xaa, 0xeb, 0x23, 0xa8, 0xb7, 0xfa, 0xaf, 0x0f, 0x6c, 0x22, 0xec, 0x75, 0x89, 0xb7, 0x89, 0x40, - 0x0f, 0xa0, 0xa8, 0x8c, 0x87, 0x1b, 0x23, 0xb3, 0xbc, 0x97, 0xdf, 0x5f, 0xb5, 0xd7, 0xd5, 0xb8, - 0x31, 0x42, 0xdf, 0x00, 0x08, 0xd7, 0xc5, 0x89, 0x06, 0xb3, 0xb2, 0x97, 0xdb, 0x2f, 0x37, 0xf6, - 0x6e, 0xd0, 0x7b, 0xd6, 0x6e, 0xb7, 0x14, 0xc5, 0x2e, 0x09, 0xd7, 0x4d, 0x3e, 0xa5, 0x82, 0xe1, - 0x95, 0x82, 0x8d, 0xb7, 0x2a, 0x78, 0x7d, 0xa5, 0x60, 0x98, 0x29, 0x20, 0x70, 0x8f, 0x06, 0x03, - 0xad, 0x80, 0xe3, 0xc1, 0x14, 0x73, 0x1a, 0x33, 0xe2, 0x99, 0x9b, 0x7b, 0xf9, 0xfd, 0x72, 0xa3, - 0x7e, 0x83, 0x2e, 0x2b, 0x18, 0x24, 0x0a, 0xf8, 0xc1, 0xb4, 0xaf, 0x04, 0xac, 0x40, 0xc4, 0x53, - 0x1b, 0xd1, 0x6b, 0x8c, 0x6a, 0x0b, 0x4a, 0xd9, 0xd4, 0x73, 0xae, 0x95, 0x5b, 0x70, 0xad, 0x8c, - 0x37, 0xf1, 0xb4, 0x37, 0x66, 0xe3, 0xea, 0x1f, 0x72, 0x50, 0x3a, 0xfb, 0x45, 0x5a, 0x6e, 0x70, - 0xad, 0x95, 0x5b, 0xb9, 0x56, 0xfe, 0x6d, 0xae, 0x55, 0xfd, 0xeb, 0x0a, 0x54, 0x12, 0x8b, 0xfc, - 0x57, 0xad, 0x2b, 0x8d, 0xe8, 0xc2, 0x55, 0x44, 0x7f, 0x00, 0x46, 0xea, 0xf7, 0x98, 0x06, 0x64, - 0xe0, 0x51, 0x57, 0x45, 0x65, 0xd1, 0xde, 0x4a, 0xe9, 0x56, 0x42, 0x46, 0x8f, 0xa1, 0xe2, 0xd2, - 0x73, 0xe6, 0x50, 0xec, 0x78, 0x84, 0x73, 0x15, 0x93, 0x25, 0xbb, 0x9c, 0xd0, 0x5a, 0x92, 0x74, - 0x3d, 0x07, 0xac, 0x2f, 0xc9, 0x01, 0x3a, 0x86, 0x8a, 0x57, 0x31, 0xb4, 0x0b, 0xeb, 0x0e, 0xf5, - 0x3c, 0xcc, 0x5c, 0x1d, 0x59, 0x6b, 0x72, 0xd8, 0x71, 0xd1, 0xff, 0x00, 0xb0, 0x08, 0x13, 0xd7, - 0x8d, 0x29, 0xe7, 0x2a, 0xbc, 0x4a, 0x76, 0x89, 0x45, 0xcd, 0x84, 0x50, 0xfd, 0x2d, 0xec, 0xde, - 0xe0, 0x6f, 0x72, 0x92, 0x37, 0x74, 0xaa, 0x6c, 0x5d, 0xb2, 0xe5, 0x27, 0xfa, 0x02, 0x56, 0xcf, - 0x89, 0x37, 0xa1, 0xca, 0xb0, 0xe5, 0xc6, 0x93, 0x1b, 0x1d, 0xf8, 0xea, 0xd8, 0xec, 0x44, 0xe2, - 0xcb, 0x95, 0x97, 0xb9, 0xda, 0x07, 0xb0, 0xae, 0x03, 0x18, 0x6d, 0x02, 0xa8, 0xcf, 0xe6, 0x19, - 0x6e, 0x1c, 0x1a, 0x77, 0x66, 0xc7, 0x9f, 0x1d, 0x1a, 0xb9, 0xda, 0xef, 0x36, 0xa0, 0xd4, 0x63, - 0x11, 0xf5, 0x58, 0x40, 0x6f, 0x97, 0x79, 0x1f, 0x41, 0x79, 0x42, 0x31, 0x8b, 0xf0, 0xc0, 0x0b, - 0x9d, 0x37, 0x6a, 0xc5, 0x25, 0xbb, 0x34, 0xa1, 0x9d, 0xe8, 0x40, 0x12, 0x64, 0x1a, 0x0d, 0xc8, - 0xd5, 0x81, 0xe5, 0xd5, 0x81, 0x41, 0x40, 0xb2, 0xb3, 0x7a, 0x0f, 0xb6, 0x5c, 0x3a, 0x24, 0x13, - 0x4f, 0xe0, 0x78, 0xe2, 0x51, 0x69, 0xd9, 0xe4, 0xb8, 0x36, 0x34, 0xd9, 0x9e, 0x78, 0xb4, 0xe3, - 0xa2, 0x36, 0x14, 0x39, 0x8d, 0xe5, 0x01, 0x72, 0xb3, 0xb8, 0x97, 0xdf, 0xdf, 0x6c, 0xec, 0x2f, - 0xd8, 0x25, 0xdb, 0x48, 0xfd, 0x84, 0x8a, 0x8b, 0x30, 0x7e, 0xd3, 0xd7, 0x78, 0x3b, 0x93, 0x44, - 0x7d, 0xb8, 0xab, 0x92, 0x25, 0x75, 0xf1, 0x28, 0xa6, 0x38, 0xa2, 0x34, 0xe6, 0x66, 0x49, 0xe5, - 0x89, 0xf7, 0x6f, 0x54, 0xd7, 0x4c, 0x24, 0x0e, 0x63, 0xda, 0xa3, 0x34, 0xb6, 0xb7, 0xc8, 0xdc, - 0x98, 0xa3, 0x13, 0xd8, 0x62, 0x91, 0x1b, 0x63, 0x7a, 0x19, 0x85, 0xb1, 0xc0, 0x2e, 0x4f, 0xf2, - 0x6b, 0xb9, 0xf1, 0xde, 0x8d, 0x2a, 0x3b, 0xbd, 0xb6, 0x6d, 0x29, 0x78, 0x9b, 0x0b, 0x7b, 0x43, - 0x8a, 0x67, 0x43, 0xf4, 0x1c, 0xd6, 0x3c, 0x86, 0x27, 0x94, 0xeb, 0x74, 0xfa, 0xe8, 0x46, 0x35, - 0xc7, 0xec, 0x3b, 0xca, 0xed, 0x55, 0x4f, 0xfe, 0xa0, 0x2f, 0xe0, 0x01, 0x1f, 0x31, 0xec, 0x93, - 0x80, 0x8c, 0xa8, 0x0c, 0x3f, 0xcc, 0x86, 0xc4, 0xa1, 0xf8, 0xdc, 0x23, 0x81, 0xca, 0xab, 0x25, - 0xfb, 0x3e, 0x1f, 0xb1, 0x6e, 0xc6, 0xef, 0x48, 0xf6, 0xf7, 0x1e, 0x09, 0xd0, 0x2b, 0x78, 0x67, - 0xa9, 0xa8, 0x76, 0x69, 0x73, 0x53, 0x49, 0x9b, 0xd7, 0xa5, 0x3b, 0xca, 0xc3, 0xd1, 0x73, 0xd8, - 0x5d, 0x2a, 0x3f, 0xba, 0x30, 0xb7, 0x94, 0xe8, 0xce, 0x75, 0xd1, 0xc3, 0x0b, 0xf4, 0x0a, 0x4a, - 0x63, 0x9a, 0x66, 0xfe, 0xbb, 0x6a, 0xaf, 0x8f, 0x6f, 0xdc, 0xeb, 0x91, 0xa5, 0x5d, 0xbd, 0x38, - 0xa6, 0x3a, 0x57, 0xbd, 0x82, 0x07, 0x2e, 0xe3, 0xd2, 0x8f, 0xf0, 0x98, 0x12, 0x97, 0xc6, 0x98, - 0x06, 0x31, 0x73, 0xc6, 0x2a, 0x2b, 0x19, 0xd2, 0xd5, 0x0e, 0x56, 0xcc, 0x9c, 0xbd, 0xab, 0x41, - 0x47, 0x0a, 0x63, 0x65, 0x90, 0x6a, 0x03, 0x36, 0xe7, 0xcf, 0x16, 0x6d, 0xc2, 0x0a, 0x8b, 0x74, - 0x2c, 0xae, 0xb0, 0x28, 0x0d, 0x4e, 0xe9, 0xd6, 0x1b, 0x2a, 0x38, 0xab, 0x9f, 0xc1, 0xc6, 0xdc, - 0xe1, 0x5d, 0x13, 0x41, 0x50, 0x90, 0x2c, 0x2d, 0xa3, 0xbe, 0xab, 0xbf, 0x86, 0x55, 0x75, 0x54, - 0x68, 0x07, 0x56, 0x99, 0xcf, 0x19, 0x37, 0x73, 0x7b, 0xf9, 0xfd, 0x92, 0x9d, 0x0c, 0x90, 0x09, - 0xeb, 0xf2, 0xd7, 0x0d, 0xb8, 0xb9, 0xa2, 0xe8, 0xe9, 0x50, 0x2a, 0xf3, 0x89, 0xc3, 0xcd, 0xbc, - 0x22, 0xab, 0x6f, 0xb9, 0x26, 0x16, 0x71, 0xb3, 0xa0, 0x48, 0xf2, 0xb3, 0xfa, 0xb7, 0x02, 0x14, - 0x53, 0xf3, 0xc8, 0x9c, 0x9b, 0x44, 0xdb, 0x12, 0x9b, 0xe4, 0x54, 0xf8, 0xdd, 0x4f, 0xf8, 0x8b, - 0xe6, 0x40, 0x1f, 0xc1, 0x5d, 0x2d, 0x49, 0x03, 0x27, 0x9e, 0x46, 0x82, 0x85, 0x81, 0xda, 0x46, - 0xd1, 0x36, 0x12, 0x86, 0x95, 0xd1, 0xd1, 0x08, 0xb6, 0xaf, 0x50, 0x4d, 0x6f, 0x14, 0xc6, 0x4c, - 0x8c, 0x7d, 0x15, 0xe0, 0x9b, 0x8d, 0xe7, 0x3f, 0x7b, 0x8a, 0x75, 0xeb, 0xba, 0xb0, 0xbd, 0x4c, - 0x23, 0xb2, 0xa1, 0x32, 0x26, 0x7c, 0xfc, 0x7a, 0x12, 0x38, 0x6a, 0x41, 0x05, 0x35, 0x43, 0xfd, - 0xe7, 0x67, 0x38, 0x9a, 0x91, 0xb2, 0xe7, 0x74, 0x48, 0x9d, 0x34, 0x70, 0x42, 0x97, 0x05, 0xa3, - 0xb3, 0x69, 0x44, 0xd5, 0x3d, 0xf2, 0x8b, 0x74, 0x5a, 0x33, 0x52, 0xf6, 0x9c, 0x0e, 0xf4, 0x14, - 0x36, 0xaf, 0x96, 0x8f, 0xa5, 0xd7, 0xe8, 0x3c, 0x76, 0x45, 0xfd, 0x7f, 0x3a, 0xad, 0xf9, 0xb0, - 0xbd, 0x64, 0xeb, 0x68, 0x1d, 0xf2, 0x76, 0xeb, 0x73, 0xe3, 0x0e, 0xda, 0x85, 0xed, 0xa6, 0xd5, - 0x6f, 0x3c, 0x7f, 0x81, 0x5b, 0x07, 0x2d, 0x7c, 0xd4, 0x6d, 0xb6, 0x70, 0xb7, 0xfd, 0xdc, 0xc8, - 0xcd, 0x30, 0xac, 0xd6, 0xc1, 0x15, 0x63, 0x05, 0x3d, 0x84, 0xdd, 0xc3, 0x1f, 0x3b, 0xbd, 0x9e, - 0xd5, 0xc6, 0x33, 0x80, 0xfe, 0x51, 0xf3, 0xff, 0x8c, 0x7c, 0xed, 0x63, 0xa8, 0xcc, 0xda, 0x41, - 0xce, 0x23, 0xa5, 0xee, 0xc8, 0x8f, 0x23, 0xeb, 0x07, 0x23, 0x87, 0x00, 0xd6, 0xfa, 0x47, 0xcd, - 0xc6, 0xf3, 0x17, 0xc6, 0x4a, 0xed, 0x7d, 0x59, 0x0c, 0xcc, 0xec, 0x09, 0x60, 0xed, 0xa0, 0xd9, - 0xb7, 0x5e, 0xc8, 0x85, 0x95, 0x61, 0xfd, 0xc8, 0xfa, 0xa1, 0x71, 0xd0, 0x39, 0x31, 0x72, 0xb5, - 0xaf, 0x61, 0x6b, 0x21, 0xc9, 0x22, 0x03, 0x8a, 0x5d, 0xeb, 0xcc, 0xb2, 0x3b, 0x27, 0x87, 0xc6, - 0x9d, 0xea, 0x4a, 0x31, 0x27, 0xa7, 0x68, 0xf7, 0x3a, 0x46, 0x0e, 0x6d, 0x41, 0xd9, 0x3a, 0x79, - 0x7d, 0x6a, 0xb7, 0xac, 0xae, 0x75, 0x72, 0x66, 0xac, 0x7c, 0x5b, 0x28, 0x16, 0x8c, 0xd5, 0x6f, - 0x0b, 0xc5, 0xb2, 0x51, 0xa9, 0xfd, 0x3d, 0x07, 0xc5, 0x3e, 0xe5, 0x9c, 0x85, 0xc1, 0xed, 0xae, - 0xa0, 0x27, 0xb0, 0x11, 0x53, 0x8f, 0x4c, 0xb3, 0x4b, 0x26, 0x71, 0xd9, 0x8a, 0x22, 0xa6, 0xd7, - 0x0c, 0x06, 0xf3, 0x82, 0x78, 0x1e, 0x15, 0x98, 0x5e, 0x8e, 0xc9, 0x84, 0x0b, 0xec, 0x52, 0x41, - 0x13, 0x8f, 0xca, 0xab, 0xcc, 0xf3, 0x74, 0xe1, 0xf4, 0x7f, 0xa5, 0xe0, 0x56, 0x82, 0x6e, 0xa7, - 0x60, 0xfb, 0xfe, 0xc5, 0x52, 0x3a, 0xfa, 0x04, 0xb6, 0x47, 0x97, 0x78, 0x34, 0xc5, 0xf3, 0x6b, - 0x29, 0x24, 0xe1, 0x33, 0xba, 0x3c, 0x9c, 0xda, 0x33, 0xeb, 0xa9, 0xfd, 0x25, 0x07, 0xf7, 0x97, - 0xcf, 0x80, 0x3e, 0x85, 0x1d, 0x41, 0x63, 0x9f, 0x05, 0x44, 0x50, 0x1c, 0x06, 0xe9, 0x82, 0x75, - 0xf0, 0xa2, 0x8c, 0x77, 0x1a, 0x68, 0x51, 0xd4, 0x86, 0x35, 0x9f, 0x8a, 0x71, 0x98, 0x6c, 0x7d, - 0xb3, 0xf1, 0xf1, 0x2f, 0xda, 0x4a, 0xbd, 0xab, 0x64, 0x6c, 0x2d, 0x2b, 0x4b, 0x18, 0xc1, 0x7c, - 0x1a, 0x4e, 0x04, 0xf6, 0xb9, 0x32, 0xca, 0x86, 0x5d, 0xd2, 0x94, 0x2e, 0xaf, 0xbd, 0x03, 0x6b, - 0x89, 0x00, 0x42, 0xb0, 0x79, 0x78, 0x79, 0x16, 0x13, 0xe7, 0x0d, 0x75, 0xe5, 0xed, 0xcc, 0x8d, - 0x3b, 0xb5, 0x57, 0x50, 0xec, 0x85, 0x1e, 0x73, 0xa6, 0xed, 0x83, 0xdb, 0x1c, 0x62, 0xed, 0x1b, - 0x28, 0xd9, 0xd4, 0x65, 0x31, 0x75, 0xc4, 0xad, 0xbc, 0xa0, 0xf6, 0xfb, 0x3c, 0x94, 0xba, 0xe1, - 0x80, 0x79, 0x4c, 0x4c, 0x6f, 0xe7, 0x47, 0x0f, 0xa0, 0xb8, 0x50, 0xc7, 0xac, 0x33, 0x5d, 0xc5, - 0x9c, 0xc1, 0x5d, 0x79, 0x15, 0x7a, 0x5e, 0xe8, 0x10, 0x11, 0xc6, 0x58, 0xc8, 0xa4, 0x91, 0xa4, - 0xba, 0xc5, 0x2a, 0x24, 0x5b, 0x43, 0xbd, 0x13, 0x35, 0x53, 0x01, 0x95, 0x2e, 0xb6, 0xd8, 0x3c, - 0x01, 0x7d, 0x08, 0x77, 0xb9, 0x20, 0x82, 0x39, 0xf2, 0x9e, 0x9d, 0x77, 0x98, 0xad, 0x84, 0xd1, - 0x89, 0x52, 0xff, 0xfd, 0x08, 0x90, 0x3f, 0xf1, 0x04, 0xc3, 0x24, 0x0a, 0x70, 0xba, 0x96, 0xb4, - 0xfe, 0x55, 0x9c, 0x66, 0x14, 0xe8, 0x19, 0x93, 0x62, 0xf4, 0xfc, 0x85, 0xde, 0x4b, 0x56, 0x8c, - 0x9e, 0xbf, 0x48, 0x76, 0xf3, 0x35, 0x3c, 0x54, 0xec, 0x28, 0xa6, 0x43, 0x76, 0x99, 0x6e, 0x4b, - 0x66, 0x2d, 0xb5, 0xaf, 0x72, 0x72, 0xd9, 0x4b, 0x48, 0x4f, 0x21, 0x9a, 0x19, 0x40, 0x2e, 0xbb, - 0xb6, 0x0f, 0x5b, 0x0b, 0x5b, 0x93, 0xb9, 0xa1, 0xd3, 0xc3, 0xbd, 0xd3, 0xd3, 0x63, 0xe3, 0x0e, - 0x2a, 0x42, 0xa1, 0x7d, 0xd4, 0xea, 0x19, 0xb9, 0xda, 0x9f, 0x4b, 0x90, 0xef, 0x76, 0xad, 0xdb, - 0xb6, 0xf4, 0xbe, 0xe3, 0xe8, 0x83, 0x90, 0x9f, 0x8a, 0x12, 0x38, 0xca, 0xec, 0x92, 0x12, 0x38, - 0x69, 0x7d, 0x5e, 0x98, 0xab, 0xcf, 0x7d, 0x9f, 0xe2, 0x11, 0x4b, 0x7a, 0x83, 0x55, 0x7b, 0xcd, - 0xf7, 0xe9, 0x21, 0x73, 0xe5, 0xe1, 0x4a, 0x86, 0x13, 0xba, 0x54, 0xb7, 0xe8, 0x12, 0xd8, 0x0a, - 0x5d, 0x8a, 0x3e, 0x06, 0xa4, 0xaf, 0x3d, 0x37, 0xe0, 0xd8, 0x21, 0xce, 0x98, 0x05, 0x23, 0xd5, - 0x0f, 0x64, 0xf7, 0x5e, 0x3b, 0xe0, 0xad, 0x84, 0x7e, 0x3d, 0xdb, 0x14, 0x97, 0x64, 0x9b, 0xdf, - 0xc0, 0x6e, 0x20, 0x23, 0x37, 0xe2, 0x58, 0x97, 0x9e, 0xb2, 0xca, 0x11, 0x71, 0xe8, 0xa9, 0xb6, - 0x61, 0xf3, 0x5a, 0xb1, 0xd9, 0xed, 0x5a, 0xf5, 0x93, 0x30, 0xb0, 0x7a, 0x7d, 0x9d, 0x4f, 0x5b, - 0x09, 0xdc, 0xde, 0x09, 0xc2, 0xc0, 0x8a, 0xf8, 0x3c, 0x55, 0xee, 0x46, 0xb5, 0xf2, 0xd2, 0x42, - 0xc9, 0xf1, 0xaa, 0x56, 0xbd, 0xeb, 0x38, 0x57, 0xac, 0xc0, 0xd1, 0x27, 0x99, 0xb0, 0x12, 0x73, - 0x79, 0xc4, 0x51, 0x45, 0xe5, 0xaa, 0x2d, 0x3f, 0xd1, 0x57, 0x50, 0x75, 0xbc, 0x70, 0xe2, 0xca, - 0xee, 0x8c, 0x3b, 0x31, 0x1b, 0xd0, 0xd8, 0x1d, 0x64, 0x3b, 0xdb, 0x54, 0x3b, 0x33, 0x15, 0xa2, - 0x3f, 0x03, 0x48, 0x77, 0xf9, 0x29, 0xec, 0x10, 0x21, 0x88, 0x33, 0xa6, 0x6e, 0xf6, 0x32, 0x21, - 0x8b, 0x95, 0x2d, 0xf5, 0x42, 0x80, 0x52, 0x9e, 0x7e, 0x98, 0x90, 0xa5, 0xcb, 0xbb, 0x50, 0x96, - 0x36, 0x8e, 0x62, 0xe6, 0x93, 0x78, 0x6a, 0xee, 0x24, 0x8f, 0x2a, 0x6e, 0xc0, 0x7b, 0x09, 0x45, - 0x5a, 0x57, 0x02, 0x38, 0x75, 0xc2, 0xc0, 0x95, 0x90, 0x7b, 0x0a, 0x52, 0x71, 0x03, 0xde, 0x4f, - 0x69, 0x8b, 0x3d, 0xc5, 0xfd, 0x6b, 0x3d, 0xc5, 0x87, 0x70, 0x77, 0xcc, 0xf9, 0x42, 0x26, 0xde, - 0x4d, 0x62, 0x65, 0xcc, 0xf9, 0x6c, 0x22, 0x46, 0x0d, 0xb8, 0xa7, 0x4f, 0x5f, 0x46, 0x96, 0x13, - 0xc6, 0xb1, 0xbe, 0x15, 0xcc, 0xe4, 0x6d, 0x25, 0x61, 0x36, 0xa3, 0xa0, 0x95, 0xb1, 0xd0, 0x8f, - 0xb0, 0x3b, 0x0f, 0xc6, 0x3e, 0x89, 0x92, 0x77, 0xa2, 0x07, 0xaa, 0x97, 0x78, 0xb2, 0xe4, 0x78, - 0xe7, 0x54, 0x74, 0x49, 0x64, 0xef, 0x90, 0x05, 0x8a, 0x7a, 0x56, 0xfa, 0x04, 0xb6, 0x59, 0x74, - 0xfe, 0x39, 0x8e, 0xb0, 0xc3, 0x9d, 0x61, 0xd6, 0x51, 0x56, 0x95, 0x1d, 0x0c, 0xc9, 0xea, 0xb5, - 0xb8, 0x33, 0xd4, 0x8d, 0xa5, 0x86, 0xbf, 0x58, 0x84, 0x3f, 0xcc, 0xe0, 0x2f, 0xe6, 0xe0, 0xfb, - 0xa0, 0x68, 0xca, 0xd3, 0x53, 0xec, 0x23, 0x85, 0xdd, 0x94, 0xf4, 0x76, 0xc0, 0xd3, 0x8e, 0xf5, - 0x7b, 0x30, 0x16, 0x57, 0x2c, 0x0d, 0x2f, 0x0b, 0x56, 0x9d, 0x38, 0x74, 0xcd, 0x0b, 0x92, 0x94, - 0x24, 0x0a, 0xd9, 0x78, 0x4b, 0xc3, 0x84, 0xe7, 0x34, 0x8e, 0x99, 0x4b, 0x75, 0xf4, 0x96, 0x49, - 0x14, 0x9c, 0x6a, 0x52, 0xed, 0x4f, 0x39, 0xd8, 0x59, 0xe6, 0xe9, 0xe8, 0x5d, 0x78, 0x78, 0x72, - 0x7a, 0x82, 0xad, 0x5e, 0x1f, 0xf7, 0x2d, 0xfb, 0xfb, 0x4e, 0xcb, 0xc2, 0xad, 0xd3, 0x93, 0x33, - 0xfb, 0xf4, 0x18, 0x9f, 0xbe, 0x7e, 0x6d, 0xdc, 0x41, 0xff, 0x0b, 0x7b, 0x37, 0x01, 0x64, 0x53, - 0x8b, 0xfb, 0xdd, 0xbe, 0x91, 0x7b, 0x9b, 0x1a, 0x09, 0x58, 0x41, 0x4f, 0xe1, 0xf1, 0x5b, 0x00, - 0xf8, 0xd4, 0x6e, 0xbd, 0xb4, 0x8d, 0x7c, 0xed, 0x5f, 0x79, 0xa8, 0x5c, 0x39, 0xfd, 0xed, 0x6e, - 0x35, 0xd9, 0x1d, 0x7b, 0x82, 0x62, 0x32, 0x11, 0x63, 0x1c, 0x46, 0xca, 0x1c, 0x15, 0xbb, 0xe4, - 0x09, 0xda, 0x9c, 0x88, 0xf1, 0x69, 0x84, 0xf6, 0xa0, 0x92, 0xf1, 0x89, 0x3f, 0x54, 0xb9, 0xad, - 0x62, 0x83, 0x06, 0x34, 0xfd, 0x21, 0x3a, 0x85, 0x0a, 0x9f, 0x0c, 0x70, 0x14, 0x87, 0x43, 0xe6, - 0xd1, 0xa4, 0xea, 0x2f, 0x5f, 0xbb, 0xe0, 0x67, 0x17, 0x2a, 0x07, 0x3d, 0x0d, 0x4f, 0x5e, 0xb4, - 0xca, 0xfc, 0x8a, 0x72, 0x3d, 0x7f, 0xad, 0x2e, 0xc9, 0x5f, 0x4b, 0x03, 0x68, 0x6d, 0x69, 0x00, - 0x55, 0x1d, 0xd8, 0xd6, 0xd3, 0xab, 0x92, 0x56, 0x4f, 0x84, 0x9e, 0xc2, 0x96, 0x4f, 0x2e, 0xf1, - 0xc4, 0xc3, 0x03, 0x26, 0x70, 0x4c, 0x04, 0x55, 0x46, 0x2b, 0xd8, 0x15, 0x9f, 0x5c, 0x7e, 0xe7, - 0x1d, 0x30, 0x61, 0x13, 0x91, 0xc1, 0xdc, 0x19, 0xd8, 0x4a, 0x06, 0x6b, 0xa7, 0xb0, 0x6a, 0x08, - 0xc6, 0xe2, 0xb6, 0x96, 0x3c, 0x9c, 0x58, 0xf3, 0x0f, 0x27, 0xcf, 0x7e, 0xc6, 0x4a, 0x8b, 0x6b, - 0x9e, 0x7d, 0x44, 0xa1, 0x50, 0x3c, 0x66, 0xa3, 0xb1, 0x10, 0xd1, 0xed, 0x8a, 0x09, 0xd5, 0x0d, - 0xa8, 0xb4, 0x92, 0x5e, 0x28, 0x49, 0x55, 0xba, 0x91, 0x50, 0xf5, 0x6d, 0x52, 0x63, 0x50, 0xec, - 0x86, 0x01, 0x13, 0x61, 0x7c, 0xbb, 0x69, 0x3e, 0x00, 0x23, 0x0a, 0x3d, 0x8f, 0x05, 0x23, 0xcc, - 0x02, 0x41, 0xe3, 0x73, 0xe2, 0x99, 0x5f, 0xa9, 0xfc, 0xbe, 0xa5, 0xe9, 0x1d, 0x4d, 0xae, 0x7d, - 0x09, 0x85, 0x76, 0xaf, 0x73, 0xbb, 0xe2, 0xea, 0x1f, 0x39, 0x28, 0xb4, 0x03, 0xfe, 0x9f, 0x34, - 0x05, 0xaa, 0x42, 0x51, 0x96, 0x14, 0xde, 0xd9, 0xd9, 0x71, 0xfa, 0x62, 0x9f, 0x8e, 0x91, 0x05, - 0xeb, 0x31, 0x75, 0xc2, 0xd8, 0x4d, 0x03, 0xe0, 0xa3, 0x85, 0xa3, 0x3d, 0x24, 0x82, 0x5e, 0x90, - 0x69, 0xfb, 0xa4, 0xaf, 0x1f, 0x09, 0x12, 0x74, 0x47, 0x50, 0x9f, 0xdb, 0xa9, 0x2c, 0xaa, 0xc3, - 0xb6, 0x3b, 0x76, 0x22, 0x75, 0x27, 0xab, 0xc6, 0x78, 0x36, 0x02, 0xee, 0x4a, 0x56, 0x5f, 0x71, - 0xd2, 0x22, 0xfd, 0x8f, 0x39, 0x78, 0xe7, 0x6d, 0x9a, 0xd5, 0xbb, 0x38, 0x4e, 0xb4, 0xeb, 0x8e, - 0x7e, 0x9d, 0x24, 0x00, 0x99, 0x2b, 0x09, 0x21, 0x19, 0x37, 0xe9, 0xeb, 0x41, 0x92, 0x34, 0xe0, - 0x31, 0x54, 0x9c, 0x80, 0xf8, 0x34, 0x45, 0x24, 0x2d, 0x7e, 0x59, 0xd1, 0x34, 0xe4, 0x3e, 0xac, - 0xb9, 0xa1, 0x4f, 0x58, 0xd2, 0xf4, 0x96, 0x6c, 0x3d, 0x3a, 0x78, 0xf2, 0xe3, 0x63, 0xb5, 0xfd, - 0x67, 0x9e, 0xa0, 0xcf, 0xd4, 0xf5, 0xfc, 0x6c, 0x14, 0x2e, 0xfc, 0x19, 0x33, 0x58, 0x53, 0xe3, - 0xcf, 0xfe, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x02, 0xe6, 0xcb, 0xa9, 0x19, 0x00, 0x00, + // 2634 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x59, 0x6f, 0x6f, 0x1b, 0xc7, + 0xd1, 0x37, 0x45, 0x4a, 0x22, 0x87, 0x94, 0x74, 0x5e, 0xc9, 0xd6, 0x99, 0xce, 0xe3, 0xc8, 0xf4, + 0xe3, 0x44, 0xf9, 0x47, 0xe7, 0x61, 0x62, 0xc3, 0x09, 0x12, 0x07, 0x14, 0x79, 0x96, 0x98, 0x47, + 0x94, 0x88, 0xa3, 0x92, 0x06, 0x01, 0xda, 0xc5, 0xf2, 0x6e, 0x49, 0x6e, 0x7d, 0xff, 0x70, 0xbb, + 0x94, 0xc4, 0xbc, 0xeb, 0x47, 0x68, 0x81, 0x7e, 0x84, 0x7e, 0x9c, 0x7e, 0x8b, 0xbe, 0x2d, 0xfa, + 0xa2, 0xed, 0xbb, 0xbe, 0x28, 0x76, 0x6f, 0xef, 0x44, 0x52, 0x94, 0x13, 0x08, 0x28, 0xd0, 0x57, + 0xbc, 0x9d, 0xf9, 0xcd, 0xec, 0xee, 0xec, 0xcc, 0xec, 0xcc, 0x12, 0x1e, 0x7b, 0x82, 0x3e, 0x8b, + 0xe2, 0x50, 0x84, 0xfc, 0x99, 0xef, 0x84, 0xc1, 0x90, 0x8d, 0xd2, 0x5f, 0x5e, 0x57, 0x74, 0xb4, + 0xe1, 0x93, 0x91, 0x4f, 0xea, 0x9a, 0x5a, 0x7d, 0x10, 0xc6, 0xce, 0xcb, 0x38, 0x95, 0x71, 0x42, + 0xdf, 0x0f, 0x83, 0x04, 0x59, 0xfb, 0x27, 0xc0, 0xba, 0x15, 0x84, 0x2e, 0x1d, 0xb4, 0x51, 0x03, + 0x4a, 0x5e, 0x38, 0xc2, 0x1e, 0x3d, 0xa7, 0x9e, 0x99, 0xdb, 0xcb, 0xed, 0x6f, 0x36, 0xee, 0xd5, + 0x13, 0x4d, 0x4a, 0x41, 0xfd, 0x38, 0x1c, 0x1d, 0x4b, 0xa6, 0x5d, 0xf4, 0xf4, 0x17, 0x32, 0x20, + 0x1f, 0x39, 0xcc, 0x5c, 0xd9, 0xcb, 0xed, 0xaf, 0xda, 0xf2, 0x13, 0x55, 0xa1, 0x48, 0x49, 0x3c, + 0x74, 0x02, 0xd7, 0x33, 0xf3, 0x8a, 0x9c, 0x8d, 0xd1, 0x13, 0xd8, 0x18, 0x90, 0xc0, 0xbd, 0x60, + 0xae, 0x18, 0x63, 0x7f, 0xfc, 0x93, 0x59, 0x50, 0x80, 0x4a, 0x46, 0xec, 0x8e, 0x7f, 0x42, 0xef, + 0x42, 0x39, 0xf2, 0xfc, 0x80, 0xb9, 0xd8, 0x63, 0x5c, 0x98, 0xab, 0x7b, 0xb9, 0xfd, 0x92, 0x0d, + 0x09, 0xe9, 0x98, 0x71, 0x81, 0x9e, 0xc1, 0x36, 0x9f, 0x0c, 0x86, 0x31, 0xf1, 0x29, 0x26, 0x9c, + 0xb3, 0x51, 0xe0, 0xd3, 0x40, 0x98, 0x6b, 0x4a, 0x17, 0x4a, 0x59, 0xcd, 0x8c, 0x83, 0x5e, 0x82, + 0xc9, 0x23, 0xea, 0x30, 0xe2, 0xe1, 0x4c, 0x30, 0x22, 0x42, 0xd0, 0x38, 0x30, 0xd7, 0x95, 0xd4, + 0x7d, 0xcd, 0xef, 0x6b, 0x76, 0x2f, 0xe1, 0xa2, 0x06, 0xdc, 0x23, 0x9e, 0x17, 0x5e, 0x60, 0xaa, + 0x6c, 0x84, 0x45, 0x4c, 0x02, 0xee, 0x33, 0x61, 0x16, 0xf7, 0x72, 0xfb, 0x45, 0x7b, 0x5b, 0x31, + 0x13, 0xfb, 0x9d, 0x69, 0x96, 0x34, 0x89, 0x20, 0x8e, 0x59, 0x4a, 0x4c, 0x22, 0x88, 0x83, 0xbe, + 0x80, 0xa2, 0xc3, 0x87, 0x03, 0x1c, 0x13, 0x61, 0x82, 0xb2, 0xeb, 0xa3, 0xfa, 0xdc, 0x09, 0xd5, + 0xf5, 0x11, 0xd4, 0x5b, 0xfd, 0xd7, 0x07, 0x36, 0x11, 0xf6, 0xba, 0xc4, 0xdb, 0x44, 0xa0, 0x07, + 0x50, 0x54, 0xc6, 0xc3, 0x8d, 0x91, 0x59, 0xde, 0xcb, 0xef, 0xaf, 0xda, 0xeb, 0x6a, 0xdc, 0x18, + 0xa1, 0x6f, 0x00, 0x84, 0xeb, 0xe2, 0x44, 0x83, 0x59, 0xd9, 0xcb, 0xed, 0x97, 0x1b, 0x7b, 0x37, + 0xe8, 0x3d, 0x6b, 0xb7, 0x5b, 0x8a, 0x62, 0x97, 0x84, 0xeb, 0x26, 0x9f, 0x52, 0xc1, 0xf0, 0x4a, + 0xc1, 0xc6, 0x5b, 0x15, 0xbc, 0xbe, 0x52, 0x30, 0xcc, 0x14, 0x10, 0xb8, 0x47, 0x83, 0x81, 0x56, + 0xc0, 0xf1, 0x60, 0x8a, 0x39, 0x8d, 0x19, 0xf1, 0xcc, 0xcd, 0xbd, 0xfc, 0x7e, 0xb9, 0x51, 0xbf, + 0x41, 0x97, 0x15, 0x0c, 0x12, 0x05, 0xfc, 0x60, 0xda, 0x57, 0x02, 0x56, 0x20, 0xe2, 0xa9, 0x8d, + 0xe8, 0x35, 0x46, 0xb5, 0x05, 0xa5, 0x6c, 0xea, 0x39, 0xd7, 0xca, 0x2d, 0xb8, 0x56, 0xc6, 0x9b, + 0x78, 0xda, 0x1b, 0xb3, 0x71, 0xf5, 0x0f, 0x39, 0x28, 0x9d, 0xfd, 0x22, 0x2d, 0x37, 0xb8, 0xd6, + 0xca, 0xad, 0x5c, 0x2b, 0xff, 0x36, 0xd7, 0xaa, 0xfe, 0x65, 0x05, 0x2a, 0x89, 0x45, 0xfe, 0xab, + 0xd6, 0x95, 0x46, 0x74, 0xe1, 0x2a, 0xa2, 0x3f, 0x00, 0x23, 0xf5, 0x7b, 0x4c, 0x03, 0x32, 0xf0, + 0xa8, 0xab, 0xa2, 0xb2, 0x68, 0x6f, 0xa5, 0x74, 0x2b, 0x21, 0xa3, 0xc7, 0x50, 0x71, 0xe9, 0x39, + 0x73, 0x28, 0x76, 0x3c, 0xc2, 0xb9, 0x8a, 0xc9, 0x92, 0x5d, 0x4e, 0x68, 0x2d, 0x49, 0xba, 0x9e, + 0x03, 0xd6, 0x97, 0xe4, 0x00, 0x1d, 0x43, 0xc5, 0xab, 0x18, 0xda, 0x85, 0x75, 0x87, 0x7a, 0x1e, + 0x66, 0xae, 0x8e, 0xac, 0x35, 0x39, 0xec, 0xb8, 0xe8, 0x7f, 0x00, 0x58, 0x84, 0x89, 0xeb, 0xc6, + 0x94, 0x73, 0x15, 0x5e, 0x25, 0xbb, 0xc4, 0xa2, 0x66, 0x42, 0xa8, 0xfe, 0x16, 0x76, 0x6f, 0xf0, + 0x37, 0x39, 0xc9, 0x1b, 0x3a, 0x55, 0xb6, 0x2e, 0xd9, 0xf2, 0x13, 0x7d, 0x01, 0xab, 0xe7, 0xc4, + 0x9b, 0x50, 0x65, 0xd8, 0x72, 0xe3, 0xc9, 0x8d, 0x0e, 0x7c, 0x75, 0x6c, 0x76, 0x22, 0xf1, 0xe5, + 0xca, 0xcb, 0x5c, 0xed, 0x03, 0x58, 0xd7, 0x01, 0x8c, 0x36, 0x01, 0xd4, 0x67, 0xf3, 0x0c, 0x37, + 0x0e, 0x8d, 0x3b, 0xb3, 0xe3, 0xcf, 0x0e, 0x8d, 0x5c, 0xed, 0x77, 0x1b, 0x50, 0xea, 0xb1, 0x88, + 0x7a, 0x2c, 0xa0, 0xb7, 0xcb, 0xbc, 0x8f, 0xa0, 0x3c, 0xa1, 0x98, 0x45, 0x78, 0xe0, 0x85, 0xce, + 0x1b, 0xb5, 0xe2, 0x92, 0x5d, 0x9a, 0xd0, 0x4e, 0x74, 0x20, 0x09, 0x32, 0x8d, 0x06, 0xe4, 0xea, + 0xc0, 0xf2, 0xea, 0xc0, 0x20, 0x20, 0xd9, 0x59, 0xbd, 0x07, 0x5b, 0x2e, 0x1d, 0x92, 0x89, 0x27, + 0x70, 0x3c, 0xf1, 0xa8, 0xb4, 0x6c, 0x72, 0x5c, 0x1b, 0x9a, 0x6c, 0x4f, 0x3c, 0xda, 0x71, 0x51, + 0x1b, 0x8a, 0x9c, 0xc6, 0xf2, 0x00, 0xb9, 0x59, 0xdc, 0xcb, 0xef, 0x6f, 0x36, 0xf6, 0x17, 0xec, + 0x92, 0x6d, 0xa4, 0x7e, 0x42, 0xc5, 0x45, 0x18, 0xbf, 0xe9, 0x6b, 0xbc, 0x9d, 0x49, 0xa2, 0x3e, + 0xdc, 0x55, 0xc9, 0x92, 0xba, 0x78, 0x14, 0x53, 0x1c, 0x51, 0x1a, 0x73, 0xb3, 0xa4, 0xf2, 0xc4, + 0xfb, 0x37, 0xaa, 0x6b, 0x26, 0x12, 0x87, 0x31, 0xed, 0x51, 0x1a, 0xdb, 0x5b, 0x64, 0x6e, 0xcc, + 0xd1, 0x09, 0x6c, 0xb1, 0xc8, 0x8d, 0x31, 0xbd, 0x8c, 0xc2, 0x58, 0x60, 0x97, 0x27, 0xf9, 0xb5, + 0xdc, 0x78, 0xef, 0x46, 0x95, 0x9d, 0x5e, 0xdb, 0xb6, 0x14, 0xbc, 0xcd, 0x85, 0xbd, 0x21, 0xc5, + 0xb3, 0x21, 0x7a, 0x0e, 0x6b, 0x1e, 0xc3, 0x13, 0xca, 0x75, 0x3a, 0x7d, 0x74, 0xa3, 0x9a, 0x63, + 0xf6, 0x1d, 0xe5, 0xf6, 0xaa, 0x27, 0x7f, 0xd0, 0x17, 0xf0, 0x80, 0x8f, 0x18, 0xf6, 0x49, 0x40, + 0x46, 0x54, 0x86, 0x1f, 0x66, 0x43, 0xe2, 0x50, 0x7c, 0xee, 0x91, 0x40, 0xe5, 0xd5, 0x92, 0x7d, + 0x9f, 0x8f, 0x58, 0x37, 0xe3, 0x77, 0x24, 0xfb, 0x7b, 0x8f, 0x04, 0xe8, 0x15, 0xbc, 0xb3, 0x54, + 0x54, 0xbb, 0xb4, 0xb9, 0xa9, 0xa4, 0xcd, 0xeb, 0xd2, 0x1d, 0xe5, 0xe1, 0xe8, 0x39, 0xec, 0x2e, + 0x95, 0x1f, 0x5d, 0x98, 0x5b, 0x4a, 0x74, 0xe7, 0xba, 0xe8, 0xe1, 0x05, 0x7a, 0x05, 0xa5, 0x31, + 0x4d, 0x33, 0xff, 0x5d, 0xb5, 0xd7, 0xc7, 0x37, 0xee, 0xf5, 0xc8, 0xd2, 0xae, 0x5e, 0x1c, 0x53, + 0x9d, 0xab, 0x5e, 0xc1, 0x03, 0x97, 0x71, 0xe9, 0x47, 0x78, 0x4c, 0x89, 0x4b, 0x63, 0x4c, 0x83, + 0x98, 0x39, 0x63, 0x95, 0x95, 0x0c, 0xe9, 0x6a, 0x07, 0x2b, 0x66, 0xce, 0xde, 0xd5, 0xa0, 0x23, + 0x85, 0xb1, 0x32, 0x48, 0xb5, 0x01, 0x9b, 0xf3, 0x67, 0x8b, 0x36, 0x61, 0x85, 0x45, 0x3a, 0x16, + 0x57, 0x58, 0x94, 0x06, 0xa7, 0x74, 0xeb, 0x0d, 0x15, 0x9c, 0xd5, 0xcf, 0x60, 0x63, 0xee, 0xf0, + 0xae, 0x89, 0x20, 0x28, 0x48, 0x96, 0x96, 0x51, 0xdf, 0xd5, 0x5f, 0xc3, 0xaa, 0x3a, 0x2a, 0xb4, + 0x03, 0xab, 0xcc, 0xe7, 0x8c, 0x9b, 0xb9, 0xbd, 0xfc, 0x7e, 0xc9, 0x4e, 0x06, 0xc8, 0x84, 0x75, + 0xf9, 0xeb, 0x06, 0xdc, 0x5c, 0x51, 0xf4, 0x74, 0x28, 0x95, 0xf9, 0xc4, 0xe1, 0x66, 0x5e, 0x91, + 0xd5, 0xb7, 0x5c, 0x13, 0x8b, 0xb8, 0x59, 0x50, 0x24, 0xf9, 0x59, 0xfd, 0x5b, 0x01, 0x8a, 0xa9, + 0x79, 0x64, 0xce, 0x4d, 0xa2, 0x6d, 0x89, 0x4d, 0x72, 0x2a, 0xfc, 0xee, 0x27, 0xfc, 0x45, 0x73, + 0xa0, 0x8f, 0xe0, 0xae, 0x96, 0xa4, 0x81, 0x13, 0x4f, 0x23, 0xc1, 0xc2, 0x40, 0x6d, 0xa3, 0x68, + 0x1b, 0x09, 0xc3, 0xca, 0xe8, 0x68, 0x04, 0xdb, 0x57, 0xa8, 0xa6, 0x37, 0x0a, 0x63, 0x26, 0xc6, + 0xbe, 0x0a, 0xf0, 0xcd, 0xc6, 0xf3, 0x9f, 0x3d, 0xc5, 0xba, 0x75, 0x5d, 0xd8, 0x5e, 0xa6, 0x11, + 0xd9, 0x50, 0x19, 0x13, 0x3e, 0x7e, 0x3d, 0x09, 0x1c, 0xb5, 0xa0, 0x82, 0x9a, 0xa1, 0xfe, 0xf3, + 0x33, 0x1c, 0xcd, 0x48, 0xd9, 0x73, 0x3a, 0xa4, 0x4e, 0x1a, 0x38, 0xa1, 0xcb, 0x82, 0xd1, 0xd9, + 0x34, 0xa2, 0xea, 0x1e, 0xf9, 0x45, 0x3a, 0xad, 0x19, 0x29, 0x7b, 0x4e, 0x07, 0x7a, 0x0a, 0x9b, + 0x57, 0xcb, 0xc7, 0xd2, 0x6b, 0x74, 0x1e, 0xbb, 0xa2, 0xfe, 0x3f, 0x9d, 0xd6, 0x7c, 0xd8, 0x5e, + 0xb2, 0x75, 0xb4, 0x0e, 0x79, 0xbb, 0xf5, 0xb9, 0x71, 0x07, 0xed, 0xc2, 0x76, 0xd3, 0xea, 0x37, + 0x9e, 0xbf, 0xc0, 0xad, 0x83, 0x16, 0x3e, 0xea, 0x36, 0x5b, 0xb8, 0xdb, 0x7e, 0x6e, 0xe4, 0x66, + 0x18, 0x56, 0xeb, 0xe0, 0x8a, 0xb1, 0x82, 0x1e, 0xc2, 0xee, 0xe1, 0x8f, 0x9d, 0x5e, 0xcf, 0x6a, + 0xe3, 0x19, 0x40, 0xff, 0xa8, 0xf9, 0x7f, 0x46, 0xbe, 0xf6, 0x31, 0x54, 0x66, 0xed, 0x20, 0xe7, + 0x91, 0x52, 0x77, 0xe4, 0xc7, 0x91, 0xf5, 0x83, 0x91, 0x43, 0x00, 0x6b, 0xfd, 0xa3, 0x66, 0xe3, + 0xf9, 0x0b, 0x63, 0xa5, 0xf6, 0xbe, 0x2c, 0x06, 0x66, 0xf6, 0x04, 0xb0, 0x76, 0xd0, 0xec, 0x5b, + 0x2f, 0xe4, 0xc2, 0xca, 0xb0, 0x7e, 0x64, 0xfd, 0xd0, 0x38, 0xe8, 0x9c, 0x18, 0xb9, 0xda, 0xd7, + 0xb0, 0xb5, 0x90, 0x64, 0x91, 0x01, 0xc5, 0xae, 0x75, 0x66, 0xd9, 0x9d, 0x93, 0x43, 0xe3, 0x4e, + 0x75, 0xa5, 0x98, 0x93, 0x53, 0xb4, 0x7b, 0x1d, 0x23, 0x87, 0xb6, 0xa0, 0x6c, 0x9d, 0xbc, 0x3e, + 0xb5, 0x5b, 0x56, 0xd7, 0x3a, 0x39, 0x33, 0x56, 0xbe, 0x2d, 0x14, 0x0b, 0xc6, 0xea, 0xb7, 0x85, + 0x62, 0xd9, 0xa8, 0xd4, 0xfe, 0x9e, 0x83, 0x62, 0x9f, 0x72, 0xce, 0xc2, 0xe0, 0x76, 0x57, 0xd0, + 0x13, 0xd8, 0x88, 0xa9, 0x47, 0xa6, 0xd9, 0x25, 0x93, 0xb8, 0x6c, 0x45, 0x11, 0xd3, 0x6b, 0x06, + 0x83, 0x79, 0x41, 0x3c, 0x8f, 0x0a, 0x4c, 0x2f, 0xc7, 0x64, 0xc2, 0x05, 0x76, 0xa9, 0xa0, 0x89, + 0x47, 0xe5, 0x55, 0xe6, 0x79, 0xba, 0x70, 0xfa, 0xbf, 0x52, 0x70, 0x2b, 0x41, 0xb7, 0x53, 0xb0, + 0x7d, 0xff, 0x62, 0x29, 0x1d, 0x7d, 0x02, 0xdb, 0xa3, 0x4b, 0x3c, 0x9a, 0xe2, 0xf9, 0xb5, 0x14, + 0x92, 0xf0, 0x19, 0x5d, 0x1e, 0x4e, 0xed, 0x99, 0xf5, 0xd4, 0xfe, 0x9c, 0x83, 0xfb, 0xcb, 0x67, + 0x40, 0x9f, 0xc2, 0x8e, 0xa0, 0xb1, 0xcf, 0x02, 0x22, 0x28, 0x0e, 0x83, 0x74, 0xc1, 0x3a, 0x78, + 0x51, 0xc6, 0x3b, 0x0d, 0xb4, 0x28, 0x6a, 0xc3, 0x9a, 0x4f, 0xc5, 0x38, 0x4c, 0xb6, 0xbe, 0xd9, + 0xf8, 0xf8, 0x17, 0x6d, 0xa5, 0xde, 0x55, 0x32, 0xb6, 0x96, 0x95, 0x25, 0x8c, 0x60, 0x3e, 0x0d, + 0x27, 0x02, 0xfb, 0x5c, 0x19, 0x65, 0xc3, 0x2e, 0x69, 0x4a, 0x97, 0xd7, 0xde, 0x81, 0xb5, 0x44, + 0x00, 0x21, 0xd8, 0x3c, 0xbc, 0x3c, 0x8b, 0x89, 0xf3, 0x86, 0xba, 0xf2, 0x76, 0xe6, 0xc6, 0x9d, + 0xda, 0x2b, 0x28, 0xf6, 0x42, 0x8f, 0x39, 0xd3, 0xf6, 0xc1, 0x6d, 0x0e, 0xb1, 0xf6, 0x0d, 0x94, + 0x6c, 0xea, 0xb2, 0x98, 0x3a, 0xe2, 0x56, 0x5e, 0x50, 0xfb, 0x7d, 0x1e, 0x4a, 0xdd, 0x70, 0xc0, + 0x3c, 0x26, 0xa6, 0xb7, 0xf3, 0xa3, 0x07, 0x50, 0x5c, 0xa8, 0x63, 0xd6, 0x99, 0xae, 0x62, 0xce, + 0xe0, 0xae, 0xbc, 0x0a, 0x3d, 0x2f, 0x74, 0x88, 0x08, 0x63, 0x2c, 0x64, 0xd2, 0x48, 0x52, 0xdd, + 0x62, 0x15, 0x92, 0xad, 0xa1, 0xde, 0x89, 0x9a, 0xa9, 0x80, 0x4a, 0x17, 0x5b, 0x6c, 0x9e, 0x80, + 0x3e, 0x84, 0xbb, 0x5c, 0x10, 0xc1, 0x1c, 0x79, 0xcf, 0xce, 0x3b, 0xcc, 0x56, 0xc2, 0xe8, 0x44, + 0xa9, 0xff, 0x7e, 0x04, 0xc8, 0x9f, 0x78, 0x82, 0x61, 0x12, 0x05, 0x38, 0x5d, 0x4b, 0x5a, 0xff, + 0x2a, 0x4e, 0x33, 0x0a, 0xf4, 0x8c, 0x49, 0x31, 0x7a, 0xfe, 0x42, 0xef, 0x25, 0x2b, 0x46, 0xcf, + 0x5f, 0x24, 0xbb, 0xf9, 0x1a, 0x1e, 0x2a, 0x76, 0x14, 0xd3, 0x21, 0xbb, 0x4c, 0xb7, 0x25, 0xb3, + 0x96, 0xda, 0x57, 0x39, 0xb9, 0xec, 0x25, 0xa4, 0xa7, 0x10, 0xcd, 0x0c, 0x20, 0x97, 0x5d, 0xdb, + 0x87, 0xad, 0x85, 0xad, 0xc9, 0xdc, 0xd0, 0xe9, 0xe1, 0xde, 0xe9, 0xe9, 0xb1, 0x71, 0x07, 0x15, + 0xa1, 0xd0, 0x3e, 0x6a, 0xf5, 0x8c, 0x5c, 0xed, 0xaf, 0x25, 0xc8, 0x77, 0xbb, 0xd6, 0x6d, 0x5b, + 0x7a, 0xdf, 0x71, 0xf4, 0x41, 0xc8, 0x4f, 0x45, 0x09, 0x1c, 0x65, 0x76, 0x49, 0x09, 0x9c, 0xb4, + 0x3e, 0x2f, 0xcc, 0xd5, 0xe7, 0xbe, 0x4f, 0xf1, 0x88, 0x25, 0xbd, 0xc1, 0xaa, 0xbd, 0xe6, 0xfb, + 0xf4, 0x90, 0xb9, 0xf2, 0x70, 0x25, 0xc3, 0x09, 0x5d, 0xaa, 0x5b, 0x74, 0x09, 0x6c, 0x85, 0x2e, + 0x45, 0x1f, 0x03, 0xd2, 0xd7, 0x9e, 0x1b, 0x70, 0xec, 0x10, 0x67, 0xcc, 0x82, 0x91, 0xea, 0x07, + 0xb2, 0x7b, 0xaf, 0x1d, 0xf0, 0x56, 0x42, 0xbf, 0x9e, 0x6d, 0x8a, 0x4b, 0xb2, 0xcd, 0x6f, 0x60, + 0x37, 0x90, 0x91, 0x1b, 0x71, 0xac, 0x4b, 0x4f, 0x59, 0xe5, 0x88, 0x38, 0xf4, 0x54, 0xdb, 0xb0, + 0x79, 0xad, 0xd8, 0xec, 0x76, 0xad, 0xfa, 0x49, 0x18, 0x58, 0xbd, 0xbe, 0xce, 0xa7, 0xad, 0x04, + 0x6e, 0xef, 0x04, 0x61, 0x60, 0x45, 0x7c, 0x9e, 0x2a, 0x77, 0xa3, 0x5a, 0x79, 0x69, 0xa1, 0xe4, + 0x78, 0x55, 0xab, 0xde, 0x75, 0x9c, 0x2b, 0x56, 0xe0, 0xe8, 0x93, 0x4c, 0x58, 0x89, 0xb9, 0x3c, + 0xe2, 0xa8, 0xa2, 0x72, 0xd5, 0x96, 0x9f, 0xe8, 0x2b, 0xa8, 0x3a, 0x5e, 0x38, 0x71, 0x65, 0x77, + 0xc6, 0x9d, 0x98, 0x0d, 0x68, 0xec, 0x0e, 0xb2, 0x9d, 0x6d, 0xaa, 0x9d, 0x99, 0x0a, 0xd1, 0x9f, + 0x01, 0xa4, 0xbb, 0xfc, 0x14, 0x76, 0x88, 0x10, 0xc4, 0x19, 0x53, 0x37, 0x7b, 0x99, 0x90, 0xc5, + 0xca, 0x96, 0x7a, 0x21, 0x40, 0x29, 0x4f, 0x3f, 0x4c, 0xc8, 0xd2, 0xa5, 0x01, 0xf7, 0xe4, 0x29, + 0x48, 0x5b, 0x09, 0x76, 0x4e, 0xb1, 0x43, 0x22, 0xe2, 0x30, 0x31, 0x55, 0xc5, 0xda, 0xaa, 0xbd, + 0xed, 0xfb, 0xd4, 0xd6, 0xbc, 0x96, 0x66, 0xc9, 0x0e, 0x42, 0x9e, 0x4b, 0x14, 0x33, 0x9f, 0xc4, + 0x53, 0x73, 0x27, 0x79, 0x88, 0x71, 0x03, 0xde, 0x4b, 0x28, 0xf2, 0x44, 0x24, 0x80, 0x53, 0x27, + 0x0c, 0x5c, 0x09, 0xb9, 0xa7, 0x20, 0x15, 0x37, 0xe0, 0xfd, 0x94, 0xb6, 0xd8, 0x87, 0xdc, 0xbf, + 0xd6, 0x87, 0x7c, 0x08, 0x77, 0xc7, 0x9c, 0x2f, 0x64, 0xef, 0xdd, 0x24, 0xbe, 0xc6, 0x9c, 0xcf, + 0x26, 0x6f, 0xb9, 0x0d, 0xed, 0x31, 0x32, 0x1a, 0x9d, 0x30, 0x8e, 0xf5, 0x4d, 0x62, 0x26, 0xef, + 0x31, 0x09, 0xb3, 0x19, 0x05, 0xad, 0x8c, 0x85, 0x7e, 0x84, 0xdd, 0x79, 0x30, 0xf6, 0x49, 0x94, + 0xbc, 0x2d, 0x3d, 0x50, 0xfd, 0xc7, 0x93, 0x25, 0x2e, 0x31, 0xa7, 0xa2, 0x4b, 0x22, 0x7b, 0x87, + 0x2c, 0x50, 0xd4, 0x53, 0xd4, 0x27, 0xb0, 0xcd, 0xa2, 0xf3, 0xcf, 0x71, 0x84, 0x1d, 0xee, 0x0c, + 0xb3, 0x2e, 0xb4, 0xaa, 0xec, 0x60, 0x48, 0x56, 0xaf, 0xc5, 0x9d, 0xa1, 0x6e, 0x46, 0x35, 0xfc, + 0xc5, 0x22, 0xfc, 0x61, 0x06, 0x7f, 0x31, 0x07, 0xdf, 0x07, 0x45, 0x53, 0xd1, 0x91, 0x62, 0x1f, + 0x29, 0xec, 0xa6, 0xa4, 0xb7, 0x03, 0x9e, 0x76, 0xb9, 0xdf, 0x83, 0xb1, 0xb8, 0x62, 0x69, 0x78, + 0x59, 0xe4, 0xea, 0x64, 0xa3, 0xeb, 0x64, 0x90, 0xa4, 0x24, 0xb9, 0xc8, 0x66, 0x5d, 0x1a, 0x26, + 0x3c, 0xa7, 0x71, 0xcc, 0x5c, 0xaa, 0x23, 0xbe, 0x4c, 0xa2, 0xe0, 0x54, 0x93, 0x6a, 0x7f, 0xca, + 0xc1, 0xce, 0xb2, 0xe8, 0x40, 0xef, 0xc2, 0xc3, 0x93, 0xd3, 0x13, 0x6c, 0xf5, 0xfa, 0xb8, 0x6f, + 0xd9, 0xdf, 0x77, 0x5a, 0x16, 0x6e, 0x9d, 0x9e, 0x9c, 0xd9, 0xa7, 0xc7, 0xf8, 0xf4, 0xf5, 0x6b, + 0xe3, 0x0e, 0xfa, 0x5f, 0xd8, 0xbb, 0x09, 0x20, 0x1b, 0x61, 0xdc, 0xef, 0xf6, 0x8d, 0xdc, 0xdb, + 0xd4, 0x48, 0xc0, 0x0a, 0x7a, 0x0a, 0x8f, 0xdf, 0x02, 0xc0, 0xa7, 0x76, 0xeb, 0xa5, 0x6d, 0xe4, + 0x6b, 0xff, 0xca, 0x43, 0xe5, 0x2a, 0x50, 0x6e, 0x77, 0x13, 0xca, 0x8e, 0xda, 0x13, 0x14, 0x93, + 0x89, 0x18, 0xe3, 0x30, 0x52, 0xe6, 0xa8, 0xd8, 0x25, 0x4f, 0xd0, 0xe6, 0x44, 0x8c, 0x4f, 0x23, + 0xb4, 0x07, 0x95, 0x8c, 0x4f, 0xfc, 0xa1, 0xca, 0x87, 0x15, 0x1b, 0x34, 0xa0, 0xe9, 0x0f, 0xd1, + 0x29, 0x54, 0xf8, 0x64, 0x80, 0xa3, 0x38, 0x1c, 0x32, 0x8f, 0x26, 0x9d, 0x42, 0xf9, 0x5a, 0x51, + 0x30, 0xbb, 0x50, 0x39, 0xe8, 0x69, 0x78, 0xf2, 0x0a, 0x56, 0xe6, 0x57, 0x94, 0xeb, 0x39, 0x6f, + 0x75, 0x49, 0xce, 0x5b, 0x1a, 0x40, 0x6b, 0x4b, 0x03, 0xa8, 0xea, 0xc0, 0xb6, 0x9e, 0x5e, 0x95, + 0xc1, 0x7a, 0x22, 0xf4, 0x14, 0xb6, 0x7c, 0x72, 0x89, 0x27, 0x1e, 0x1e, 0x30, 0x81, 0x63, 0x22, + 0xa8, 0x32, 0x5a, 0xc1, 0xae, 0xf8, 0xe4, 0xf2, 0x3b, 0xef, 0x80, 0x09, 0x9b, 0x88, 0x0c, 0xe6, + 0xce, 0xc0, 0x56, 0x32, 0x58, 0x3b, 0x85, 0x55, 0x43, 0x30, 0x16, 0xb7, 0xb5, 0xe4, 0xb1, 0xc5, + 0x9a, 0x7f, 0x6c, 0x79, 0xf6, 0x33, 0x56, 0x5a, 0x5c, 0xf3, 0xec, 0xc3, 0x0b, 0x85, 0xe2, 0x31, + 0x1b, 0x8d, 0x85, 0x88, 0x6e, 0x57, 0x80, 0xa8, 0x0e, 0x42, 0xa5, 0x95, 0xf4, 0x12, 0x4a, 0x2a, + 0xd9, 0x8d, 0x84, 0xaa, 0x6f, 0xa0, 0x1a, 0x83, 0x62, 0x37, 0x0c, 0x98, 0x08, 0xe3, 0xdb, 0x4d, + 0xf3, 0x01, 0x18, 0x51, 0xe8, 0x79, 0x2c, 0x18, 0x61, 0x16, 0x08, 0x1a, 0x9f, 0x13, 0xcf, 0xfc, + 0x4a, 0xe5, 0xdf, 0x2d, 0x4d, 0xef, 0x68, 0x72, 0xed, 0x4b, 0x28, 0xb4, 0x7b, 0x9d, 0xdb, 0x15, + 0x64, 0xff, 0xc8, 0x41, 0xa1, 0x1d, 0xf0, 0xff, 0xa4, 0x29, 0x50, 0x15, 0x8a, 0xb2, 0x0c, 0xf1, + 0xce, 0xce, 0x8e, 0xd3, 0x57, 0xfe, 0x74, 0x8c, 0x2c, 0x58, 0x8f, 0xa9, 0x13, 0xc6, 0x6e, 0x1a, + 0x00, 0x1f, 0x2d, 0x1c, 0xed, 0x21, 0x11, 0xf4, 0x82, 0x4c, 0xdb, 0x27, 0x7d, 0xfd, 0xb0, 0x90, + 0xa0, 0x3b, 0x82, 0xfa, 0xdc, 0x4e, 0x65, 0x51, 0x1d, 0xb6, 0xdd, 0xb1, 0x13, 0xa9, 0x7b, 0x5c, + 0x35, 0xd3, 0xb3, 0x11, 0x70, 0x57, 0xb2, 0xfa, 0x8a, 0x93, 0x16, 0xf6, 0x7f, 0xcc, 0xc1, 0x3b, + 0x6f, 0xd3, 0xac, 0xde, 0xd2, 0x71, 0xa2, 0x5d, 0xbf, 0x02, 0xac, 0x93, 0x04, 0x20, 0x73, 0x25, + 0x21, 0x24, 0xe3, 0x26, 0x6f, 0x01, 0x20, 0x49, 0x1a, 0xf0, 0x18, 0x2a, 0x4e, 0x40, 0xd4, 0x0d, + 0xaa, 0x10, 0xc9, 0xb3, 0x40, 0x59, 0xd1, 0x34, 0xe4, 0x3e, 0xac, 0xb9, 0xa1, 0x4f, 0x58, 0xd2, + 0x28, 0x97, 0x6c, 0x3d, 0x3a, 0x78, 0xf2, 0xe3, 0x63, 0xb5, 0xfd, 0x67, 0x9e, 0xa0, 0xcf, 0xd4, + 0x95, 0xfe, 0x6c, 0x14, 0x2e, 0xfc, 0x81, 0x33, 0x58, 0x53, 0xe3, 0xcf, 0xfe, 0x1d, 0x00, 0x00, + 0xff, 0xff, 0x3c, 0x13, 0xa8, 0xe3, 0xdd, 0x19, 0x00, 0x00, } diff --git a/lte/cloud/go/services/ha/doc.go b/lte/cloud/go/services/ha/doc.go new file mode 100644 index 000000000000..9d67db0aea74 --- /dev/null +++ b/lte/cloud/go/services/ha/doc.go @@ -0,0 +1,35 @@ +/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* Package ha provides the LTE HA orc8r service. + +This service has a single RPC endpoint. The RPC endpoint will be used by +secondary gateways' MME to know when to offload its users for a given ENB +back to the primary. + +To gather this state, this service looks at the primary gateways in the +gateway pool(s) of the calling gateway. For each primary it fetches the +configured ENBs and then checks the following state: + +1. Has the primary checked in within last 3 mins +2. Is the ENB connected to the primary within last 3 mins +3. Does the ENB have throughput on the primary + +The service then sends back the ENB ID -> offload state +for all of these. +*/ +package ha + +const ( + ServiceName = "ha" +) diff --git a/lte/cloud/go/services/ha/ha/main.go b/lte/cloud/go/services/ha/ha/main.go new file mode 100644 index 000000000000..334645efbf5d --- /dev/null +++ b/lte/cloud/go/services/ha/ha/main.go @@ -0,0 +1,57 @@ +/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "magma/lte/cloud/go/lte" + "magma/lte/cloud/go/protos" + "magma/lte/cloud/go/services/ha" + "magma/lte/cloud/go/services/ha/servicers" + "magma/orc8r/cloud/go/orc8r" + "magma/orc8r/cloud/go/service" + "magma/orc8r/cloud/go/services/metricsd" + "magma/orc8r/lib/go/service/config" + + "github.com/golang/glog" + "github.com/prometheus/client_golang/api" + "github.com/prometheus/client_golang/api/prometheus/v1" +) + +func main() { + srv, err := service.NewOrchestratorService(lte.ModuleName, ha.ServiceName) + if err != nil { + glog.Fatalf("Error creating had service: %s", err) + } + promClient := GetPrometheusClient() + servicer := servicers.NewHAServicer(promClient) + protos.RegisterHaServer(srv.GrpcServer, servicer) + + err = srv.Run() + if err != nil { + glog.Fatalf("Error while running ha service and echo server: %s", err) + } +} + +// GetPrometheusClient returns prometheus client +func GetPrometheusClient() v1.API { + metricsConfig, err := config.GetServiceConfig(orc8r.ModuleName, metricsd.ServiceName) + if err != nil { + glog.Fatalf("Could not retrieve metricsd configuration needed to query ENB stats: %s", err) + } + promClient, err := api.NewClient(api.Config{Address: metricsConfig.MustGetString(metricsd.PrometheusQueryAddress)}) + if err != nil { + glog.Fatalf("Error creating prometheus client: %s", promClient) + } + return v1.NewAPI(promClient) +} diff --git a/lte/cloud/go/services/ha/servicers/servicer.go b/lte/cloud/go/services/ha/servicers/servicer.go new file mode 100644 index 000000000000..17d53c29caa2 --- /dev/null +++ b/lte/cloud/go/services/ha/servicers/servicer.go @@ -0,0 +1,274 @@ +/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package servicers + +import ( + "context" + "fmt" + "time" + + "magma/lte/cloud/go/lte" + lte_protos "magma/lte/cloud/go/protos" + "magma/lte/cloud/go/serdes" + lte_models "magma/lte/cloud/go/services/lte/obsidian/models" + "magma/orc8r/cloud/go/orc8r" + "magma/orc8r/cloud/go/services/analytics/query_api" + "magma/orc8r/cloud/go/services/configurator" + "magma/orc8r/cloud/go/services/metricsd/prometheus/restrictor" + "magma/orc8r/cloud/go/services/state" + "magma/orc8r/cloud/go/services/state/wrappers" + "magma/orc8r/lib/go/metrics" + "magma/orc8r/lib/go/protos" + + "github.com/golang/glog" + "github.com/pkg/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + validSecsSinceStateReported = 180 +) + +type HAServicer struct { + promClient query_api.PrometheusAPI +} + +// NewHAServicer creates a new service implementing the HaD proto file +func NewHAServicer(promClient query_api.PrometheusAPI) lte_protos.HaServer { + return &HAServicer{ + promClient: promClient, + } +} + +// GetEnodebOffloadState fetches all primary gateways that the calling gateway +// is in a gateway pool with. For each of these gateways, it then fetches the +// offload state for each of the gateway's ENBs. +func (s *HAServicer) GetEnodebOffloadState(ctx context.Context, req *lte_protos.GetEnodebOffloadStateRequest) (*lte_protos.GetEnodebOffloadStateResponse, error) { + ret := <e_protos.GetEnodebOffloadStateResponse{} + secondaryGw := protos.GetClientGateway(ctx) + if secondaryGw == nil { + return ret, status.Errorf(codes.PermissionDenied, "missing gateway identity") + } + if !secondaryGw.Registered() { + return ret, status.Errorf(codes.PermissionDenied, "gateway is not registered") + } + cfg, err := configurator.LoadEntityConfig(secondaryGw.GetNetworkId(), lte.CellularGatewayEntityType, secondaryGw.LogicalId, lte_models.EntitySerdes) + if err != nil { + errors.Wrap(err, "unable to load cellular gateway configs to find primary gateway's in its pool") + return ret, err + } + cellularCfg, ok := cfg.(*lte_models.GatewayCellularConfigs) + if !ok { + return ret, status.Errorf(codes.Internal, "could not convert stored config to type GatewayCellularConfigs for gw %s", secondaryGw.LogicalId) + } + if cellularCfg.Pooling == nil || len(cellularCfg.Pooling) == 0 { + return ret, fmt.Errorf("gateway '%s' is not configured in a gateway pool", secondaryGw.LogicalId) + } + + // All gateway pool records must have the same capacity, so use the first + // entry + callingRelativeCapacity := cellularCfg.Pooling[0].MmeRelativeCapacity + gwIDsToEnbs := map[string][]string{} + for _, record := range cellularCfg.Pooling { + gwIDsToEnbsInPool, err := s.getPrimaryGatewaysToEnodebs(secondaryGw.GetNetworkId(), string(record.GatewayPoolID), callingRelativeCapacity) + if err != nil { + return <e_protos.GetEnodebOffloadStateResponse{}, err + } + // Since a gateway can be in multiple pools, it is possible + // there could be key collisions. Since the ENBs values will be the + // same each time, this is okay. + for k, v := range gwIDsToEnbsInPool { + gwIDsToEnbs[k] = v + } + } + glog.V(2).Infof("Found the following primary gatewayIDs to ENB SNs: %v", gwIDsToEnbs) + + enbSNsToOffloadState := map[uint32]lte_protos.GetEnodebOffloadStateResponse_EnodebOffloadState{} + for primaryGwID, enbs := range gwIDsToEnbs { + isCheckinValid, err := s.isGatewayCheckinValid(secondaryGw.NetworkId, primaryGwID) + // Since a secondary gateway can serve multiple primary gateways, if + // we are unable to fetch checkin state for a primary, we should + // still continue gathering the offload state to return, rather + // than returning the error. + if err != nil { + glog.Error(err) + continue + } else if !isCheckinValid { + continue + } + for _, enb := range enbs { + offloadState, err := s.getOffloadStateForEnb(secondaryGw.NetworkId, primaryGwID, enb) + // Since a secondary gateway can offload multiple ENBs, if we are + // unable to fetch offload state for an ENB, we should continue + // gathering offload state for other ENBs, rather than returning + // the error. + if err != nil { + glog.Error(err) + continue + } + enbID, err := s.getEnodebID(secondaryGw.NetworkId, secondaryGw.LogicalId, enb) + if err != nil { + glog.Error(err) + continue + } + enbSNsToOffloadState[enbID] = offloadState + } + } + return <e_protos.GetEnodebOffloadStateResponse{EnodebOffloadStates: enbSNsToOffloadState}, nil +} + +func (s *HAServicer) getPrimaryGatewaysToEnodebs(networkID string, gatewayPoolID string, callingRelativeCapacity uint32) (map[string][]string, error) { + poolEnt, err := configurator.LoadEntity( + networkID, lte.CellularGatewayPoolEntityType, gatewayPoolID, + configurator.EntityLoadCriteria{LoadAssocsFromThis: true}, + lte_models.EntitySerdes, + ) + if err != nil { + return map[string][]string{}, err + } + primaryGatewaysToEnbs := map[string][]string{} + for _, gw := range poolEnt.Associations.Filter(lte.CellularGatewayEntityType) { + ent, err := configurator.LoadEntity( + networkID, lte.CellularGatewayEntityType, gw.Key, + configurator.EntityLoadCriteria{LoadConfig: true, LoadAssocsFromThis: true, LoadAssocsToThis: true}, + lte_models.EntitySerdes, + ) + if err != nil { + return map[string][]string{}, err + } + cellularCfg, ok := ent.Config.(*lte_models.GatewayCellularConfigs) + if !ok { + return map[string][]string{}, fmt.Errorf("could not convert stored config to type GatewayCellularConfigs for gw %s", gw.Key) + } + if cellularCfg.Pooling == nil || len(cellularCfg.Pooling) == 0 { + return map[string][]string{}, fmt.Errorf("gateway '%s' is not configured in a gateway pool", gw.Key) + } + // primary gateways are those with strictly higher relative capacity + // than the calling gateway + currentRelativeCapacity := cellularCfg.Pooling[0].MmeRelativeCapacity + if currentRelativeCapacity > callingRelativeCapacity { + enbs := ent.Associations.Filter(lte.CellularEnodebEntityType).Keys() + primaryGatewaysToEnbs[gw.Key] = enbs + } + } + return primaryGatewaysToEnbs, nil +} + +func (s *HAServicer) isGatewayCheckinValid(networkID string, gatewayID string) (bool, error) { + hwID, err := s.getHardwareIDFromGatewayID(networkID, gatewayID) + if err != nil { + return false, err + } + status, err := wrappers.GetGatewayStatus(networkID, hwID) + if err != nil { + return false, err + } + timeSinceCheckin := time.Now().Unix() - int64(status.CheckinTime)/1000 + return timeSinceCheckin < validSecsSinceStateReported, nil +} + +func (s *HAServicer) getOffloadStateForEnb(networkID string, primaryGwID string, enbSN string) (lte_protos.GetEnodebOffloadStateResponse_EnodebOffloadState, error) { + st, err := state.GetState(networkID, lte.EnodebStateType, enbSN, serdes.State) + if err != nil { + return lte_protos.GetEnodebOffloadStateResponse_NO_OP, err + } + enodebState, ok := st.ReportedState.(*lte_models.EnodebState) + if !ok || enodebState == nil { + return lte_protos.GetEnodebOffloadStateResponse_NO_OP, fmt.Errorf("could not convert enodeb state to valid model for enodeb %s", enbSN) + } + enodebState.TimeReported = st.TimeMs + ent, err := configurator.LoadEntityForPhysicalID(st.ReporterID, configurator.EntityLoadCriteria{}, serdes.Entity) + if err == nil { + enodebState.ReportingGatewayID = ent.Key + } + timeSinceReported := time.Now().Unix() - int64(enodebState.TimeReported)/1000 + if timeSinceReported > validSecsSinceStateReported { + glog.V(2).Infof("Returning NO_OP offload state for ENB %s; Time is %d secs too stale", enbSN, timeSinceReported) + return lte_protos.GetEnodebOffloadStateResponse_NO_OP, nil + } + if enodebState.ReportingGatewayID != primaryGwID { + glog.V(2).Infof("Returning NO_OP offload state for ENB %s; Enodeb state gateway ID is not primary gateway but %s", enbSN, enodebState.ReportingGatewayID) + return lte_protos.GetEnodebOffloadStateResponse_NO_OP, nil + } + if !*enodebState.EnodebConnected || !*enodebState.MmeConnected { + glog.V(2).Infof("Returning NO_OP offload state for ENB %s; Enodeb state does not have Enodeb connected or MME connected", enbSN) + return lte_protos.GetEnodebOffloadStateResponse_NO_OP, nil + } + isEnbServing, err := s.isEnbServingTraffic(networkID, primaryGwID, enodebState.IPAddress.String()) + if err != nil { + glog.Error(err) + } + if err != nil || !isEnbServing { + return lte_protos.GetEnodebOffloadStateResponse_PRIMARY_CONNECTED, nil + } + return lte_protos.GetEnodebOffloadStateResponse_PRIMARY_CONNECTED_AND_SERVING_UES, nil +} + +func (s *HAServicer) isEnbServingTraffic(networkID string, gatewayID string, enbIP string) (bool, error) { + // TODO: Update metric query to use the number of users connected to an ENB + // once this metric exists. This will avoid the edge case issue of + // throughput equal to 0 while there are only users connected in IDLE mode. + enbDLQuery := fmt.Sprintf("gtp_port_user_plane_dl_bytes{gatewayID=\"%s\", service=\"pipelined\", ip_addr=\"%s\"}", gatewayID, enbIP) + networkQueryRestrictor := *restrictor.NewQueryRestrictor(restrictor.DefaultOpts).AddMatcher(metrics.NetworkLabelName, networkID) + query, err := networkQueryRestrictor.RestrictQuery(enbDLQuery) + if err != nil { + return false, err + } + trafficVec, err := query_api.QueryPrometheusVector(s.promClient, query) + if err != nil { + return false, err + } + trafficSample := trafficVec[0] + if trafficSample.Value == 0 { + return false, nil + } + return true, nil +} + +func (s *HAServicer) getEnodebID(networkID string, hwID string, enodebSn string) (uint32, error) { + cfg, err := configurator.LoadEntityConfig(networkID, lte.CellularEnodebEntityType, enodebSn, serdes.Entity) + if err != nil { + return 0, err + } + enodebCfg, ok := cfg.(*lte_models.EnodebConfig) + if !ok { + return 0, fmt.Errorf("could not convert Enodeb config to proper type for ENB '%s'", enodebSn) + } + switch enodebCfg.ConfigType { + case "MANAGED": + if enodebCfg == nil || enodebCfg.ManagedConfig == nil { + return 0, fmt.Errorf("could not extract ENB ID from config for ENB '%s'; config was nil", enodebSn) + } + return *enodebCfg.ManagedConfig.CellID, nil + case "UNMANAGED": + if enodebCfg == nil || enodebCfg.UnmanagedConfig == nil { + return 0, fmt.Errorf("could not extract ENB ID from config for ENB '%s'; config was nil", enodebSn) + } + return *enodebCfg.UnmanagedConfig.CellID, nil + default: + return 0, fmt.Errorf("invalid enodeb config type '%s' for ENB '%s'", enodebCfg.ConfigType, enodebSn) + } +} + +func (s *HAServicer) getHardwareIDFromGatewayID(networkID string, gatewayID string) (string, error) { + ent, err := configurator.LoadEntity( + networkID, orc8r.MagmadGatewayType, gatewayID, + configurator.EntityLoadCriteria{LoadMetadata: true}, serdes.Entity, + ) + if err != nil { + return "", err + } + return ent.PhysicalID, nil +} diff --git a/lte/cloud/go/services/ha/servicers/servicer_test.go b/lte/cloud/go/services/ha/servicers/servicer_test.go new file mode 100644 index 000000000000..8209aa069cb7 --- /dev/null +++ b/lte/cloud/go/services/ha/servicers/servicer_test.go @@ -0,0 +1,338 @@ +/* + Copyright 2020 The Magma Authors. + + This source code is licensed under the BSD-style license found in the + LICENSE file in the root directory of this source tree. + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package servicers_test + +import ( + "context" + "fmt" + "testing" + "time" + + "magma/lte/cloud/go/lte" + lte_plugin "magma/lte/cloud/go/plugin" + "magma/lte/cloud/go/protos" + "magma/lte/cloud/go/serdes" + "magma/lte/cloud/go/services/ha/servicers" + lte_models "magma/lte/cloud/go/services/lte/obsidian/models" + "magma/orc8r/cloud/go/clock" + "magma/orc8r/cloud/go/orc8r" + "magma/orc8r/cloud/go/plugin" + "magma/orc8r/cloud/go/pluginimpl" + "magma/orc8r/cloud/go/serde" + "magma/orc8r/cloud/go/services/analytics/query_api/mocks" + "magma/orc8r/cloud/go/services/configurator" + configurator_test_init "magma/orc8r/cloud/go/services/configurator/test_init" + "magma/orc8r/cloud/go/services/orchestrator/obsidian/models" + "magma/orc8r/cloud/go/services/state" + state_test_init "magma/orc8r/cloud/go/services/state/test_init" + "magma/orc8r/cloud/go/services/state/test_utils" + "magma/orc8r/cloud/go/storage" + orc8r_protos "magma/orc8r/lib/go/protos" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func init() { + //_ = flag.Set("alsologtostderr", "true") // uncomment to view logs during test +} + +func TestHAServicer_GetEnodebOffloadState(t *testing.T) { + assert.NoError(t, plugin.RegisterPluginForTests(t, &pluginimpl.BaseOrchestratorPlugin{})) + assert.NoError(t, plugin.RegisterPluginForTests(t, <e_plugin.LteOrchestratorPlugin{})) + configurator_test_init.StartTestService(t) + state_test_init.StartTestService(t) + mockPromClient := &mocks.PrometheusAPI{} + servicer := servicers.NewHAServicer(mockPromClient) + + testNetworkId := "n1" + testGwHwId1 := "hw1" + testGwId1 := "g1" + testGwHwId2 := "hw2" + testGwId2 := "g2" + testGwPool := "pool1" + enbSn := "enb1" + err := configurator.CreateNetwork(configurator.Network{ID: testNetworkId}, serdes.Network) + assert.NoError(t, err) + + // Initialize HA network topology + _, err = configurator.CreateEntity( + testNetworkId, + configurator.NetworkEntity{ + Type: lte.CellularEnodebEntityType, + Key: enbSn, + Config: newDefaultUnmanagedEnodebConfig(), + }, + serdes.Entity, + ) + assert.NoError(t, err) + + _, err = configurator.CreateEntities( + testNetworkId, + []configurator.NetworkEntity{ + { + Type: lte.CellularGatewayEntityType, Key: testGwId1, + Config: newDefaultGatewayConfig(1, 255), + Associations: []storage.TypeAndKey{{Type: lte.CellularEnodebEntityType, Key: enbSn}}, + }, + { + Type: orc8r.MagmadGatewayType, Key: testGwId1, + Name: "foobar", Description: "foo bar", + PhysicalID: testGwHwId1, + Associations: []storage.TypeAndKey{{Type: lte.CellularGatewayEntityType, Key: testGwId1}}, + }, + { + Type: lte.CellularGatewayEntityType, Key: testGwId2, + Config: newDefaultGatewayConfig(2, 1), + Associations: []storage.TypeAndKey{{Type: lte.CellularEnodebEntityType, Key: enbSn}}, + }, + { + Type: orc8r.MagmadGatewayType, Key: testGwId2, + Name: "foobar2", Description: "foo bar", + PhysicalID: testGwHwId2, + Associations: []storage.TypeAndKey{{Type: lte.CellularGatewayEntityType, Key: testGwId2}}, + }, + }, + serdes.Entity, + ) + assert.NoError(t, err) + + _, err = configurator.CreateEntities( + testNetworkId, + []configurator.NetworkEntity{ + { + Type: lte.CellularGatewayPoolEntityType, Key: testGwPool, + Config: <e_models.CellularGatewayPoolConfigs{ + MmeGroupID: 1, + }, + Associations: []storage.TypeAndKey{ + {Type: lte.CellularGatewayEntityType, Key: testGwId1}, + {Type: lte.CellularGatewayEntityType, Key: testGwId2}, + }, + }, + }, + serdes.Entity, + ) + assert.NoError(t, err) + + // Initialize network state for given devices + gwStatus := &models.GatewayStatus{ + CheckinTime: uint64(time.Now().Unix()), + HardwareID: testGwHwId1, + } + ctx1 := test_utils.GetContextWithCertificate(t, testGwHwId1) + test_utils.ReportGatewayStatus(t, ctx1, gwStatus) + gwStatus2 := &models.GatewayStatus{ + CheckinTime: uint64(time.Now().Unix()), + HardwareID: testGwHwId2, + } + ctx2 := test_utils.GetContextWithCertificate(t, testGwHwId2) + test_utils.ReportGatewayStatus(t, ctx2, gwStatus2) + + enbState := getDefaultEnodebState(testGwId1) + reportEnodebState(t, ctx1, enbSn, enbState) + + metric1 := model.Metric{} + metric1["networkID"] = "n1" + metric1["gatewayID"] = "g1" + throughputVec := model.Vector{{ + Metric: metric1, + Value: 10000000, + }} + mockPromClient.On("Query", mock.Anything, mock.Anything, mock.Anything).Return(throughputVec, nil, nil).Once() + + // First test primary healthy + ctx := orc8r_protos.NewGatewayIdentity(testGwHwId2, testNetworkId, testGwId2).NewContextWithIdentity(context.Background()) + res, err := servicer.GetEnodebOffloadState(ctx, &protos.GetEnodebOffloadStateRequest{}) + assert.NoError(t, err) + expectedRes := &protos.GetEnodebOffloadStateResponse{ + EnodebOffloadStates: map[uint32]protos.GetEnodebOffloadStateResponse_EnodebOffloadState{ + 138: protos.GetEnodebOffloadStateResponse_PRIMARY_CONNECTED_AND_SERVING_UES, + }, + } + assert.Equal(t, expectedRes, res) + mockPromClient.AssertExpectations(t) + + // Now simulate failed primary not checking in + stateTooOld := time.Now().Add(-time.Second * 600) + clock.SetAndFreezeClock(t, stateTooOld) + test_utils.ReportGatewayStatus(t, ctx1, gwStatus) + + res, err = servicer.GetEnodebOffloadState(ctx, &protos.GetEnodebOffloadStateRequest{}) + assert.NoError(t, err) + expectedRes = &protos.GetEnodebOffloadStateResponse{ + EnodebOffloadStates: map[uint32]protos.GetEnodebOffloadStateResponse_EnodebOffloadState{}, + } + assert.Equal(t, expectedRes, res) + clock.UnfreezeClock(t) + test_utils.ReportGatewayStatus(t, ctx1, gwStatus) + + // Simulate too old of ENB state + clock.SetAndFreezeClock(t, stateTooOld) + reportEnodebState(t, ctx1, enbSn, enbState) + + res, err = servicer.GetEnodebOffloadState(ctx, &protos.GetEnodebOffloadStateRequest{}) + assert.NoError(t, err) + expectedRes = &protos.GetEnodebOffloadStateResponse{ + EnodebOffloadStates: map[uint32]protos.GetEnodebOffloadStateResponse_EnodebOffloadState{ + 138: protos.GetEnodebOffloadStateResponse_NO_OP, + }, + } + assert.Equal(t, expectedRes, res) + clock.UnfreezeClock(t) + + // ENB not connected + enbState.EnodebConnected = swag.Bool(false) + reportEnodebState(t, ctx1, enbSn, enbState) + + res, err = servicer.GetEnodebOffloadState(ctx, &protos.GetEnodebOffloadStateRequest{}) + assert.NoError(t, err) + expectedRes = &protos.GetEnodebOffloadStateResponse{ + EnodebOffloadStates: map[uint32]protos.GetEnodebOffloadStateResponse_EnodebOffloadState{ + 138: protos.GetEnodebOffloadStateResponse_NO_OP, + }, + } + assert.Equal(t, expectedRes, res) + + // Connected but could not query metrics + enbState.EnodebConnected = swag.Bool(true) + reportEnodebState(t, ctx1, enbSn, enbState) + mockPromClient.On("Query", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil, fmt.Errorf("error")).Once() + + res, err = servicer.GetEnodebOffloadState(ctx, &protos.GetEnodebOffloadStateRequest{}) + assert.NoError(t, err) + expectedRes = &protos.GetEnodebOffloadStateResponse{ + EnodebOffloadStates: map[uint32]protos.GetEnodebOffloadStateResponse_EnodebOffloadState{ + 138: protos.GetEnodebOffloadStateResponse_PRIMARY_CONNECTED, + }, + } + assert.Equal(t, expectedRes, res) + mockPromClient.AssertExpectations(t) + + // Connected but no throughput + throughputVec2 := model.Vector{{ + Metric: metric1, + Value: 0, + }} + mockPromClient.On("Query", mock.Anything, mock.Anything, mock.Anything).Return(throughputVec2, nil, nil).Once() + + res, err = servicer.GetEnodebOffloadState(ctx, &protos.GetEnodebOffloadStateRequest{}) + assert.NoError(t, err) + expectedRes = &protos.GetEnodebOffloadStateResponse{ + EnodebOffloadStates: map[uint32]protos.GetEnodebOffloadStateResponse_EnodebOffloadState{ + 138: protos.GetEnodebOffloadStateResponse_PRIMARY_CONNECTED, + }, + } + assert.Equal(t, expectedRes, res) + mockPromClient.AssertExpectations(t) + + // Back to connected with users + mockPromClient.On("Query", mock.Anything, mock.Anything, mock.Anything).Return(throughputVec, nil, nil).Once() + res, err = servicer.GetEnodebOffloadState(ctx, &protos.GetEnodebOffloadStateRequest{}) + assert.NoError(t, err) + expectedRes = &protos.GetEnodebOffloadStateResponse{ + EnodebOffloadStates: map[uint32]protos.GetEnodebOffloadStateResponse_EnodebOffloadState{ + 138: protos.GetEnodebOffloadStateResponse_PRIMARY_CONNECTED_AND_SERVING_UES, + }, + } + assert.Equal(t, expectedRes, res) + mockPromClient.AssertExpectations(t) +} + +func reportEnodebState(t *testing.T, ctx context.Context, enodebSerial string, req *lte_models.EnodebState) { + client, err := state.GetStateClient() + assert.NoError(t, err) + + serializedEnodebState, err := serde.Serialize(req, lte.EnodebStateType, serdes.State) + assert.NoError(t, err) + states := []*orc8r_protos.State{ + { + Type: lte.EnodebStateType, + DeviceID: enodebSerial, + Value: serializedEnodebState, + }, + } + _, err = client.ReportStates( + ctx, + &orc8r_protos.ReportStatesRequest{States: states}, + ) + assert.NoError(t, err) +} + +func newDefaultGatewayConfig(mmeCode uint32, mmeRelCap uint32) *lte_models.GatewayCellularConfigs { + return <e_models.GatewayCellularConfigs{ + Ran: <e_models.GatewayRanConfigs{ + Pci: 260, + TransmitEnabled: swag.Bool(true), + }, + Epc: <e_models.GatewayEpcConfigs{ + NatEnabled: swag.Bool(true), + IPBlock: "192.168.128.0/24", + }, + NonEpsService: <e_models.GatewayNonEpsConfigs{ + CsfbMcc: "001", + CsfbMnc: "01", + Lac: swag.Uint32(1), + CsfbRat: swag.Uint32(0), + Arfcn2g: nil, + NonEpsServiceControl: swag.Uint32(0), + }, + DNS: <e_models.GatewayDNSConfigs{ + DhcpServerEnabled: swag.Bool(true), + EnableCaching: swag.Bool(false), + LocalTTL: swag.Int32(0), + }, + HeConfig: <e_models.GatewayHeConfig{}, + Pooling: lte_models.CellularGatewayPoolRecords{ + { + GatewayPoolID: "pool1", + MmeCode: mmeCode, + MmeRelativeCapacity: mmeRelCap, + }, + }, + } +} + +func newDefaultUnmanagedEnodebConfig() *lte_models.EnodebConfig { + ip := strfmt.IPv4("192.168.0.124") + return <e_models.EnodebConfig{ + ConfigType: "UNMANAGED", + UnmanagedConfig: <e_models.UnmanagedEnodebConfiguration{ + CellID: swag.Uint32(138), + Tac: swag.Uint32(1), + IPAddress: &ip, + }, + } +} + +func getDefaultEnodebState(gwID string) *lte_models.EnodebState { + return <e_models.EnodebState{ + MmeConnected: swag.Bool(true), + EnodebConnected: swag.Bool(true), + IPAddress: "10.0.0.1", + ReportingGatewayID: gwID, + EnodebConfigured: swag.Bool(true), + GpsConnected: swag.Bool(true), + GpsLatitude: swag.String("foo"), + GpsLongitude: swag.String("bar"), + OpstateEnabled: swag.Bool(true), + PtpConnected: swag.Bool(true), + RfTxOn: swag.Bool(true), + RfTxDesired: swag.Bool(true), + FsmState: swag.String("abc"), + } +} diff --git a/lte/cloud/go/services/lte/servicers/builder_servicer.go b/lte/cloud/go/services/lte/servicers/builder_servicer.go index 3f32f6cba5ef..87e87dcb05e3 100644 --- a/lte/cloud/go/services/lte/servicers/builder_servicer.go +++ b/lte/cloud/go/services/lte/servicers/builder_servicer.go @@ -98,6 +98,11 @@ func (s *builderServicer) Build(ctx context.Context, request *builder_protos.Bui enbConfigsBySerial := getEnodebConfigsBySerial(cellularNwConfig, cellularGwConfig, enodebs) heConfig := getHEConfig(cellularGwConfig.HeConfig) + mmePoolRecord, mmeGroupID, err := getMMEPoolConfigs(network.ID, cellularGwConfig.Pooling, cellGW, graph) + if err != nil { + return nil, err + } + vals := map[string]proto.Message{ "enodebd": <e_mconfig.EnodebD{ LogLevel: protos.LogLevel_INFO, @@ -126,8 +131,9 @@ func (s *builderServicer) Build(ctx context.Context, request *builder_protos.Bui Mcc: nwEpc.Mcc, Mnc: nwEpc.Mnc, Tac: int32(nwEpc.Tac), - MmeCode: 1, - MmeGid: 1, + MmeCode: int32(mmePoolRecord.MmeCode), + MmeGid: int32(mmeGroupID), + MmeRelativeCapacity: int32(mmePoolRecord.MmeRelativeCapacity), EnableDnsCaching: shouldEnableDNSCaching(cellularGwConfig.DNS), NonEpsServiceControl: nonEPSServiceMconfig.nonEpsServiceControl, CsfbMcc: nonEPSServiceMconfig.csfbMcc, @@ -301,6 +307,33 @@ func getHEConfig(gwConfig *lte_models.GatewayHeConfig) *lte_mconfig.PipelineD_HE EncryptionKey: gwConfig.EncryptionKey, } } + +// getMMEPoolConfigs returns the gateway pool record and a uint32 specifying +// the MME group ID for a given gateway. If a gateway does not exist in a pool, +// default values are returned. +func getMMEPoolConfigs(networkID string, poolingConfig lte_models.CellularGatewayPoolRecords, cellGateway configurator.NetworkEntity, graph configurator.EntityGraph) (*lte_models.CellularGatewayPoolRecord, uint32, error) { + // Currently, having multiple (mme group ID, mme code, mme relative + // capacity) tuples is unsupported. As such, use the first pool record + // to set all of these values. + if len(poolingConfig) == 0 { + return <e_models.CellularGatewayPoolRecord{ + MmeCode: 1, + MmeRelativeCapacity: 10, + }, 1, nil + } + pool, err := graph.GetFirstAncestorOfType(cellGateway, lte.CellularGatewayPoolEntityType) + if err != nil { + return nil, 0, err + } + poolRecord := poolingConfig[0] + cfg, ok := pool.Config.(*lte_models.CellularGatewayPoolConfigs) + if !ok { + err := fmt.Errorf("unable to convert gateway pool config for pool '%s'; pool has invalid config", pool.Key) + return nil, 0, err + } + return poolRecord, cfg.MmeGroupID, nil +} + func getEnodebConfigsBySerial(nwConfig *lte_models.NetworkCellularConfigs, gwConfig *lte_models.GatewayCellularConfigs, enodebs []configurator.NetworkEntity) map[string]*lte_mconfig.EnodebD_EnodebConfig { ret := make(map[string]*lte_mconfig.EnodebD_EnodebConfig, len(enodebs)) for _, ent := range enodebs { diff --git a/lte/cloud/go/services/lte/servicers/builder_servicer_test.go b/lte/cloud/go/services/lte/servicers/builder_servicer_test.go index f9b23a9e2e2c..2dc591eaece7 100644 --- a/lte/cloud/go/services/lte/servicers/builder_servicer_test.go +++ b/lte/cloud/go/services/lte/servicers/builder_servicer_test.go @@ -119,6 +119,7 @@ func TestBuilder_Build(t *testing.T) { Tac: 1, MmeCode: 1, MmeGid: 1, + MmeRelativeCapacity: 10, NonEpsServiceControl: lte_mconfig.MME_NON_EPS_SERVICE_CONTROL_OFF, CsfbMcc: "001", CsfbMnc: "01", @@ -252,6 +253,7 @@ func TestBuilder_Build_NonNat(t *testing.T) { Tac: 1, MmeCode: 1, MmeGid: 1, + MmeRelativeCapacity: 10, NonEpsServiceControl: lte_mconfig.MME_NON_EPS_SERVICE_CONTROL_OFF, CsfbMcc: "001", CsfbMnc: "01", @@ -515,6 +517,7 @@ func TestBuilder_Build_BaseCase(t *testing.T) { Tac: 1, MmeCode: 1, MmeGid: 1, + MmeRelativeCapacity: 10, NonEpsServiceControl: lte_mconfig.MME_NON_EPS_SERVICE_CONTROL_OFF, CsfbMcc: "001", CsfbMnc: "01", @@ -658,6 +661,7 @@ func TestBuilder_BuildInheritedProperties(t *testing.T) { Tac: 1, MmeCode: 1, MmeGid: 1, + MmeRelativeCapacity: 10, NonEpsServiceControl: lte_mconfig.MME_NON_EPS_SERVICE_CONTROL_OFF, CsfbMcc: "001", CsfbMnc: "01", @@ -782,6 +786,7 @@ func TestBuilder_BuildUnmanagedEnbConfig(t *testing.T) { Tac: 1, MmeCode: 1, MmeGid: 1, + MmeRelativeCapacity: 10, NonEpsServiceControl: lte_mconfig.MME_NON_EPS_SERVICE_CONTROL_OFF, CsfbMcc: "001", CsfbMnc: "01", @@ -831,6 +836,132 @@ func TestBuilder_BuildUnmanagedEnbConfig(t *testing.T) { assert.Equal(t, expected, actual) } +func TestBuilder_Build_MMEPool(t *testing.T) { + assert.NoError(t, plugin.RegisterPluginForTests(t, &pluginimpl.BaseOrchestratorPlugin{})) + assert.NoError(t, plugin.RegisterPluginForTests(t, <e_plugin.LteOrchestratorPlugin{})) + lte_test_init.StartTestService(t) + + nw := configurator.Network{ + ID: "n1", + Configs: map[string]interface{}{ + lte.CellularNetworkConfigType: lte_models.NewDefaultTDDNetworkConfig(), + orc8r.DnsdNetworkType: &models.NetworkDNSConfig{ + EnableCaching: swag.Bool(true), + }, + }, + } + gw := configurator.NetworkEntity{ + Type: orc8r.MagmadGatewayType, Key: "gw1", + Associations: []storage.TypeAndKey{ + {Type: lte.CellularGatewayEntityType, Key: "gw1"}, + }, + } + lteGatewayPool := configurator.NetworkEntity{ + Type: lte.CellularGatewayPoolEntityType, Key: "pool1", + Config: <e_models.CellularGatewayPoolConfigs{ + MmeGroupID: 2, + }, + } + lteGatewayConfigs := newDefaultGatewayConfig() + lteGatewayConfigs.Pooling = lte_models.CellularGatewayPoolRecords{ + { + GatewayPoolID: "pool1", + MmeCode: 3, + MmeRelativeCapacity: 255, + }, + } + lteGW := configurator.NetworkEntity{ + Type: lte.CellularGatewayEntityType, Key: "gw1", + Config: lteGatewayConfigs, + Associations: []storage.TypeAndKey{}, + ParentAssociations: []storage.TypeAndKey{gw.GetTypeAndKey(), lteGatewayPool.GetTypeAndKey()}, + } + lteGatewayPool.Associations = []storage.TypeAndKey{lteGW.GetTypeAndKey()} + + graph := configurator.EntityGraph{ + Entities: []configurator.NetworkEntity{lteGatewayPool, lteGW, gw}, + Edges: []configurator.GraphEdge{ + {From: gw.GetTypeAndKey(), To: lteGW.GetTypeAndKey()}, + {From: lteGatewayPool.GetTypeAndKey(), To: lteGW.GetTypeAndKey()}, + }, + } + + expected := map[string]proto.Message{ + "enodebd": <e_mconfig.EnodebD{ + LogLevel: protos.LogLevel_INFO, + Pci: 260, + TddConfig: <e_mconfig.EnodebD_TDDConfig{ + Earfcndl: 44590, + SubframeAssignment: 2, + SpecialSubframePattern: 7, + }, + BandwidthMhz: 20, + AllowEnodebTransmit: true, + Tac: 1, + PlmnidList: "00101", + CsfbRat: lte_mconfig.EnodebD_CSFBRAT_2G, + Arfcn_2G: nil, + EnbConfigsBySerial: nil, + }, + "mobilityd": <e_mconfig.MobilityD{ + LogLevel: protos.LogLevel_INFO, + IpBlock: "192.168.128.0/24", + }, + "mme": <e_mconfig.MME{ + LogLevel: protos.LogLevel_INFO, + Mcc: "001", + Mnc: "01", + Tac: 1, + MmeCode: 3, + MmeGid: 2, + MmeRelativeCapacity: 255, + NonEpsServiceControl: lte_mconfig.MME_NON_EPS_SERVICE_CONTROL_OFF, + CsfbMcc: "001", + CsfbMnc: "01", + Lac: 1, + HssRelayEnabled: false, + CloudSubscriberdbEnabled: false, + AttachedEnodebTacs: nil, + NatEnabled: true, + }, + "pipelined": <e_mconfig.PipelineD{ + LogLevel: protos.LogLevel_INFO, + UeIpBlock: "192.168.128.0/24", + NatEnabled: true, + DefaultRuleId: "", + Services: []lte_mconfig.PipelineD_NetworkServices{ + lte_mconfig.PipelineD_ENFORCEMENT, + }, + HeConfig: <e_mconfig.PipelineD_HEConfig{}, + }, + "subscriberdb": <e_mconfig.SubscriberDB{ + LogLevel: protos.LogLevel_INFO, + LteAuthOp: []byte("\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11"), + LteAuthAmf: []byte("\x80\x00"), + SubProfiles: nil, + HssRelayEnabled: false, + }, + "policydb": <e_mconfig.PolicyDB{ + LogLevel: protos.LogLevel_INFO, + }, + "sessiond": <e_mconfig.SessionD{ + LogLevel: protos.LogLevel_INFO, + GxGyRelayEnabled: false, + WalletExhaustDetection: <e_mconfig.WalletExhaustDetection{ + TerminateOnExhaust: false, + }, + }, + "dnsd": <e_mconfig.DnsD{ + LogLevel: protos.LogLevel_INFO, + DhcpServerEnabled: true, + }, + } + + actual, err := build(&nw, &graph, "gw1") + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} + func build(network *configurator.Network, graph *configurator.EntityGraph, gatewayID string) (map[string]proto.Message, error) { networkProto, err := network.ToProto(serdes.Network) if err != nil { @@ -878,6 +1009,7 @@ func newDefaultGatewayConfig() *lte_models.GatewayCellularConfigs { LocalTTL: swag.Int32(0), }, HeConfig: <e_models.GatewayHeConfig{}, + Pooling: lte_models.CellularGatewayPoolRecords{}, } } diff --git a/lte/cloud/helm/lte-orc8r/templates/lte.deployment.yaml b/lte/cloud/helm/lte-orc8r/templates/lte.deployment.yaml new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lte/gateway/Vagrantfile b/lte/gateway/Vagrantfile index a480d17f3c2b..0ea52b42c670 100644 --- a/lte/gateway/Vagrantfile +++ b/lte/gateway/Vagrantfile @@ -72,11 +72,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| ["--timeout=30"] ansible.verbose = 'v' end - - magma.vm.provision "shell", - run: "always", - inline: "sudo apt-get -y remove oai-asn1c && sudo apt-get -y install oai-asn1c-rel15" - end config.vm.define :magma_trfserver, autostart: false do |magma_trfserver| @@ -257,4 +252,3 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end end - diff --git a/lte/gateway/c/oai/tasks/mme_app/mme_app_context.c b/lte/gateway/c/oai/tasks/mme_app/mme_app_context.c index 325d712175ed..8a91996a185f 100644 --- a/lte/gateway/c/oai/tasks/mme_app/mme_app_context.c +++ b/lte/gateway/c/oai/tasks/mme_app/mme_app_context.c @@ -2412,6 +2412,15 @@ static void mme_app_resume_esm_ebr_timer(ue_mm_context_t* ue_context_p) { default_eps_bearer_activate_t3485_handler( bearer_context_p->esm_ebr_context.args, &ue_context_p->emm_context._imsi64); + } else { // Invoke callback registered for default bearer's + // deactivation procedure + if ((ue_context_p->bearer_contexts[idx]->esm_ebr_context.args) && + (ue_context_p->bearer_contexts[idx]->esm_ebr_context.status == + ESM_EBR_INACTIVE_PENDING)) { + eps_bearer_deactivate_t3495_handler( + ue_context_p->bearer_contexts[idx]->esm_ebr_context.args, + &ue_context_p->emm_context._imsi64); + } } } else { // Invoke callback registered for dedicated bearer's activation @@ -2421,6 +2430,15 @@ static void mme_app_resume_esm_ebr_timer(ue_mm_context_t* ue_context_p) { dedicated_eps_bearer_activate_t3485_handler( bearer_context_p->esm_ebr_context.args, &ue_context_p->emm_context._imsi64); + } else { // Invoke callback registered for dedicated bearer's + // deactivation procedure + if ((ue_context_p->bearer_contexts[idx]->esm_ebr_context.args) && + (ue_context_p->bearer_contexts[idx]->esm_ebr_context.status == + ESM_EBR_INACTIVE_PENDING)) { + eps_bearer_deactivate_t3495_handler( + ue_context_p->bearer_contexts[idx]->esm_ebr_context.args, + &ue_context_p->emm_context._imsi64); + } } } } diff --git a/lte/gateway/c/oai/tasks/nas/esm/EpsBearerContextDeactivation.c b/lte/gateway/c/oai/tasks/nas/esm/EpsBearerContextDeactivation.c index d16bee74f593..19b2e0c560ba 100644 --- a/lte/gateway/c/oai/tasks/nas/esm/EpsBearerContextDeactivation.c +++ b/lte/gateway/c/oai/tasks/nas/esm/EpsBearerContextDeactivation.c @@ -59,8 +59,6 @@ /* Timer handlers */ -static void _eps_bearer_deactivate_t3495_handler(void*, imsi64_t* imsi64); - /* Maximum value of the deactivate EPS bearer context request retransmission counter */ #define EPS_BEARER_DEACTIVATE_COUNTER_MAX 5 @@ -372,7 +370,7 @@ pdn_cid_t esm_proc_eps_bearer_context_deactivate_accept( */ /**************************************************************************** ** ** - ** Name: _eps_bearer_deactivate_t3495_handler() ** + ** Name: eps_bearer_deactivate_t3495_handler() ** ** ** ** Description: T3495 timeout handler ** ** ** @@ -392,7 +390,7 @@ pdn_cid_t esm_proc_eps_bearer_context_deactivate_accept( ** Others: None ** ** ** ***************************************************************************/ -static void _eps_bearer_deactivate_t3495_handler(void* args, imsi64_t* imsi64) { +void eps_bearer_deactivate_t3495_handler(void* args, imsi64_t* imsi64) { OAILOG_FUNC_IN(LOG_NAS_ESM); int rc; bool delete_default_bearer = false; @@ -552,7 +550,7 @@ static int _eps_bearer_deactivate( */ rc = esm_ebr_start_timer( emm_context_p, ebi, msg_dup, mme_config.nas_config.t3495_sec, - _eps_bearer_deactivate_t3495_handler); + eps_bearer_deactivate_t3495_handler); } bdestroy_wrapper(&msg_dup); OAILOG_FUNC_RETURN(LOG_NAS_ESM, rc); diff --git a/lte/gateway/c/oai/tasks/nas/esm/esm_ebr_context.h b/lte/gateway/c/oai/tasks/nas/esm/esm_ebr_context.h index bc3d04703aa7..f3df1d7371df 100644 --- a/lte/gateway/c/oai/tasks/nas/esm/esm_ebr_context.h +++ b/lte/gateway/c/oai/tasks/nas/esm/esm_ebr_context.h @@ -77,4 +77,5 @@ void default_eps_bearer_activate_t3485_handler(void* args, imsi64_t* imsi64); void dedicated_eps_bearer_activate_t3485_handler(void* args, imsi64_t* imsi64); +void eps_bearer_deactivate_t3495_handler(void*, imsi64_t* imsi64); #endif /* ESM_EBR_CONTEXT_SEEN */ diff --git a/lte/gateway/c/session_manager/LocalEnforcer.cpp b/lte/gateway/c/session_manager/LocalEnforcer.cpp index b95f166e18e3..9992865a3e64 100644 --- a/lte/gateway/c/session_manager/LocalEnforcer.cpp +++ b/lte/gateway/c/session_manager/LocalEnforcer.cpp @@ -2196,8 +2196,8 @@ static SubscriberQuotaUpdate make_subscriber_quota_update( UpdateRequestsBySession::UpdateRequestsBySession( const UpdateSessionRequest& request) { for (const auto& charging_request : request.updates()) { - const auto id = - ImsiAndSessionID(charging_request.sid(), charging_request.session_id()); + const auto imsi = charging_request.common_context().sid().id(); + const auto id = ImsiAndSessionID(imsi, charging_request.session_id()); requests_by_id[id].charging_requests.push_back(charging_request); } for (const auto& monitor_request : request.usage_monitors()) { diff --git a/lte/gateway/c/session_manager/RestartHandler.cpp b/lte/gateway/c/session_manager/RestartHandler.cpp index a4f235da2082..8cc8dc7b50f2 100644 --- a/lte/gateway/c/session_manager/RestartHandler.cpp +++ b/lte/gateway/c/session_manager/RestartHandler.cpp @@ -143,7 +143,8 @@ bool RestartHandler::populate_sessions_to_terminate_with_retries() { void RestartHandler::terminate_previous_session( const std::string& sid, const std::string& session_id) { SessionTerminateRequest term_req; - term_req.set_sid(sid); + term_req.set_sid(sid); // TODO Deprecate + term_req.mutable_common_context()->mutable_sid()->set_id(sid); term_req.set_session_id(session_id); std::promise termination_res; std::future termination_future = termination_res.get_future(); diff --git a/lte/gateway/c/session_manager/SessionReporter.cpp b/lte/gateway/c/session_manager/SessionReporter.cpp index d081ba50793b..35d7f88b2d04 100644 --- a/lte/gateway/c/session_manager/SessionReporter.cpp +++ b/lte/gateway/c/session_manager/SessionReporter.cpp @@ -41,11 +41,11 @@ SessionReporter::get_terminate_logging_cb( if (!status.ok()) { MLOG(MERROR) << "Failed to terminate session in controller for " "subscriber " - << request.sid() << ": " << status.error_message(); + << request.session_id() << ": " << status.error_message(); } else { MLOG(MDEBUG) << "Termination successful in controller for " "subscriber " - << request.sid(); + << request.session_id(); } }; } diff --git a/lte/gateway/c/session_manager/SessionStore.cpp b/lte/gateway/c/session_manager/SessionStore.cpp index cce55452a63e..9a86dfa18699 100644 --- a/lte/gateway/c/session_manager/SessionStore.cpp +++ b/lte/gateway/c/session_manager/SessionStore.cpp @@ -47,7 +47,7 @@ void SessionStore::set_and_save_reporting_flag( for (const CreditUsageUpdate& credit_update : update_session_request.updates()) { - const std::string imsi = credit_update.sid(); + const std::string imsi = credit_update.common_context().sid().id(); const std::string session_id = credit_update.session_id(); const CreditKey& ckey = credit_update.usage().charging_key(); const std::string mkey = credit_update.usage().monitoring_key(); diff --git a/lte/gateway/c/session_manager/test/Matchers.h b/lte/gateway/c/session_manager/test/Matchers.h index 8e619f906e72..a1f25616cfb2 100644 --- a/lte/gateway/c/session_manager/test/Matchers.h +++ b/lte/gateway/c/session_manager/test/Matchers.h @@ -80,7 +80,8 @@ MATCHER_P(CheckCoreRequest, expected_request, "") { MATCHER_P3(CheckTerminateRequestCount, imsi, monitorCount, chargingCount, "") { auto req = static_cast(arg); - return req.sid() == imsi && req.credit_usages().size() == chargingCount && + return req.common_context().sid().id() == imsi && + req.credit_usages().size() == chargingCount && req.monitor_usages().size() == monitorCount; } @@ -184,14 +185,15 @@ MATCHER_P(CheckSingleUpdate, expected_update, "") { update.usage().type() == expected_update.usage().type() && update.usage().bytes_tx() == expected_update.usage().bytes_tx() && update.usage().bytes_rx() == expected_update.usage().bytes_rx() && - update.sid() == expected_update.sid() && + update.common_context().sid().id() == + expected_update.common_context().sid().id() && update.usage().charging_key() == expected_update.usage().charging_key(); return val; } MATCHER_P(CheckTerminate, imsi, "") { auto request = static_cast(arg); - return request->sid() == imsi; + return request->common_context().sid().id() == imsi; } MATCHER_P4(CheckActivateFlows, imsi, rule_count, ipv4, ipv6, "") { diff --git a/lte/gateway/c/session_manager/test/ProtobufCreators.cpp b/lte/gateway/c/session_manager/test/ProtobufCreators.cpp index f72ef9edf430..0d429f52ba3c 100644 --- a/lte/gateway/c/session_manager/test/ProtobufCreators.cpp +++ b/lte/gateway/c/session_manager/test/ProtobufCreators.cpp @@ -244,7 +244,7 @@ void create_usage_update( uint64_t bytes_tx, CreditUsage::UpdateType type, CreditUsageUpdate* update) { auto usage = update->mutable_usage(); - update->set_sid(imsi); + update->mutable_common_context()->mutable_sid()->set_id(imsi); usage->set_charging_key(charging_key); usage->set_bytes_rx(bytes_rx); usage->set_bytes_tx(bytes_tx); diff --git a/lte/gateway/c/session_manager/test/test_local_enforcer.cpp b/lte/gateway/c/session_manager/test/test_local_enforcer.cpp index f2034014be31..8e6124ca2734 100644 --- a/lte/gateway/c/session_manager/test/test_local_enforcer.cpp +++ b/lte/gateway/c/session_manager/test/test_local_enforcer.cpp @@ -58,25 +58,27 @@ class LocalEnforcerTest : public ::testing::Test { evb = folly::EventBaseManager::get()->getEventBase(); local_enforcer->attachEventBase(evb); session_map = SessionMap{}; - initialize_lte_test_config(); + test_cfg_ = get_default_config(""); } virtual void TearDown() { folly::EventBaseManager::get()->clearEventBase(); } - void initialize_lte_test_config() { - test_cfg_.common_context = - build_common_context(IMSI1, IP1, IPv6_1, APN1, MSISDN, TGPP_LTE); + void run_evb() { + evb->runAfterDelay([this]() { local_enforcer->stop(); }, 100); + local_enforcer->start(); + } + + SessionConfig get_default_config(const std::string& imsi) { + SessionConfig cfg; + cfg.common_context = + build_common_context(imsi, IP1, IPv6_1, APN1, MSISDN, TGPP_LTE); QosInformationRequest qos_info; qos_info.set_apn_ambr_dl(32); qos_info.set_apn_ambr_dl(64); const auto& lte_context = build_lte_context(IP2, "", "", "", "", 0, &qos_info); - test_cfg_.rat_specific_context.mutable_lte_context()->CopyFrom(lte_context); - } - - void run_evb() { - evb->runAfterDelay([this]() { local_enforcer->stop(); }, 100); - local_enforcer->start(); + cfg.rat_specific_context.mutable_lte_context()->CopyFrom(lte_context); + return cfg; } void insert_static_rule( @@ -359,7 +361,7 @@ TEST_F(LocalEnforcerTest, test_aggregate_records_for_termination) { create_credit_update_response( IMSI1, SESSION_ID_1, 2, 1024, response.mutable_credits()->Add()); local_enforcer->init_session_credit( - session_map, IMSI1, SESSION_ID_1, test_cfg_, response); + session_map, IMSI1, SESSION_ID_1, get_default_config(IMSI1), response); insert_static_rule(1, "", "rule1"); insert_static_rule(1, "", "rule2"); @@ -553,11 +555,11 @@ TEST_F(LocalEnforcerTest, test_terminate_credit) { session_map = session_store->read_sessions(SessionRead{IMSI1}); local_enforcer->init_session_credit( - session_map, IMSI1, SESSION_ID_1, test_cfg_, response); + session_map, IMSI1, SESSION_ID_1, get_default_config(IMSI1), response); session_store->create_sessions(IMSI1, std::move(session_map[IMSI1])); session_map = session_store->read_sessions(SessionRead{IMSI2}); local_enforcer->init_session_credit( - session_map, IMSI2, "4321", test_cfg_, response2); + session_map, IMSI2, SESSION_ID_2, get_default_config(IMSI2), response2); session_store->create_sessions(IMSI2, std::move(session_map[IMSI2])); session_map = session_store->read_sessions(SessionRead{IMSI1}); @@ -589,6 +591,7 @@ TEST_F(LocalEnforcerTest, test_terminate_credit) { } TEST_F(LocalEnforcerTest, test_terminate_credit_during_reporting) { + const auto cfg1 = get_default_config(IMSI1); CreateSessionResponse response; create_credit_update_response( IMSI1, SESSION_ID_1, 1, 3072, response.mutable_credits()->Add()); @@ -598,7 +601,7 @@ TEST_F(LocalEnforcerTest, test_terminate_credit_during_reporting) { IMSI1, SESSION_ID_1, "m1", MonitoringLevel::PCC_RULE_LEVEL, 1024, response.mutable_usage_monitors()->Add()); local_enforcer->init_session_credit( - session_map, IMSI1, SESSION_ID_1, test_cfg_, response); + session_map, IMSI1, SESSION_ID_1, cfg1, response); insert_static_rule(1, "", "rule1"); insert_static_rule(2, "", "rule2"); @@ -606,7 +609,7 @@ TEST_F(LocalEnforcerTest, test_terminate_credit_during_reporting) { RuleRecordTable table; auto record_list = table.mutable_records(); create_rule_record( - IMSI1, test_cfg_.common_context.ue_ipv4(), "rule1", 1024, 2048, + IMSI1, cfg1.common_context.ue_ipv4(), "rule1", 1024, 2048, record_list->Add()); auto update = SessionStore::get_default_session_update(session_map); local_enforcer->aggregate_records(session_map, table, update); @@ -643,11 +646,11 @@ TEST_F(LocalEnforcerTest, test_terminate_credit_during_reporting) { RuleRecordTable only_drop_rule_table; record_list = only_drop_rule_table.mutable_records(); create_rule_record( - IMSI1, test_cfg_.common_context.ue_ipv4(), "rule1", 0, 0, 1000, 2000, + IMSI1, cfg1.common_context.ue_ipv4(), "rule1", 0, 0, 1000, 2000, record_list->Add()); create_rule_record( - IMSI1, test_cfg_.common_context.ue_ipv4(), - "internal_default_drop_flow_rule", 0, 0, record_list->Add()); + IMSI1, cfg1.common_context.ue_ipv4(), "internal_default_drop_flow_rule", + 0, 0, record_list->Add()); local_enforcer->aggregate_records(session_map, only_drop_rule_table, update); run_evb(); @@ -878,16 +881,18 @@ TEST_F(LocalEnforcerTest, test_all) { insert_static_rule(2, "", "rule3"); // insert initial session credit + const auto cfg1 = get_default_config(IMSI1); + const auto cfg2 = get_default_config(IMSI2); CreateSessionResponse response; create_credit_update_response( IMSI1, SESSION_ID_1, 1, 1024, response.mutable_credits()->Add()); local_enforcer->init_session_credit( - session_map, IMSI1, SESSION_ID_1, test_cfg_, response); + session_map, IMSI1, SESSION_ID_1, cfg1, response); CreateSessionResponse response2; create_credit_update_response( IMSI2, SESSION_ID_2, 2, 2048, response2.mutable_credits()->Add()); local_enforcer->init_session_credit( - session_map, IMSI2, SESSION_ID_2, test_cfg_, response2); + session_map, IMSI2, SESSION_ID_2, cfg2, response2); assert_charging_credit( session_map, IMSI1, SESSION_ID_1, ALLOWED_TOTAL, {{1, 1024}}); @@ -898,13 +903,12 @@ TEST_F(LocalEnforcerTest, test_all) { RuleRecordTable table; auto record_list = table.mutable_records(); create_rule_record( - IMSI1, test_cfg_.common_context.ue_ipv4(), "rule1", 10, 20, + IMSI1, cfg1.common_context.ue_ipv4(), "rule1", 10, 20, record_list->Add()); create_rule_record( - IMSI1, test_cfg_.common_context.ue_ipv4(), "rule2", 5, 15, - record_list->Add()); + IMSI1, cfg1.common_context.ue_ipv4(), "rule2", 5, 15, record_list->Add()); create_rule_record( - IMSI2, test_cfg_.common_context.ue_ipv4(), "rule3", 1024, 1024, + IMSI2, cfg2.common_context.ue_ipv4(), "rule3", 1024, 1024, record_list->Add()); auto update = SessionStore::get_default_session_update(session_map); local_enforcer->aggregate_records(session_map, table, update); @@ -1121,6 +1125,7 @@ TEST_F(LocalEnforcerTest, test_reauth_with_redirected_suspended_credit) { // insert initial suspended and redirected credit CreateSessionResponse response; auto credits = response.mutable_credits(); + test_cfg_.common_context.mutable_sid()->set_id(IMSI1); create_credit_update_response_with_error( IMSI1, SESSION_ID_1, 1, false, DIAMETER_CREDIT_LIMIT_REACHED, ChargingCredit_FinalAction_REDIRECT, "12.7.7.4", "", credits->Add()); @@ -1158,7 +1163,7 @@ TEST_F(LocalEnforcerTest, test_reauth_with_redirected_suspended_credit) { local_enforcer->collect_updates(session_map, actions, update); local_enforcer->execute_actions(session_map, actions, update); EXPECT_EQ(update_req.updates_size(), 1); - EXPECT_EQ(update_req.updates(0).sid(), IMSI1); + EXPECT_EQ(update_req.updates(0).common_context().sid().id(), IMSI1); EXPECT_EQ(update_req.updates(0).usage().type(), CreditUsage::REAUTH_REQUIRED); // 3- SUSPENSION IS CLEARED @@ -1185,7 +1190,7 @@ TEST_F(LocalEnforcerTest, test_re_auth) { insert_static_rule(1, "", "rule1"); CreateSessionResponse response; local_enforcer->init_session_credit( - session_map, IMSI1, SESSION_ID_1, test_cfg_, response); + session_map, IMSI1, SESSION_ID_1, get_default_config(IMSI1), response); ChargingReAuthRequest reauth; reauth.set_sid(IMSI1); @@ -1202,7 +1207,7 @@ TEST_F(LocalEnforcerTest, test_re_auth) { local_enforcer->collect_updates(session_map, actions, update); local_enforcer->execute_actions(session_map, actions, update); EXPECT_EQ(update_req.updates_size(), 1); - EXPECT_EQ(update_req.updates(0).sid(), IMSI1); + EXPECT_EQ(update_req.updates(0).common_context().sid().id(), IMSI1); EXPECT_EQ(update_req.updates(0).usage().type(), CreditUsage::REAUTH_REQUIRED); // Give credit after re-auth @@ -2549,6 +2554,7 @@ TEST_F(LocalEnforcerTest, test_final_unit_redirect_activation_and_termination) { TEST_F(LocalEnforcerTest, test_final_unit_activation_and_canceling) { CreateSessionResponse response; + const auto cfg1 = get_default_config(IMSI1); create_credit_update_response( IMSI1, SESSION_ID_1, 1, 1024, ChargingCredit_FinalAction_RESTRICT_ACCESS, "", "rule1", response.mutable_credits()->Add()); @@ -2565,9 +2571,9 @@ TEST_F(LocalEnforcerTest, test_final_unit_activation_and_canceling) { insert_static_rule(1, "", "rule1"); insert_static_rule(1, "", "rule3"); - auto& ip_addr = test_cfg_.common_context.ue_ipv4(); - auto& ipv6_addr = test_cfg_.common_context.ue_ipv6(); - const auto& msisdn = test_cfg_.common_context.msisdn(); + auto& ip_addr = cfg1.common_context.ue_ipv4(); + auto& ipv6_addr = cfg1.common_context.ue_ipv6(); + const auto& msisdn = cfg1.common_context.msisdn(); // The activation for the static rules (rule1,rule3) and dynamic rule (rule2) EXPECT_CALL( *pipelined_client, activate_flows_for_rules( @@ -2577,7 +2583,7 @@ TEST_F(LocalEnforcerTest, test_final_unit_activation_and_canceling) { .WillOnce(testing::Return(true)); local_enforcer->init_session_credit( - session_map, IMSI1, SESSION_ID_1, test_cfg_, response); + session_map, IMSI1, SESSION_ID_1, cfg1, response); // Insert record and aggregate over them RuleRecordTable table; @@ -2625,7 +2631,7 @@ TEST_F(LocalEnforcerTest, test_final_unit_activation_and_canceling) { local_enforcer->collect_updates(session_map, actions, update); local_enforcer->execute_actions(session_map, actions, update); EXPECT_EQ(update_req.updates_size(), 1); - EXPECT_EQ(update_req.updates(0).sid(), IMSI1); + EXPECT_EQ(update_req.updates(0).common_context().sid().id(), IMSI1); EXPECT_EQ(update_req.updates(0).usage().type(), CreditUsage::REAUTH_REQUIRED); // Give credit after ReAuth @@ -2796,4 +2802,4 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } -} // namespace magma +} // namespace magma \ No newline at end of file diff --git a/lte/gateway/configs/ctraced.yml b/lte/gateway/configs/ctraced.yml index e4ef28aba475..79d383fbd944 100644 --- a/lte/gateway/configs/ctraced.yml +++ b/lte/gateway/configs/ctraced.yml @@ -11,3 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. log_level: INFO + +# Directory for temporary storage of packet captures +trace_directory: /var/opt/magma/tmp/trace + +# Interface to capture on +trace_interfaces: + - eth0 + +# Options available: +# - tshark +# - tcpdump +# tshark has more capabilities - see command_builder.py +trace_tool: tshark diff --git a/lte/gateway/configs/health.yml b/lte/gateway/configs/health.yml index 38f0ffd7232e..93dbe745c4f6 100644 --- a/lte/gateway/configs/health.yml +++ b/lte/gateway/configs/health.yml @@ -21,7 +21,7 @@ state_recovery: - pipelined # Number of restarts of services_check that triggers recovery process - restart_threshold: 3 + restart_threshold: 2 interval_check_mins: 1 # Destination path to save redis RDB temp snapshots snapshots_dir: /tmp/redis_snapshots diff --git a/lte/gateway/configs/pipelined.yml_prod b/lte/gateway/configs/pipelined.yml_prod index 85f6ab1ca79a..c742b3a5ab63 100644 --- a/lte/gateway/configs/pipelined.yml_prod +++ b/lte/gateway/configs/pipelined.yml_prod @@ -33,6 +33,7 @@ setup_type: LTE # after these static services. static_services: [ 'arpd', + 'proxy', 'access_control', 'startup_flows', ] diff --git a/lte/gateway/configs/state.yml b/lte/gateway/configs/state.yml index ab19cb36d343..469963961652 100644 --- a/lte/gateway/configs/state.yml +++ b/lte/gateway/configs/state.yml @@ -13,6 +13,8 @@ # log_level is set in mconfig. it can be overridden here +sync_interval: 60 + #state_protos: # - proto_file: - file to load proto from # proto_msg: - msg to load from proto file diff --git a/lte/gateway/python/integ_tests/s1aptests/s1ap_utils.py b/lte/gateway/python/integ_tests/s1aptests/s1ap_utils.py index 3e34fff9f3f7..2e4f46628cf4 100644 --- a/lte/gateway/python/integ_tests/s1aptests/s1ap_utils.py +++ b/lte/gateway/python/integ_tests/s1aptests/s1ap_utils.py @@ -178,7 +178,7 @@ def get_response(self): return self._msg.get(True) def populate_pco( - self, protCfgOpts_pr, pcscf_addr_type=None, dns_ipv6_addr=False + self, protCfgOpts_pr, pcscf_addr_type=None, dns_ipv6_addr=False ): """ Populates the PCO values. @@ -235,17 +235,17 @@ def populate_pco( ].cid = S1ApUtil.PROT_CFG_CID_DNS_SERVER_IPV6_ADDR_REQUEST def attach( - self, - ue_id, - attach_type, - resp_type, - resp_msg_type, - sec_ctxt=s1ap_types.TFW_CREATE_NEW_SECURITY_CONTEXT, - id_type=s1ap_types.TFW_MID_TYPE_IMSI, - eps_type=s1ap_types.TFW_EPS_ATTACH_TYPE_EPS_ATTACH, - pdn_type=1, - pcscf_addr_type=None, - dns_ipv6_addr=False, + self, + ue_id, + attach_type, + resp_type, + resp_msg_type, + sec_ctxt=s1ap_types.TFW_CREATE_NEW_SECURITY_CONTEXT, + id_type=s1ap_types.TFW_MID_TYPE_IMSI, + eps_type=s1ap_types.TFW_EPS_ATTACH_TYPE_EPS_ATTACH, + pdn_type=1, + pcscf_addr_type=None, + dns_ipv6_addr=False, ): """ Given a UE issue the attach request of specified type @@ -291,8 +291,8 @@ def attach( elif s1ap_types.tfwCmd.UE_ATTACH_ACCEPT_IND.value == response.msg_type: context_setup = self.get_response() assert ( - context_setup.msg_type - == s1ap_types.tfwCmd.INT_CTX_SETUP_IND.value + context_setup.msg_type + == s1ap_types.tfwCmd.INT_CTX_SETUP_IND.value ) logging.debug( @@ -335,14 +335,14 @@ def detach(self, ue_id, reason_type, wait_for_s1_ctxt_release=True): detach_req.ue_Id = ue_id detach_req.ueDetType = reason_type assert ( - self.issue_cmd(s1ap_types.tfwCmd.UE_DETACH_REQUEST, detach_req) - == 0 + self.issue_cmd(s1ap_types.tfwCmd.UE_DETACH_REQUEST, detach_req) + == 0 ) if reason_type == s1ap_types.ueDetachType_t.UE_NORMAL_DETACH.value: response = self.get_response() assert ( - s1ap_types.tfwCmd.UE_DETACH_ACCEPT_IND.value - == response.msg_type + s1ap_types.tfwCmd.UE_DETACH_ACCEPT_IND.value + == response.msg_type ) # Now wait for the context release response @@ -378,8 +378,8 @@ def _verify_dl_flow(self, dl_flow_rules=None): for item in value: for flow in item: if ( - flow["direction"] == FlowMatch.DOWNLINK - and key_to_be_matched in flow + flow["direction"] == FlowMatch.DOWNLINK + and key_to_be_matched in flow ): total_num_dl_flows_to_be_verified += 1 total_dl_ovs_flows_created = get_flows( @@ -394,16 +394,16 @@ def _verify_dl_flow(self, dl_flow_rules=None): }, ) assert ( - len(total_dl_ovs_flows_created) - == total_num_dl_flows_to_be_verified + len(total_dl_ovs_flows_created) + == total_num_dl_flows_to_be_verified ) # Now verify the rules for every flow for item in value: for flow in item: if ( - flow["direction"] == FlowMatch.DOWNLINK - and key_to_be_matched in flow + flow["direction"] == FlowMatch.DOWNLINK + and key_to_be_matched in flow ): ip_src_addr = flow[key_to_be_matched] ip_src = "ipv4_src" if key.version == 4 else "ipv6_src" @@ -432,7 +432,7 @@ def _verify_dl_flow(self, dl_flow_rules=None): 5 ) # sleep for 5 seconds before retrying assert ( - len(downlink_flows) >= num_dl_flows + len(downlink_flows) >= num_dl_flows ), "Downlink flow missing for UE" assert downlink_flows[0]["match"][ip_dst] == ue_ip_addr actions = downlink_flows[0]["instructions"][0][ @@ -459,10 +459,10 @@ def verify_flow_rules(self, num_ul_flows, dl_flow_rules=None): uplink_flows = get_flows( self.datapath, { - "table_id": self.SPGW_TABLE, - "match": { - "in_port": GTP_PORT, - } + "table_id": self.SPGW_TABLE, + "match": { + "in_port": GTP_PORT, + } }, ) if len(uplink_flows) == num_ul_flows: @@ -499,7 +499,7 @@ def verify_paging_flow_rules(self, ip_list): break time.sleep(5) # sleep for 5 seconds before retrying assert ( - len(paging_flows) == num_paging_flows_to_be_verified + len(paging_flows) == num_paging_flows_to_be_verified ), "Paging flow missing for UE" # TODO - Verify that the action is to send to controller @@ -621,7 +621,7 @@ def __init__(self, magmad_client): def exec_command(self, command): """ - Run a command remotly on magma_dev VM. + Run a command remotely on magma_dev VM. Args: command: command (str) to be executed on remote host @@ -640,7 +640,7 @@ def exec_command(self, command): def exec_command_output(self, command): """ - Run a command remotly on magma_dev VM. + Run a command remotely on magma_dev VM. Args: command: command (str) to be executed on remote host @@ -674,7 +674,7 @@ def config_stateless(self, cmd): ret_code = self.exec_command( magtivate_cmd + " && " + venvsudo_cmd + " python3 " + config_stateless_script + " " + cmd.name.lower() - ) + ) if ret_code == 0: print("AGW is stateless") @@ -685,6 +685,19 @@ def config_stateless(self, cmd): else: print("Unknown command") + def corrupt_agw_state(self, key: str): + """ + Corrupts data on redis of stateless AGW + Args: + key: + + """ + magtivate_cmd = "source /home/vagrant/build/python/bin/activate" + state_corrupt_cmd = "state_cli.py corrupt %s" % key.lower() + + self.exec_command(magtivate_cmd + " && " + state_corrupt_cmd) + print("Corrupted %s on redis" % key) + def restart_all_services(self): """ Restart all magma services on magma_dev VM @@ -726,29 +739,29 @@ def update_mme_config_for_sanity(self, cmd): print("MME configuration is updated successfully") elif ret_code == 1: assert False, ( - "Failed to " - + action - + " MME configuration. Error: Invalid command" + "Failed to " + + action + + " MME configuration. Error: Invalid command" ) elif ret_code == 2: assert False, ( - "Failed to " - + action - + " MME configuration. Error: MME configuration file is " - + "missing" + "Failed to " + + action + + " MME configuration. Error: MME configuration file is " + + "missing" ) elif ret_code == 3: assert False, ( - "Failed to " - + action - + " MME configuration. Error: MME configuration's backup file " - + "is missing" + "Failed to " + + action + + " MME configuration. Error: MME configuration's backup file " + + "is missing" ) else: assert False, ( - "Failed to " - + action - + " MME configuration. Error: Unknown error" + "Failed to " + + action + + " MME configuration. Error: Unknown error" ) def config_apn_correction(self, cmd): @@ -776,6 +789,13 @@ def config_apn_correction(self, cmd): else: print("APN Correction failed") + def restart_mme_and_wait(self): + print("Restarting mme service on gateway") + self.restart_services(["mme"]) + print("Waiting for mme to restart. 20 sec") + time.sleep(20) + + class MobilityUtil(object): """ Utility wrapper for interacting with mobilityd """ @@ -988,7 +1008,7 @@ def create_bearer(self, imsi, lbi, qci_val=1): self._stub.CreateBearer(req) def create_bearer_ipv4v6( - self, imsi, lbi, qci_val=1, ipv4=False, ipv6=False + self, imsi, lbi, qci_val=1, ipv4=False, ipv6=False ): """ Sends a CreateBearer Request with ipv4/ipv6/ipv4v6 packet """ @@ -1267,14 +1287,16 @@ def create_AbortSessionRequest(self, imsi: str) -> AbortSessionResult: class GTPBridgeUtils: def __init__(self): self.magma_utils = MagmadUtil(None) - ret = self.magma_utils.exec_command_output("sudo grep ovs_multi_tunnel /etc/magma/spgw.yml") + ret = self.magma_utils.exec_command_output( + "sudo grep ovs_multi_tunnel /etc/magma/spgw.yml") if "false" in ret: self.gtp_port_name = "gtp0" else: self.gtp_port_name = "g_8d3ca8c0" def get_gtp_port_no(self) -> Optional[int]: - output = self.magma_utils.exec_command_output("sudo ovsdb-client dump Interface name ofport") + output = self.magma_utils.exec_command_output( + "sudo ovsdb-client dump Interface name ofport") for line in output.split('\n'): if self.gtp_port_name in line: port_info = line.split() diff --git a/lte/gateway/python/integ_tests/s1aptests/test_attach_detach_with_corrupt_stateless_mme.py b/lte/gateway/python/integ_tests/s1aptests/test_attach_detach_with_corrupt_stateless_mme.py new file mode 100644 index 000000000000..595b6dc98e59 --- /dev/null +++ b/lte/gateway/python/integ_tests/s1aptests/test_attach_detach_with_corrupt_stateless_mme.py @@ -0,0 +1,100 @@ +""" +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import time +import unittest + +import s1ap_types +import s1ap_wrapper +from s1ap_utils import MagmadUtil + + +class TestAttachDetachWithCorruptStatelessMME(unittest.TestCase): + + def setUp(self): + self._s1ap_wrapper = s1ap_wrapper.TestWrapper( + stateless_mode=MagmadUtil.stateless_cmds.ENABLE) + + def tearDown(self): + self._s1ap_wrapper.cleanup() + + def test_attach_detach_with_corrupt_stateless_mme(self): + """ + Basic attach/detach test with two UEs, + with purpose of validating corruption of MME state and recovery + """ + detach_type = s1ap_types.ueDetachType_t.UE_NORMAL_DETACH.value + wait_for_s1 = True + self._s1ap_wrapper.configUEDevice(1) + + services_state_dict = { + 'mme': 'mme_nas_state', + } + + req = self._s1ap_wrapper.ue_req + print("************************* Running End to End attach for ", + "UE id ", req.ue_id) + + for s in services_state_dict: + # Now actually complete the attach + self._s1ap_wrapper._s1_util.attach( + req.ue_id, s1ap_types.tfwCmd.UE_END_TO_END_ATTACH_REQUEST, + s1ap_types.tfwCmd.UE_ATTACH_ACCEPT_IND, + s1ap_types.ueAttachAccept_t) + + # Wait on EMM Information from MME + self._s1ap_wrapper._s1_util.receive_emm_info() + + print("************************* Corrupting %s state" % s) + self._s1ap_wrapper.magmad_util.corrupt_agw_state( + services_state_dict[s]) + + print("************************* Restarting %s service on" % s) + self._s1ap_wrapper.magmad_util.restart_services(s) + + for j in range(100): + print("Waiting for", j, "seconds") + time.sleep(1) + + # Re-establish S1 connection between eNB and MME + self._s1ap_wrapper._s1setup() + + print( + "************************* Re-running End to End attach for ", + "UE id ", + req.ue_id, + ) + + # Repeat the attach + self._s1ap_wrapper._s1_util.attach( + req.ue_id, + s1ap_types.tfwCmd.UE_END_TO_END_ATTACH_REQUEST, + s1ap_types.tfwCmd.UE_ATTACH_ACCEPT_IND, + s1ap_types.ueAttachAccept_t, + ) + + # Wait on EMM Information from MME + self._s1ap_wrapper._s1_util.receive_emm_info() + + # Now detach the UE + print( + "************************* Running UE detach for UE id ", + req.ue_id, + ) + self._s1ap_wrapper.s1_util.detach( + req.ue_id, detach_type, wait_for_s1 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/lte/gateway/python/integ_tests/s1aptests/test_attach_with_multiple_mme_restarts.py b/lte/gateway/python/integ_tests/s1aptests/test_attach_with_multiple_mme_restarts.py new file mode 100644 index 000000000000..ba7fb388564a --- /dev/null +++ b/lte/gateway/python/integ_tests/s1aptests/test_attach_with_multiple_mme_restarts.py @@ -0,0 +1,123 @@ +""" +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import time +import unittest + +import orc8r.protos.metricsd_pb2 as metricsd +import s1ap_types +import s1ap_wrapper +from s1ap_utils import MagmadUtil + +class TestAttachWithMultipleMmeRestarts(unittest.TestCase): + + def setUp(self): + self._s1ap_wrapper = s1ap_wrapper.TestWrapper( + stateless_mode=MagmadUtil.stateless_cmds.ENABLE) + + def tearDown(self): + self._s1ap_wrapper.cleanup() + + def test_attach_with_multiple_mme_restarts(self): + num_ues = 1 + self._s1ap_wrapper.configUEDevice(num_ues) + print("************************* Sending Attach Request for ue-id : 1") + attach_req = s1ap_types.ueAttachRequest_t() + attach_req.ue_Id = 1 + sec_ctxt = s1ap_types.TFW_CREATE_NEW_SECURITY_CONTEXT + id_type = s1ap_types.TFW_MID_TYPE_IMSI + eps_type = s1ap_types.TFW_EPS_ATTACH_TYPE_EPS_ATTACH + attach_req.mIdType = id_type + attach_req.epsAttachType = eps_type + attach_req.useOldSecCtxt = sec_ctxt + + self._s1ap_wrapper._s1_util.issue_cmd( + s1ap_types.tfwCmd.UE_ATTACH_REQUEST, attach_req + ) + + response = self._s1ap_wrapper.s1_util.get_response() + self.assertEqual( + response.msg_type, s1ap_types.tfwCmd.UE_AUTH_REQ_IND.value + ) + print("************************* Received UE_AUTH_REQ_IND") + + # Try consecutive mme restarts + self._s1ap_wrapper.magmad_util.restart_mme_and_wait() + self._s1ap_wrapper.magmad_util.restart_mme_and_wait() + + auth_res = s1ap_types.ueAuthResp_t() + auth_res.ue_Id = 1 + sqn_recvd = s1ap_types.ueSqnRcvd_t() + sqn_recvd.pres = 0 + auth_res.sqnRcvd = sqn_recvd + print("************************* Sending Auth Response ue-id", auth_res.ue_Id) + self._s1ap_wrapper._s1_util.issue_cmd( + s1ap_types.tfwCmd.UE_AUTH_RESP, auth_res + ) + + response = self._s1ap_wrapper.s1_util.get_response() + self.assertEqual( + response.msg_type, s1ap_types.tfwCmd.UE_SEC_MOD_CMD_IND.value + ) + print("************************* Received UE_SEC_MOD_CMD_IND") + + self._s1ap_wrapper.magmad_util.restart_mme_and_wait() + + print("************************* Sending UE_SEC_MOD_COMPLETE") + sec_mode_complete = s1ap_types.ueSecModeComplete_t() + sec_mode_complete.ue_Id = 1 + self._s1ap_wrapper._s1_util.issue_cmd( + s1ap_types.tfwCmd.UE_SEC_MOD_COMPLETE, sec_mode_complete + ) + + response = self._s1ap_wrapper.s1_util.get_response() + self.assertEqual( + response.msg_type, s1ap_types.tfwCmd.INT_CTX_SETUP_IND.value + ) + print("************************* Received INT_CTX_SETUP_IND") + + response = self._s1ap_wrapper.s1_util.get_response() + self.assertEqual( + response.msg_type, s1ap_types.tfwCmd.UE_ATTACH_ACCEPT_IND.value + ) + print("************************* Received UE_ATTACH_ACCEPT_IND") + + self._s1ap_wrapper.magmad_util.restart_mme_and_wait() + + # Trigger Attach Complete + print("************************* Sending UE_ATTACH_COMPLETE") + attach_complete = s1ap_types.ueAttachComplete_t() + attach_complete.ue_Id = 1 + self._s1ap_wrapper._s1_util.issue_cmd( + s1ap_types.tfwCmd.UE_ATTACH_COMPLETE, attach_complete + ) + + # Wait on EMM Information from MME + self._s1ap_wrapper._s1_util.receive_emm_info() + print("************************* Running UE detach") + # Now detach the UE + detach_req = s1ap_types.uedetachReq_t() + detach_req.ue_Id = 1 + detach_req.ueDetType = ( + s1ap_types.ueDetachType_t.UE_SWITCHOFF_DETACH.value + ) + self._s1ap_wrapper._s1_util.issue_cmd( + s1ap_types.tfwCmd.UE_DETACH_REQUEST, detach_req + ) + + # Wait for UE context release command + return self._s1ap_wrapper.s1_util.get_response() + + +if __name__ == "__main__": + unittest.main() diff --git a/lte/gateway/python/magma/pipelined/app/inout.py b/lte/gateway/python/magma/pipelined/app/inout.py index 7f0eba16a9a8..8b0470eaf160 100644 --- a/lte/gateway/python/magma/pipelined/app/inout.py +++ b/lte/gateway/python/magma/pipelined/app/inout.py @@ -438,9 +438,7 @@ def _install_proxy_flows(self, dp): if self.config.he_proxy_port <= 0: return - self.logger.info("Header Enrichment : %s", self.config.he_proxy_port) parser = dp.ofproto_parser - match = MagmaMatch(proxy_tag=PROXY_TAG_TO_PROXY) actions = [parser.NXActionRegLoad2(dst='eth_dst', value=self.config.he_proxy_eth_mac)] diff --git a/lte/gateway/python/scripts/config_stateless_agw.py b/lte/gateway/python/scripts/config_stateless_agw.py index d3cb9433ccaa..f03a9e9f2ac9 100755 --- a/lte/gateway/python/scripts/config_stateless_agw.py +++ b/lte/gateway/python/scripts/config_stateless_agw.py @@ -186,6 +186,7 @@ def clear_redis_and_restart(): def flushall_redis_and_restart(): _flushall_redis() _start_magmad() + _restart_sctpd() sys.exit(0) diff --git a/lte/gateway/python/scripts/state_cli.py b/lte/gateway/python/scripts/state_cli.py index 918f833dc701..791a949a9991 100644 --- a/lte/gateway/python/scripts/state_cli.py +++ b/lte/gateway/python/scripts/state_cli.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + """ Copyright 2020 The Magma Authors. @@ -39,6 +41,7 @@ def _deserialize_session_json(serialized_json_str: bytes) -> str: dumped = json.dumps(res, indent=2, sort_keys=True) return dumped + def _deserialize_generic_json( element: Union[str, dict, list])-> Union[str, dict, list]: """ @@ -68,6 +71,7 @@ def _deserialize_generic_json( element[k] = _deserialize_generic_json(element[k]) return element + class StateCLI(object): """ CLI for debugging current Magma services state and displaying it diff --git a/lte/gateway/release/build-magma.sh b/lte/gateway/release/build-magma.sh index e183e02b6823..9fc9eaa6d16e 100755 --- a/lte/gateway/release/build-magma.sh +++ b/lte/gateway/release/build-magma.sh @@ -121,7 +121,7 @@ MAGMA_DEPS=( OAI_DEPS=( "libasan3" "libconfig9" - "oai-asn1c-rel15" + "oai-asn1c" "oai-freediameter >= 1.2.0-1" "oai-gnutls >= 3.1.23" "oai-nettle >= 1.0.1" diff --git a/lte/protos/had_orc8r.proto b/lte/protos/ha_orc8r.proto similarity index 95% rename from lte/protos/had_orc8r.proto rename to lte/protos/ha_orc8r.proto index 90e11cfc543f..038fd43eb51e 100644 --- a/lte/protos/had_orc8r.proto +++ b/lte/protos/ha_orc8r.proto @@ -28,8 +28,8 @@ message GetEnodebOffloadStateResponse { map enodeb_offload_states = 1; } -// had-cloud service definition -service HaD { +// ha-cloud service definition +service Ha { // Fetch all ENB state for ENBs served within the pools of the calling AGW rpc GetEnodebOffloadState(GetEnodebOffloadStateRequest) returns (GetEnodebOffloadStateResponse) {} } diff --git a/lte/protos/mconfig/mconfigs.proto b/lte/protos/mconfig/mconfigs.proto index a13c732afb24..50b423d78260 100644 --- a/lte/protos/mconfig/mconfigs.proto +++ b/lte/protos/mconfig/mconfigs.proto @@ -272,6 +272,8 @@ message MME { // Overrides field 4 if this is not empty. Field 4 is in the process of // being deprecated repeated int32 attached_enodeb_tacs = 15; + // MME relative capacity - capacity within an MME group. 8-bit + int32 mme_relative_capacity = 16; // DEPRECATED // Use relay_enabled instead diff --git a/nms/app/packages/magmalte/app/components/ActionTable.js b/nms/app/packages/magmalte/app/components/ActionTable.js index d0226a777bbf..40565d68d54b 100644 --- a/nms/app/packages/magmalte/app/components/ActionTable.js +++ b/nms/app/packages/magmalte/app/components/ActionTable.js @@ -262,7 +262,7 @@ export default function ActionTable(props: ActionTableProps) { icons={tableIcons} data={props.data} actions={ - props.menuItems + props.menuItems?.length ? [ ...(props.actions ? props.actions : []), { diff --git a/nms/app/packages/magmalte/app/components/context/SubscriberContext.js b/nms/app/packages/magmalte/app/components/context/SubscriberContext.js index f735d9384b7e..b13d5068943b 100644 --- a/nms/app/packages/magmalte/app/components/context/SubscriberContext.js +++ b/nms/app/packages/magmalte/app/components/context/SubscriberContext.js @@ -16,9 +16,9 @@ import type { gateway_id, mutable_subscriber, - policy_rule, subscriber, subscriber_id, + subscriber_state, } from '@fbcnms/magma-api'; import React from 'react'; @@ -28,27 +28,11 @@ export type Metrics = { dailyAvg: string, }; -type Session = { - active_policy_rules: Array, - lifecycle_state: - | 'SESSION_TERMINATED' - | 'SESSION_TERMINATING' - | 'SESSION_ACTIVE', - session_id: string, - active_duration_sec: number, - msisdn: string, - apn: string, - session_start_time: number, - ipv4: string, -}; - -export type ActiveApnSessions = {[string]: Array}; - export type SubscriberContextType = { state: {[string]: subscriber}, metrics?: {[string]: Metrics}, gwSubscriberMap: {[gateway_id]: Array}, - sessionState: {[string]: ActiveApnSessions}, + sessionState: {[string]: subscriber_state}, setState?: (key: string, val?: mutable_subscriber) => Promise, }; diff --git a/nms/app/packages/magmalte/app/components/lte/LteMetrics.js b/nms/app/packages/magmalte/app/components/lte/LteMetrics.js index 877736db2a98..9a7cabcb96fe 100644 --- a/nms/app/packages/magmalte/app/components/lte/LteMetrics.js +++ b/nms/app/packages/magmalte/app/components/lte/LteMetrics.js @@ -265,18 +265,18 @@ export default function () { ]; } else { tabList = [ - { - icon: ExploreIcon, - component: {NestedRouteLink}, - label: 'Explorer', - to: '/explorer', - }, { icon: AssessmentIcon, component: {NestedRouteLink}, label: 'Grafana', to: '/grafana', }, + { + icon: ExploreIcon, + component: {NestedRouteLink}, + label: 'Explorer', + to: '/explorer', + }, ]; } @@ -304,7 +304,7 @@ export default function () { component={GrafanaDashboard} /> - + )} diff --git a/nms/app/packages/magmalte/app/components/network/UpgradeTierEditDialog.js b/nms/app/packages/magmalte/app/components/network/UpgradeTierEditDialog.js index bbf747c5f8fd..d6fc74fbbb06 100644 --- a/nms/app/packages/magmalte/app/components/network/UpgradeTierEditDialog.js +++ b/nms/app/packages/magmalte/app/components/network/UpgradeTierEditDialog.js @@ -57,7 +57,9 @@ export default function UpgradeTierEditDialog(props: Props) { tier, }) .then(() => props.onSave(tier)) - .catch(e => enqueueSnackbar(e.response.data.message)); + .catch(e => + enqueueSnackbar(e.response.data.message, {variant: 'error'}), + ); } else { MagmaV1API.putNetworksByNetworkIdTiersByTierId({ networkId: nullthrows(match.params.networkId), @@ -65,7 +67,9 @@ export default function UpgradeTierEditDialog(props: Props) { tier, }) .then(_resp => props.onSave(tier)) - .catch(e => enqueueSnackbar(e.response.data.message)); + .catch(e => + enqueueSnackbar(e.response.data.message, {variant: 'error'}), + ); } }; diff --git a/nms/app/packages/magmalte/app/e2e/__tests__/Subscriber-test.js b/nms/app/packages/magmalte/app/e2e/__tests__/Subscriber-test.js index f635db7f5036..81759f8e8c2f 100644 --- a/nms/app/packages/magmalte/app/e2e/__tests__/Subscriber-test.js +++ b/nms/app/packages/magmalte/app/e2e/__tests__/Subscriber-test.js @@ -60,73 +60,73 @@ describe('NMS', () => { }, 60000); }); -describe('NMS Subscriber Add', () => { - test('verifying subscriber dashboard', async () => { - const page = await browser.newPage(); - try { - await page.goto( - 'https://magma-test.localhost/nms/test/subscribers/overview', - ); - - // check if the description is right - await page.waitForXPath(`//span[text()='Subscribers']`); - - //Add new subscriber - const addSubscriberSelector = await page.$x( - `//span[text()='Add Subscriber']`, - ); - addSubscriberSelector[0].click(); - await page.waitForXPath(`//span[text()='Add Subscribers']`); - - // Add subscriber - const addButtonSelector = await page.$x(`//button[@title='Add']`); - addButtonSelector[0].click(); - - // Add subscriber information attributes - const name = '[data-testid="name"]'; - await page.waitForSelector(name); - await page.click(name); - await page.type(name, 'IMSI001010002220022'); - - const IMSI = '[data-testid="IMSI"]'; - await page.waitForSelector(IMSI); - await page.click(IMSI); - await page.type(IMSI, 'IMSI001010002220022'); - - const authKey = '[data-testid="authKey"]'; - await page.waitForSelector(authKey); - await page.click(authKey); - await page.type(authKey, '8baf473f2f8fd09487cccbd7097c6862'); - - const authOpc = '[data-testid="authOpc"]'; - await page.waitForSelector(authOpc); - await page.click(authOpc); - await page.type(authOpc, '8e27b6af0e692e750f32667a3b14605d'); - - // Add subscriber to save - const saveSubscriber = await page.$x(`//button[@title='Save']`); - saveSubscriber[0].click(); - - // Save subscriber - const saveNewSubscriber = '[data-testid="saveSubscriber"]'; - await page.waitForSelector(saveNewSubscriber); - await page.click(saveNewSubscriber); - await page.waitForXPath( - `//span[text()='Subscriber(s) saved successfully']`, - ); - } catch (err) { - await page.screenshot({ - path: ARTIFACTS_DIR + 'subscriber_add_failed.png', - }); - await page.close(); - throw err; - } - - await page.screenshot({ - path: ARTIFACTS_DIR + 'subscriber_add.png', - }); - }, 60000); -}); +// describe('NMS Subscriber Add', () => { +// test('verifying subscriber dashboard', async () => { +// const page = await browser.newPage(); +// try { +// await page.goto( +// 'https://magma-test.localhost/nms/test/subscribers/overview', +// ); + +// // check if the description is right +// await page.waitForXPath(`//span[text()='Subscribers']`); + +// //Add new subscriber +// const addSubscriberSelector = await page.$x( +// `//span[text()='Add Subscriber']`, +// ); +// addSubscriberSelector[0].click(); +// await page.waitForXPath(`//span[text()='Add Subscribers']`); + +// // Add subscriber +// const addButtonSelector = await page.$x(`//button[@title='Add']`); +// addButtonSelector[0].click(); + +// // Add subscriber information attributes +// const name = '[data-testid="name"]'; +// await page.waitForSelector(name); +// await page.click(name); +// await page.type(name, 'IMSI001010002220022'); + +// const IMSI = '[data-testid="IMSI"]'; +// await page.waitForSelector(IMSI); +// await page.click(IMSI); +// await page.type(IMSI, 'IMSI001010002220022'); + +// const authKey = '[data-testid="authKey"]'; +// await page.waitForSelector(authKey); +// await page.click(authKey); +// await page.type(authKey, '8baf473f2f8fd09487cccbd7097c6862'); + +// const authOpc = '[data-testid="authOpc"]'; +// await page.waitForSelector(authOpc); +// await page.click(authOpc); +// await page.type(authOpc, '8e27b6af0e692e750f32667a3b14605d'); + +// // Add subscriber to save +// const saveSubscriber = await page.$x(`//button[@title='Save']`); +// saveSubscriber[0].click(); + +// // Save subscriber +// const saveNewSubscriber = '[data-testid="saveSubscriber"]'; +// await page.waitForSelector(saveNewSubscriber); +// await page.click(saveNewSubscriber); +// await page.waitForXPath( +// `//span[text()='Subscriber(s) saved successfully']`, +// ); +// } catch (err) { +// await page.screenshot({ +// path: ARTIFACTS_DIR + 'subscriber_add_failed.png', +// }); +// await page.close(); +// throw err; +// } + +// await page.screenshot({ +// path: ARTIFACTS_DIR + 'subscriber_add.png', +// }); +// }, 60000); +// }); describe('NMS Subscriber Edit', () => { test('verifying subscriber dashboard', async () => { diff --git a/nms/app/packages/magmalte/app/state/lte/SubscriberState.js b/nms/app/packages/magmalte/app/state/lte/SubscriberState.js index c979a728f4ae..1e5354bfbaf0 100644 --- a/nms/app/packages/magmalte/app/state/lte/SubscriberState.js +++ b/nms/app/packages/magmalte/app/state/lte/SubscriberState.js @@ -16,20 +16,18 @@ import MagmaV1API from '@fbcnms/magma-api/client/WebClient'; import {getLabelUnit} from '../../views/subscriber/SubscriberUtils'; -import type { - ActiveApnSessions, - Metrics, -} from '../../components/context/SubscriberContext'; +import type {Metrics} from '../../components/context/SubscriberContext'; import type { mutable_subscriber, network_id, subscriber, + subscriber_state, } from '@fbcnms/magma-api'; type InitSubscriberStateProps = { networkId: network_id, setSubscriberMap: ({[string]: subscriber}) => void, - setSessionState: ({[string]: ActiveApnSessions}) => void, + setSessionState: ({[string]: subscriber_state}) => void, setSubscriberMetrics?: ({[string]: Metrics}) => void, enqueueSnackbar?: (msg: string, cfg: {}) => ?(string | number), }; @@ -63,12 +61,7 @@ export default async function InitSubscriberState( networkId, }); if (state) { - const sessionState = {}; - Object.keys(state).map( - subscriber => - (sessionState[subscriber] = state[subscriber].subscriber_state || {}), - ); - setSessionState(sessionState); + setSessionState(state); } } catch (e) { enqueueSnackbar?.('failed fetching subscriber state', { diff --git a/nms/app/packages/magmalte/app/views/equipment/GatewayDetailConfigEdit.js b/nms/app/packages/magmalte/app/views/equipment/GatewayDetailConfigEdit.js index b918c7e1102f..fb03f89d76e0 100644 --- a/nms/app/packages/magmalte/app/views/equipment/GatewayDetailConfigEdit.js +++ b/nms/app/packages/magmalte/app/views/equipment/GatewayDetailConfigEdit.js @@ -104,7 +104,7 @@ const DEFAULT_GATEWAY_CONFIG = { autoupgrade_poll_interval: 60, checkin_interval: 60, checkin_timeout: 30, - dynamic_services: [], + dynamic_services: ['eventd', 'td-agent-bit'], }, name: '', status: { diff --git a/nms/app/packages/magmalte/app/views/equipment/__tests__/GatewayConfigTest.js b/nms/app/packages/magmalte/app/views/equipment/__tests__/GatewayConfigTest.js index 3d66db64dfbd..feadf5055c0e 100644 --- a/nms/app/packages/magmalte/app/views/equipment/__tests__/GatewayConfigTest.js +++ b/nms/app/packages/magmalte/app/views/equipment/__tests__/GatewayConfigTest.js @@ -249,7 +249,7 @@ describe('', () => { autoupgrade_poll_interval: 60, checkin_interval: 60, checkin_timeout: 30, - dynamic_services: [], + dynamic_services: ['eventd', 'td-agent-bit'], }, status: { platform_info: { diff --git a/nms/app/packages/magmalte/app/views/subscriber/SubscriberDetail.js b/nms/app/packages/magmalte/app/views/subscriber/SubscriberDetail.js index 929a7a69dc53..237018a89817 100644 --- a/nms/app/packages/magmalte/app/views/subscriber/SubscriberDetail.js +++ b/nms/app/packages/magmalte/app/views/subscriber/SubscriberDetail.js @@ -16,7 +16,6 @@ import type {DataRows} from '../../components/DataGrid'; import type {subscriber} from '@fbcnms/magma-api'; -import AppBar from '@material-ui/core/AppBar'; import CardTitleRow from '../../components/layout/CardTitleRow'; import DashboardIcon from '@material-ui/icons/Dashboard'; import DataGrid from '../../components/DataGrid'; @@ -24,19 +23,15 @@ import EventsTable from '../../views/events/EventsTable'; import GraphicEqIcon from '@material-ui/icons/GraphicEq'; import Grid from '@material-ui/core/Grid'; import MyLocationIcon from '@material-ui/icons/MyLocation'; -import NestedRouteLink from '@fbcnms/ui/components/NestedRouteLink'; import PersonIcon from '@material-ui/icons/Person'; import React from 'react'; import SettingsIcon from '@material-ui/icons/Settings'; import SubscriberChart from './SubscriberChart'; import SubscriberContext from '../../components/context/SubscriberContext'; import SubscriberDetailConfig from './SubscriberDetailConfig'; -import Tab from '@material-ui/core/Tab'; -import Tabs from '@material-ui/core/Tabs'; -import Text from '../../theme/design-system/Text'; +import TopBar from '../../components/TopBar'; import nullthrows from '@fbcnms/util/nullthrows'; -import {DetailTabItems, GetCurrentTabPos} from '../../components/TabUtils.js'; import {Redirect, Route, Switch} from 'react-router-dom'; import {SubscriberJsonConfig} from './SubscriberDetailConfig'; import {colors, typography} from '../../theme/default'; @@ -98,7 +93,6 @@ const useStyles = makeStyles(theme => ({ })); export default function SubscriberDetail() { - const classes = useStyles(); const {relativePath, relativeUrl, match} = useRouter(); const subscriberId: string = nullthrows(match.params.subscriberId); const ctx = useContext(SubscriberContext); @@ -107,46 +101,37 @@ export default function SubscriberDetail() { return ( <> -
- - Subscriber/{subscriberInfo.name ?? subscriberId} - -
+ - - - - - } - to="/overview" - className={classes.tab} - /> - } - to="/event" - className={classes.tab} - /> - } - to="/config" - className={classes.tab} - /> - - - - ; } - -function OverviewTabLabel() { - const classes = useStyles(); - return ( -
- Overview -
- ); -} - -function ConfigTabLabel() { - const classes = useStyles(); - - return ( -
- Config -
- ); -} - -function EventTabLabel() { - const classes = useStyles(); - - return ( -
- Event -
- ); -} diff --git a/nms/app/packages/magmalte/app/views/subscriber/SubscriberOverview.js b/nms/app/packages/magmalte/app/views/subscriber/SubscriberOverview.js index f4cb83dd89a2..274de5c846e0 100644 --- a/nms/app/packages/magmalte/app/views/subscriber/SubscriberOverview.js +++ b/nms/app/packages/magmalte/app/views/subscriber/SubscriberOverview.js @@ -14,25 +14,32 @@ * @format */ import type {WithAlert} from '@fbcnms/ui/components/Alert/withAlert'; +import type {mutable_subscriber} from '@fbcnms/magma-api'; import ActionTable from '../../components/ActionTable'; import AddSubscriberButton from './SubscriberAddDialog'; import Button from '@material-ui/core/Button'; import CardTitleRow from '../../components/layout/CardTitleRow'; +import Dialog from '@material-ui/core/Dialog'; +import DialogContent from '@material-ui/core/DialogContent'; +import DialogTitle from '@material-ui/core/DialogTitle'; import ExpandLess from '@material-ui/icons/ExpandLess'; import ExpandMore from '@material-ui/icons/ExpandMore'; import Grid from '@material-ui/core/Grid'; import Link from '@material-ui/core/Link'; import List from '@material-ui/core/List'; import ListItem from '@material-ui/core/ListItem'; +import NetworkContext from '../../components/context/NetworkContext'; import PeopleIcon from '@material-ui/icons/People'; import React from 'react'; +import ReactJson from 'react-json-view'; import SubscriberContext from '../../components/context/SubscriberContext'; import SubscriberDetail from './SubscriberDetail'; import Text from '../../theme/design-system/Text'; import TopBar from '../../components/TopBar'; import withAlert from '@fbcnms/ui/components/Alert/withAlert'; +import {FEG_LTE} from '@fbcnms/types/network'; import {Redirect, Route, Switch} from 'react-router-dom'; import {colors, typography} from '../../theme/default'; import {makeStyles} from '@material-ui/styles'; @@ -139,28 +146,74 @@ function SubscriberDashboardInternal() { ); } +type Props = { + open: boolean, + onClose?: () => void, + imsi: string, +}; + +function JsonDialog(props: Props) { + const ctx = useContext(SubscriberContext); + const sessionState = ctx.sessionState[props.imsi] || {}; + const configuredSubscriberState = ctx.state[props.imsi]; + const subscriber: mutable_subscriber = { + ...configuredSubscriberState, + state: sessionState, + }; + return ( + + {props.imsi} + + + + + ); +} + function SubscriberTableRaw(props: WithAlert) { const classes = useStyles(); const {history, relativeUrl} = useRouter(); const [currRow, setCurrRow] = useState({}); const ctx = useContext(SubscriberContext); + const networkCtx = useContext(NetworkContext); const subscriberMap = ctx.state; const sessionState = ctx.sessionState; const subscriberMetrics = ctx.metrics; const enqueueSnackbar = useEnqueueSnackbar(); + const [jsonDialog, setJsonDialog] = useState(false); const tableColumns = [ {title: 'Name', field: 'name'}, { title: 'IMSI', field: 'imsi', - render: currRow => ( - history.push(relativeUrl('/' + currRow.imsi))}> - {currRow.imsi} - - ), + render: currRow => { + const subscriberConfig = subscriberMap[currRow.imsi]; + return ( + + // Link to event tab if FEG_LTE network + history.push( + relativeUrl( + '/' + + currRow.imsi + + `${ + networkCtx.networkType === FEG_LTE && !subscriberConfig + ? '/event' + : '' + }`, + ), + ) + }> + {currRow.imsi} + + ); + }, }, {title: 'Service', field: 'service', width: 100}, {title: 'Current Usage', field: 'currentUsage', width: 175}, @@ -172,14 +225,19 @@ function SubscriberTableRaw(props: WithAlert) { width: 200, }, ]; - const tableData: Array = Object.keys(subscriberMap).map( + + const subscribersIds = Array.from( + new Set([...Object.keys(subscriberMap), ...Object.keys(sessionState)]), + ); + + const tableData: Array = subscribersIds.map( (imsi: string) => { - const subscriberInfo = subscriberMap[imsi]; + const subscriberInfo = subscriberMap[imsi] || {}; const metrics = subscriberMetrics?.[`${imsi}`]; return { name: subscriberInfo.name ?? imsi, imsi: imsi, - service: subscriberInfo.lte.state, + service: subscriberInfo.lte?.state || '', currentUsage: metrics?.currentUsage ?? '0', dailyAvg: metrics?.dailyAvg ?? '0', lastReportedTime: new Date( @@ -189,8 +247,11 @@ function SubscriberTableRaw(props: WithAlert) { }, ); + const onClose = () => setJsonDialog(false); + return (
+ - {subscriberMap ? ( + {subscriberMap || sessionState ? (
{ - const subscriber = sessionState[row.imsi]; + const subscriber = + sessionState[row.imsi]?.subscriber_state || {}; const sessions = Object.keys(subscriber || {}).map( apn => subscriber[apn].filter( @@ -216,8 +278,9 @@ function SubscriberTableRaw(props: WithAlert) { ); return { ...row, - activeApns: Object.keys(sessionState[row.imsi] || {}) - .length, + activeApns: Object.keys( + sessionState[row.imsi]?.subscriber_state || {}, + ).length, activeSessions: sessions.length ? sessions.reduce((a, b) => a + b) : 0, @@ -234,45 +297,65 @@ function SubscriberTableRaw(props: WithAlert) { ] } handleCurrRow={(row: SubscriberRowType) => setCurrRow(row)} - menuItems={[ - { - name: 'View', - handleFunc: () => { - history.push(relativeUrl('/' + currRow.imsi)); - }, - }, - { - name: 'Edit', - handleFunc: () => { - history.push(relativeUrl('/' + currRow.imsi + '/config')); - }, - }, - { - name: 'Remove', - handleFunc: () => { - props - .confirm( - `Are you sure you want to delete ${currRow.imsi}?`, - ) - .then(async confirmed => { - if (!confirmed) { - return; - } - - try { - await ctx.setState?.(currRow.imsi); - } catch (e) { - enqueueSnackbar( - 'failed deleting subscriber ' + currRow.imsi, - { - variant: 'error', - }, + menuItems={ + networkCtx.networkType === FEG_LTE + ? [ + { + name: 'View JSON', + handleFunc: () => { + setJsonDialog(true); + }, + }, + ] + : [ + { + name: 'View JSON', + handleFunc: () => { + setJsonDialog(true); + }, + }, + { + name: 'View', + handleFunc: () => { + history.push(relativeUrl('/' + currRow.imsi)); + }, + }, + { + name: 'Edit', + handleFunc: () => { + history.push( + relativeUrl('/' + currRow.imsi + '/config'), ); - } - }); - }, - }, - ]} + }, + }, + { + name: 'Remove', + handleFunc: () => { + props + .confirm( + `Are you sure you want to delete ${currRow.imsi}?`, + ) + .then(async confirmed => { + if (!confirmed) { + return; + } + + try { + await ctx.setState?.(currRow.imsi); + } catch (e) { + enqueueSnackbar( + 'failed deleting subscriber ' + + currRow.imsi, + { + variant: 'error', + }, + ); + } + }); + }, + }, + ] + } options={{ actionsColumnIndex: -1, pageSizeOptions: [10, 20], @@ -287,7 +370,9 @@ function SubscriberTableRaw(props: WithAlert) { }, openIcon: ExpandLess, render: rowData => { - const subscriber = sessionState[rowData.imsi] || {}; + const subscriber = + sessionState[rowData.imsi]?.subscriber_state || + {}; const subscriberSessionRows: Array = []; Object.keys(subscriber).map((apn: string) => { subscriber[apn].map(infos => { diff --git a/nms/app/packages/magmalte/app/views/subscriber/__tests__/SubscriberTest.js b/nms/app/packages/magmalte/app/views/subscriber/__tests__/SubscriberTest.js index 0cbd6b38aec9..6acdec83f0a1 100644 --- a/nms/app/packages/magmalte/app/views/subscriber/__tests__/SubscriberTest.js +++ b/nms/app/packages/magmalte/app/views/subscriber/__tests__/SubscriberTest.js @@ -16,11 +16,13 @@ import 'jest-dom/extend-expect'; import MuiStylesThemeProvider from '@material-ui/styles/ThemeProvider'; +import NetworkContext from '../../../components/context/NetworkContext'; import React from 'react'; import SubscriberContext from '../../../components/context/SubscriberContext'; import SubscriberDashboard from '../SubscriberOverview'; import defaultTheme from '../../../theme/default.js'; +import {FEG_LTE} from '@fbcnms/types/network'; import {MemoryRouter, Route} from 'react-router-dom'; import {MuiThemeProvider} from '@material-ui/core/styles'; import {cleanup, fireEvent, render, wait} from '@testing-library/react'; @@ -82,183 +84,189 @@ describe('', () => { const sessions = { IMSI0000000000: { - apn_0: [ - { - active_policy_rules: [ - { - id: 'policy_0', - priority: 2, - flow_list: [ - { - match: { - direction: 'UPLINK', - ip_proto: 'IPPROTO_IP', + subscriber_state: { + apn_0: [ + { + active_policy_rules: [ + { + id: 'policy_0', + priority: 2, + flow_list: [ + { + match: { + direction: 'UPLINK', + ip_proto: 'IPPROTO_IP', + }, + action: 'PERMIT', }, - action: 'PERMIT', - }, - { - match: { - direction: 'DOWNLINK', - ip_proto: 'IPPROTO_IP', + { + match: { + direction: 'DOWNLINK', + ip_proto: 'IPPROTO_IP', + }, + action: 'PERMIT', }, - action: 'PERMIT', - }, - ], - tracking_type: 'NO_TRACKING', - }, - { - id: 'policy_01', - priority: 2, - flow_list: [ - { - match: { - direction: 'UPLINK', - ip_proto: 'IPPROTO_IP', + ], + tracking_type: 'NO_TRACKING', + }, + { + id: 'policy_01', + priority: 2, + flow_list: [ + { + match: { + direction: 'UPLINK', + ip_proto: 'IPPROTO_IP', + }, + action: 'PERMIT', }, - action: 'PERMIT', - }, - { - match: { - direction: 'DOWNLINK', - ip_proto: 'IPPROTO_IP', + { + match: { + direction: 'DOWNLINK', + ip_proto: 'IPPROTO_IP', + }, + action: 'PERMIT', }, - action: 'PERMIT', - }, - ], - tracking_type: 'NO_TRACKING', - }, - ], - lifecycle_state: 'SESSION_ACTIVE', - session_id: 'IMSI0000000000-120333', - active_duration_sec: 7, - msisdn: '', - apn: 'apn_0', - session_start_time: 1605281201, - ipv4: '192.168.128.217', - }, - { - active_policy_rules: [], - lifecycle_state: 'SESSION_TERMINATED', - session_id: 'IMSI0000000000-120335', - active_duration_sec: 2, - msisdn: '', - apn: 'apn_0', - session_start_time: 1605281100, - ipv4: '192.168.128.217', - }, - ], - apn_1: [ - { - active_policy_rules: [ - { - id: 'policy_1', - priority: 2, - flow_list: [ - { - match: { - direction: 'UPLINK', - ip_proto: 'IPPROTO_IP', + ], + tracking_type: 'NO_TRACKING', + }, + ], + lifecycle_state: 'SESSION_ACTIVE', + session_id: 'IMSI0000000000-120333', + active_duration_sec: 7, + msisdn: '', + apn: 'apn_0', + session_start_time: 1605281201, + ipv4: '192.168.128.217', + }, + { + active_policy_rules: [], + lifecycle_state: 'SESSION_TERMINATED', + session_id: 'IMSI0000000000-120335', + active_duration_sec: 2, + msisdn: '', + apn: 'apn_0', + session_start_time: 1605281100, + ipv4: '192.168.128.217', + }, + ], + apn_1: [ + { + active_policy_rules: [ + { + id: 'policy_1', + priority: 2, + flow_list: [ + { + match: { + direction: 'UPLINK', + ip_proto: 'IPPROTO_IP', + }, + action: 'PERMIT', }, - action: 'PERMIT', - }, - { - match: { - direction: 'DOWNLINK', - ip_proto: 'IPPROTO_IP', + { + match: { + direction: 'DOWNLINK', + ip_proto: 'IPPROTO_IP', + }, + action: 'PERMIT', }, - action: 'PERMIT', - }, - ], - tracking_type: 'NO_TRACKING', - }, - ], - lifecycle_state: 'SESSION_ACTIVE', - session_id: 'IMSI0000000000-120337', - active_duration_sec: 7, - msisdn: '', - apn: 'apn_1', - session_start_time: 1605281209, - ipv4: '192.168.128.217', - }, - ], + ], + tracking_type: 'NO_TRACKING', + }, + ], + lifecycle_state: 'SESSION_ACTIVE', + session_id: 'IMSI0000000000-120337', + active_duration_sec: 7, + msisdn: '', + apn: 'apn_1', + session_start_time: 1605281209, + ipv4: '192.168.128.217', + }, + ], + }, }, IMSI0000000001: { - apn_3: [ - { - active_policy_rules: [], - lifecycle_state: 'SESSION_TERMINATING', - session_id: 'IMSI0000000001-120345', - active_duration_sec: 1, - msisdn: '', - apn: 'apn_3', - session_start_time: 1605281500, - ipv4: '192.168.128.217', - }, - ], + subscriber_state: { + apn_3: [ + { + active_policy_rules: [], + lifecycle_state: 'SESSION_TERMINATING', + session_id: 'IMSI0000000001-120345', + active_duration_sec: 1, + msisdn: '', + apn: 'apn_3', + session_start_time: 1605281500, + ipv4: '192.168.128.217', + }, + ], + }, }, IMSI0000000002: { - apn_4: [ - { - active_policy_rules: [ - { - id: 'policy_2', - priority: 2, - flow_list: [ - { - match: { - direction: 'UPLINK', - ip_proto: 'IPPROTO_IP', + subscriber_state: { + apn_4: [ + { + active_policy_rules: [ + { + id: 'policy_2', + priority: 2, + flow_list: [ + { + match: { + direction: 'UPLINK', + ip_proto: 'IPPROTO_IP', + }, + action: 'PERMIT', }, - action: 'PERMIT', - }, - { - match: { - direction: 'DOWNLINK', - ip_proto: 'IPPROTO_IP', + { + match: { + direction: 'DOWNLINK', + ip_proto: 'IPPROTO_IP', + }, + action: 'PERMIT', }, - action: 'PERMIT', - }, - ], - tracking_type: 'NO_TRACKING', - }, - ], - lifecycle_state: 'SESSION_ACTIVE', - session_id: 'IMSI0000000002-120347', - active_duration_sec: 27, - msisdn: '', - apn: 'apn_4', - session_start_time: 1605281600, - ipv4: '192.168.128.217', - }, - ], - apn_5: [ - { - active_policy_rules: [], - lifecycle_state: 'SESSION_TERMINATING', - session_id: 'IMSI0000000002-120348', - active_duration_sec: 1, - msisdn: '', - apn: 'apn_5', - session_start_time: 1605281560, - ipv4: '192.168.128.217', - }, - ], - apn_6: [ - { - active_policy_rules: [], - lifecycle_state: 'SESSION_TERMINATED', - session_id: 'IMSI0000000002-120349', - active_duration_sec: 1, - msisdn: '', - apn: 'apn_6', - session_start_time: 1605281590, - ipv4: '192.168.128.217', - }, - ], + ], + tracking_type: 'NO_TRACKING', + }, + ], + lifecycle_state: 'SESSION_ACTIVE', + session_id: 'IMSI0000000002-120347', + active_duration_sec: 27, + msisdn: '', + apn: 'apn_4', + session_start_time: 1605281600, + ipv4: '192.168.128.217', + }, + ], + apn_5: [ + { + active_policy_rules: [], + lifecycle_state: 'SESSION_TERMINATING', + session_id: 'IMSI0000000002-120348', + active_duration_sec: 1, + msisdn: '', + apn: 'apn_5', + session_start_time: 1605281560, + ipv4: '192.168.128.217', + }, + ], + apn_6: [ + { + active_policy_rules: [], + lifecycle_state: 'SESSION_TERMINATED', + session_id: 'IMSI0000000002-120349', + active_duration_sec: 1, + msisdn: '', + apn: 'apn_6', + session_start_time: 1605281590, + ipv4: '192.168.128.217', + }, + ], + }, }, }; - const Wrapper = () => { + const Wrapper = ({networkType}) => { const subscriberCtx = { state: subscribers, gwSubscriberMap: {}, @@ -271,12 +279,18 @@ describe('', () => { initialIndex={0}> - - - + + + + + @@ -290,7 +304,7 @@ describe('', () => { getByTestId, getAllByRole, getByText, - } = render(); + } = render(); await wait(); const rowItems = await getAllByRole('row'); @@ -316,6 +330,11 @@ describe('', () => { expect(rowItems[2]).toHaveTextContent('0'); expect(rowItems[2]).toHaveTextContent('1'); + expect(rowItems[3]).toHaveTextContent('IMSI0000000002'); + expect(rowItems[3]).toHaveTextContent('0'); + expect(rowItems[3]).toHaveTextContent('3'); + expect(rowItems[3]).toHaveTextContent('1'); + // click the actions button for subscriber0 const actionList = getAllByTitle('Actions'); expect(getByTestId('actions-menu')).not.toBeVisible(); diff --git a/nms/app/packages/magmalte/e2e_test_setup.sh b/nms/app/packages/magmalte/e2e_test_setup.sh index 2f796acd3816..57001ce35f41 100755 --- a/nms/app/packages/magmalte/e2e_test_setup.sh +++ b/nms/app/packages/magmalte/e2e_test_setup.sh @@ -15,7 +15,7 @@ set -e mkdir -p .cache mkdir -p /tmp/nms_artifacts -openssl req -nodes -new -x509 -batch -keyout .cache/mock_server.key -out .cache/mock_server.cert -subj "/" +openssl req -nodes -new -x509 -batch -keyout .cache/mock_server.key -out .cache/mock_server.cert docker-compose --env-file .env.mock -f docker-compose-e2e.yml up -d i=0 diff --git a/orc8r/cloud/configs/metricsd.yml b/orc8r/cloud/configs/metricsd.yml index 268473fcdfd6..e2fc45c7ec0f 100644 --- a/orc8r/cloud/configs/metricsd.yml +++ b/orc8r/cloud/configs/metricsd.yml @@ -17,3 +17,5 @@ prometheusQueryAddress: "http://prometheus:9090" alertmanagerApiURL: "http://alertmanager:9093/api/v2" prometheusConfigServiceURL: "http://prometheus-configurer:9100/v1" alertmanagerConfigServiceURL: "http://alertmanager-configurer:9101/v1" + +useSeriesCache: true \ No newline at end of file diff --git a/orc8r/cloud/deploy/terraform/orc8r-aws/eks.tf b/orc8r/cloud/deploy/terraform/orc8r-aws/eks.tf index 1d80d6981f11..f2fefba10ad3 100644 --- a/orc8r/cloud/deploy/terraform/orc8r-aws/eks.tf +++ b/orc8r/cloud/deploy/terraform/orc8r-aws/eks.tf @@ -45,7 +45,7 @@ module "eks" { } worker_additional_security_group_ids = concat([aws_security_group.default.id], var.eks_worker_additional_sg_ids) workers_additional_policies = var.eks_worker_additional_policy_arns - worker_groups = var.eks_worker_groups + worker_groups = var.thanos_enabled ? concat(var.eks_worker_groups, var.thanos_worker_groups) : var.eks_worker_groups map_roles = var.eks_map_roles map_users = var.eks_map_users diff --git a/orc8r/cloud/deploy/terraform/orc8r-aws/variables.tf b/orc8r/cloud/deploy/terraform/orc8r-aws/variables.tf index bb6e1accc9fd..e84ec69a2d66 100644 --- a/orc8r/cloud/deploy/terraform/orc8r-aws/variables.tf +++ b/orc8r/cloud/deploy/terraform/orc8r-aws/variables.tf @@ -87,10 +87,34 @@ variable "eks_worker_groups" { asg_min_size = 1 asg_max_size = 3 autoscaling_enabled = false + kubelet_extra_args = "" // object types must be identical (see thanos_worker_groups) }, ] } +variable "thanos_worker_groups" { + # Check the docs at https://github.com/terraform-aws-modules/terraform-aws-eks + # for the complete set of valid properties for these objects. This worker group + # exists because some thanos components (compact) require significant instance + # storage to operate. + # Use label key 'compute-type' to specify the node used by nodeSelector + # in the helm release + description = "Worker group configuration for Thanos. Default consists of 1 group consisting of 1 m5d.xlarge for thanos." + type = any + default = [ + { + name = "thanos-1" + instance_type = "m5d.xlarge" + asg_desired_capacity = 1 + asg_min_size = 1 + asg_max_size = 1 + autoscaling_enabled = false + kubelet_extra_args = "--node-labels=compute-type=thanos" + }, + ] + +} + variable "eks_map_roles" { description = "EKS IAM role mapping. Note that by default, the creator of the cluster will be in the system:master group." type = list( @@ -343,3 +367,8 @@ variable "elasticsearch_domain_tags" { default = {} } +variable "thanos_enabled" { + description = "Enable thanos infrastructure" + type = bool + default = false +} diff --git a/orc8r/cloud/deploy/terraform/orc8r-helm-aws/main.tf b/orc8r/cloud/deploy/terraform/orc8r-helm-aws/main.tf index af5dd0cca48a..c2a163d34d8d 100644 --- a/orc8r/cloud/deploy/terraform/orc8r-helm-aws/main.tf +++ b/orc8r/cloud/deploy/terraform/orc8r-helm-aws/main.tf @@ -85,7 +85,12 @@ resource "helm_release" "orc8r" { thanos_bucket = var.thanos_enabled ? aws_s3_bucket.thanos_object_store_bucket[0].bucket : "" thanos_aws_access_key = var.thanos_enabled ? aws_iam_access_key.thanos_s3_access_key[0].id : "" thanos_aws_secret_key = var.thanos_enabled ? aws_iam_access_key.thanos_s3_access_key[0].secret : "" - region = var.region + + thanos_compact_selector = var.thanos_compact_node_selector != "" ? format("compute-type: %s", var.thanos_compact_node_selector) : "{}" + thanos_query_selector = var.thanos_query_node_selector != "" ? format("compute-type: %s", var.thanos_query_node_selector) : "{}" + thanos_store_selector = var.thanos_store_node_selector != "" ? format("compute-type: %s", var.thanos_store_node_selector) : "{}" + + region = var.region })] set_sensitive { diff --git a/orc8r/cloud/deploy/terraform/orc8r-helm-aws/secrets.tf b/orc8r/cloud/deploy/terraform/orc8r-helm-aws/secrets.tf index 81949c2c1394..7aa9d65dac12 100644 --- a/orc8r/cloud/deploy/terraform/orc8r-helm-aws/secrets.tf +++ b/orc8r/cloud/deploy/terraform/orc8r-helm-aws/secrets.tf @@ -93,7 +93,7 @@ resource "kubernetes_secret" "orc8r_configs" { data = { "metricsd.yml" = yamlencode({ "profile" : "prometheus", - "prometheusQueryAddress" : format("http://%s-prometheus:9090", var.helm_deployment_name), + "prometheusQueryAddress" : var.thanos_enabled ? format("http://%s-thanos-query-http:10902", var.helm_deployment_name) : format("http://%s-prometheus:9090", var.helm_deployment_name), "alertmanagerApiURL" : format("http://%s-alertmanager:9093/api/v2", var.helm_deployment_name), "prometheusConfigServiceURL" : format("http://%s-prometheus-configurer:9100", var.helm_deployment_name), diff --git a/orc8r/cloud/deploy/terraform/orc8r-helm-aws/templates/orc8r-values.tpl b/orc8r/cloud/deploy/terraform/orc8r-helm-aws/templates/orc8r-values.tpl index ba0f161da7b6..88f9cde1bfe4 100644 --- a/orc8r/cloud/deploy/terraform/orc8r-helm-aws/templates/orc8r-values.tpl +++ b/orc8r/cloud/deploy/terraform/orc8r-helm-aws/templates/orc8r-values.tpl @@ -129,6 +129,19 @@ metrics: thanos: enabled: ${thanos_enabled} + + compact: + nodeSelector: + ${thanos_compact_selector} + + store: + nodeSelector: + ${thanos_store_selector} + + query: + nodeSelector: + ${thanos_query_selector} + objstore: type: S3 config: @@ -166,6 +179,7 @@ nms: env: api_host: ${api_hostname} + mysql_db: ${nms_db_name} mysql_host: ${nms_db_host} mysql_user: ${nms_db_user} grafana_address: ${user_grafana_hostname} diff --git a/orc8r/cloud/deploy/terraform/orc8r-helm-aws/variables.tf b/orc8r/cloud/deploy/terraform/orc8r-helm-aws/variables.tf index d885ce6f98bb..27b56ca0df25 100644 --- a/orc8r/cloud/deploy/terraform/orc8r-helm-aws/variables.tf +++ b/orc8r/cloud/deploy/terraform/orc8r-helm-aws/variables.tf @@ -281,3 +281,22 @@ variable "thanos_object_store_bucket_name" { type = string default = "" } + +variable "thanos_query_node_selector" { + description = "NodeSelector value to specify which node to run thanos query pod on. Default is 'thanos' to be deployed on the default thanos worker group." + type = string + default = "thanos" +} + + +variable "thanos_compact_node_selector" { + description = "NodeSelector value to specify which node to run thanos compact pod on. Label is 'compute-type:'" + type = string + default = "" +} + +variable "thanos_store_node_selector" { + description = "NodeSelector value to specify which node to run thanos store pod on. Label is 'compute-type:'" + type = string + default = "" +} diff --git a/orc8r/cloud/docker/controller/supervisord.conf b/orc8r/cloud/docker/controller/supervisord.conf index ba568d9896aa..79178a8dfba2 100644 --- a/orc8r/cloud/docker/controller/supervisord.conf +++ b/orc8r/cloud/docker/controller/supervisord.conf @@ -173,6 +173,14 @@ stderr_logfile=NONE stdout_events_enabled=true stderr_events_enabled=true +[program:ha] +command=/usr/bin/envdir /var/opt/magma/envdir /var/opt/magma/bin/ha -logtostderr=true -v=0 +autorestart=true +stdout_logfile=NONE +stderr_logfile=NONE +stdout_events_enabled=true +stderr_events_enabled=true + [program:policydb] command=/usr/bin/envdir /var/opt/magma/envdir /var/opt/magma/bin/policydb -run_echo_server=true -logtostderr=true -v=0 autorestart=true diff --git a/orc8r/cloud/go/services/metricsd/const.go b/orc8r/cloud/go/services/metricsd/const.go index 8d34216a96cc..85f857a3fcb0 100644 --- a/orc8r/cloud/go/services/metricsd/const.go +++ b/orc8r/cloud/go/services/metricsd/const.go @@ -20,4 +20,6 @@ const ( PrometheusConfigServiceURL = "prometheusConfigServiceURL" AlertmanagerConfigServiceURL = "alertmanagerConfigServiceURL" AlertmanagerApiURL = "alertmanagerApiURL" + + UseSeriesCache = "useSeriesCache" ) diff --git a/orc8r/cloud/go/services/metricsd/obsidian/handlers/handlers.go b/orc8r/cloud/go/services/metricsd/obsidian/handlers/handlers.go index 7f0499eded0e..6ce5d5cbc2b1 100644 --- a/orc8r/cloud/go/services/metricsd/obsidian/handlers/handlers.go +++ b/orc8r/cloud/go/services/metricsd/obsidian/handlers/handlers.go @@ -35,6 +35,7 @@ const ( // GetObsidianHandlers returns all obsidian handlers for metricsd func GetObsidianHandlers(configMap *config.ConfigMap) []obsidian.Handler { + useSeriesCache, _ := configMap.GetBool(metricsd.UseSeriesCache) var ret []obsidian.Handler client, err := promAPI.NewClient(promAPI.Config{Address: configMap.MustGetString(metricsd.PrometheusQueryAddress)}) if err != nil { @@ -72,7 +73,7 @@ func GetObsidianHandlers(configMap *config.ConfigMap) []obsidian.Handler { // Tenant Prometheus API obsidian.Handler{Path: promH.TenantPromV1QueryURL, Methods: obsidian.GET, HandlerFunc: promH.GetTenantPromQueryHandler(pAPI)}, obsidian.Handler{Path: promH.TenantPromV1QueryRangeURL, Methods: obsidian.GET, HandlerFunc: promH.GetTenantPromQueryRangeHandler(pAPI)}, - obsidian.Handler{Path: promH.TenantPromV1SeriesURL, Methods: obsidian.GET, HandlerFunc: promH.GetTenantPromSeriesHandler(pAPI)}, + obsidian.Handler{Path: promH.TenantPromV1SeriesURL, Methods: obsidian.GET, HandlerFunc: promH.GetTenantPromSeriesHandler(pAPI, useSeriesCache)}, obsidian.Handler{Path: promH.TenantPromV1ValuesURL, Methods: obsidian.GET, HandlerFunc: promH.GetTenantPromValuesHandler(pAPI)}, // TargetsMetadata diff --git a/orc8r/cloud/go/services/metricsd/prometheus/handlers/cache/series_cache.go b/orc8r/cloud/go/services/metricsd/prometheus/handlers/cache/series_cache.go new file mode 100644 index 000000000000..ffcd71ce817a --- /dev/null +++ b/orc8r/cloud/go/services/metricsd/prometheus/handlers/cache/series_cache.go @@ -0,0 +1,251 @@ +package cache + +import ( + "context" + "sort" + "strings" + "sync" + "time" + + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" +) + +const ( + estimatedBytesPerSeries = 100 +) + +// SeriesAPI interfaces the prometheus /series API to be used by the values +// handler for updating cache values +type SeriesAPI interface { + // Series finds series by label matchers. + Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, v1.Warnings, error) +} + +type SeriesCache struct { + responses map[string]*cacheData + specs Specs + backfillSpecs BackfillSpecs + // updateFunc knows how to update the cache values so it can periodically + // keep itself up to date + updateFunc func(params []string, start, end time.Time) (values, error) + sync.Mutex +} + +type Params struct { + Specs Specs + Backfill BackfillSpecs + UpdateFreq time.Duration +} + +type Specs struct { + // oldestAcceptable oldest age of a cache value to be returned + OldestAcceptable time.Duration + // ttl oldest age of individual values to keep if not seen recently + TTL time.Duration + // limitBytes maximum size of cache in memory. Delete old results if full. + LimitBytes int +} + +type BackfillSpecs struct { + Lookback time.Duration + Width time.Duration + Steps int +} + +func NewSeriesCache(params Params, updateFunc func(params []string, start, end time.Time) (values, error)) *SeriesCache { + c := &SeriesCache{ + responses: make(map[string]*cacheData), + specs: params.Specs, + backfillSpecs: params.Backfill, + updateFunc: updateFunc, + } + if params.UpdateFreq != 0 { + go c.updatePeriodically(params.UpdateFreq) + } + return c +} + +// GetCacheUpdateProvider provides a function which will query the given series API +// for a specified set of params +func GetCacheUpdateProvider(api SeriesAPI) func(params []string, start, end time.Time) (values, error) { + return func(params []string, start, end time.Time) (values, error) { + res, _, err := api.Series(context.Background(), params, start, end) + if err != nil { + return values{}, err + } + return makeSeriesValuesNow(res), nil + } +} + +// Get returns a cached value if it exists and is new enough. If it exists but +// updated too long ago, it triggers an update +func (c *SeriesCache) Get(params []string) ([]model.LabelSet, bool) { + key := paramsToKey(params) + resp, exists := c.responses[key] + if !exists { + return []model.LabelSet{}, false + } + if time.Since(resp.updateTime) > c.specs.OldestAcceptable { + go c.updateResponse(resp.params, resp.updateTime, time.Now()) + return []model.LabelSet{}, false + } + resp.requestTime = time.Now() + return resp.getSeries(), true +} + +// Set initializes a response with values and a function to update this response +func (c *SeriesCache) Set(params []string, series []model.LabelSet) { + c.Lock() + defer c.Unlock() + // Delete oldest responses until this data fits + for c.getEstimatedSize() > c.specs.LimitBytes { + c.deleteOldestResponse() + } + vals := makeSeriesValuesNow(series) + c.responses[paramsToKey(params)] = &cacheData{ + data: vals, + params: params, + requestTime: time.Now(), + updateTime: time.Now(), + } + go c.backfillValues(params) +} + +// updatePeriodically iterates through each cache value and updates the value +// after "dur" has elapsed. This is used to keep cache values up to date to +// decrease cache misses +func (c *SeriesCache) updatePeriodically(dur time.Duration) { + for range time.Tick(dur) { + for _, resp := range c.responses { + c.updateResponse(resp.params, time.Now().Add(-dur), time.Now()) + } + } +} + +// updateResponse takes merges old values with new values, updating their "lastSeen" +// time if they are already in this response +func (c *SeriesCache) updateResponse(params []string, start, end time.Time) { + key := paramsToKey(params) + newValues, err := c.updateFunc(params, start, end) + if err != nil { + return + } + c.Lock() + defer c.Unlock() + c.responses[key].updateTime = time.Now() + c.responses[key].data = mergeData(c.responses[key].data, newValues, c.specs.TTL) +} + +// backfillValues updates a cache value by iteratively looking back in time to +// get series. This is to avoid a single massive query which could overload +// the prometheus server and timeout. +func (c *SeriesCache) backfillValues(params []string) { + if c.backfillSpecs.Lookback == 0 { + return + } + now := time.Now() + stepSize := c.backfillSpecs.Lookback / time.Duration(c.backfillSpecs.Steps) + + for step := 0; step < c.backfillSpecs.Steps; step++ { + start := now.Add(-stepSize * time.Duration(step)) + end := start.Add(c.backfillSpecs.Width) + c.updateResponse(params, start, end) + } +} + +// deleteOldestResponse deletes the least recently used response in this cache +// to make room for new responses +func (c *SeriesCache) deleteOldestResponse() { + var oldest time.Duration = 0 + currentTime := time.Now() + oldestKey := "" + for key, resp := range c.responses { + age := currentTime.Sub(resp.requestTime) + if age > oldest { + oldest = age + oldestKey = key + } + } + if oldestKey != "" { + delete(c.responses, oldestKey) + } +} + +// getEstimatedSize calculates approximately the size in memory of a cache value. +// This is approximate since calculating a real size would be complex and costly +func (c *SeriesCache) getEstimatedSize() int { + numSeries := 0 + for _, resp := range c.responses { + numSeries += len(resp.data) + } + return estimatedBytesPerSeries * numSeries +} + +// mergeData takes new values and adds them to the cache if they don't already +// exist in the cache, and removes values that are too old +// Note: uses labelset.Fingerprint() to get a key for each series. I've tested that +// function on a set of >100k series and it had 0 collisions so I think it will +// be fine here especially since accuracy is not super important. +func mergeData(old, new values, ttl time.Duration) values { + seriesSet := map[model.Fingerprint]struct{}{} + mergedSeries := make([]value, 0, len(old)) + now := time.Now() + for _, val := range old { + age := now.Sub(val.lastSeen) + if age > ttl { + continue + } + seriesSet[val.series.Fingerprint()] = struct{}{} + val.lastSeen = now + mergedSeries = append(mergedSeries, val) + } + for _, val := range new { + if _, ok := seriesSet[val.series.Fingerprint()]; !ok { + mergedSeries = append(mergedSeries, val) + } + } + return mergedSeries +} + +func paramsToKey(params []string) string { + sort.Strings(params) + return strings.Join(params, ",") +} + +func makeSeriesValuesNow(series []model.LabelSet) values { + now := time.Now() + vals := make(values, len(series)) + for idx, ser := range series { + vals[idx] = value{ + series: ser, + lastSeen: now, + } + } + return vals +} + +// cacheData is a struct that holds a list of labelsets (series) along +// with the last time each string has been seen, so it can remove strings that +// have not been seen in a long time. +type cacheData struct { + data values + params []string + requestTime time.Time + updateTime time.Time +} + +func (r *cacheData) getSeries() []model.LabelSet { + series := make([]model.LabelSet, len(r.data)) + for idx, val := range r.data { + series[idx] = val.series + } + return series +} + +type values []value + +type value struct { + series model.LabelSet + lastSeen time.Time +} diff --git a/orc8r/cloud/go/services/metricsd/prometheus/handlers/cache/series_cache_test.go b/orc8r/cloud/go/services/metricsd/prometheus/handlers/cache/series_cache_test.go new file mode 100644 index 000000000000..ee16a028d4d7 --- /dev/null +++ b/orc8r/cloud/go/services/metricsd/prometheus/handlers/cache/series_cache_test.go @@ -0,0 +1,77 @@ +package cache + +import ( + "testing" + "time" + + "magma/orc8r/cloud/go/services/metricsd/prometheus/handlers/mocks" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var ( + sampleLabelSet = []model.LabelSet{{"__name__": "val1"}} +) + +func TestValuesCache_Set(t *testing.T) { + mockAPI := &mocks.SeriesAPI{} + testCache := getTestCache(time.Minute, time.Minute, 150, mockAPI) + + // Basic Cache Set works + testCache.Set([]string{"testParam1"}, sampleLabelSet) + assert.Len(t, testCache.responses, 1) + time.Sleep(time.Millisecond * 500) + testCache.Set([]string{"testParam1"}, sampleLabelSet) + assert.Len(t, testCache.responses, 1) + time.Sleep(time.Millisecond * 500) + testCache.Set([]string{"testParam2"}, sampleLabelSet) + assert.Len(t, testCache.responses, 2) + time.Sleep(time.Millisecond * 500) + + // Cache deletes oldest series when full + testCache.Set([]string{"testParam3"}, sampleLabelSet) + assert.Len(t, testCache.responses, 2) + assert.NotNil(t, testCache.responses["testParam3"]) + assert.NotNil(t, testCache.responses["testParam2"]) + assert.Nil(t, testCache.responses["testParam1"]) +} + +func TestValuesCache_Get(t *testing.T) { + mockAPI := &mocks.SeriesAPI{} + testCache := getTestCache(time.Minute, time.Minute, 10000, mockAPI) + + // Basic Cache Get works + _, ok := testCache.Get([]string{"testParam1"}) + assert.False(t, ok) + testCache.Set([]string{"testParam1"}, sampleLabelSet) + data, ok := testCache.Get([]string{"testParam1"}) + assert.True(t, ok) + assert.Equal(t, sampleLabelSet, data) + + // No cache hit if result out of date + mockAPI.On("Series", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(sampleLabelSet, nil, nil) + testCache.Set([]string{"testParam2"}, sampleLabelSet) + testCache.responses["testParam2"].updateTime = time.Now().Add(-5 * time.Minute) + data, ok = testCache.Get([]string{"testParam2"}) + assert.False(t, ok) + // Wait for goroutine updateFunc to be called + time.Sleep(time.Second) + mockAPI.AssertNumberOfCalls(t, "Series", 1) + // Now that it's been updated, cache get should return + data, ok = testCache.Get([]string{"testParam2"}) + assert.True(t, ok) +} + +func getTestCache(oldestAcceptable, ttl time.Duration, limit int, mockAPI SeriesAPI) SeriesCache { + return SeriesCache{ + responses: map[string]*cacheData{}, + specs: Specs{ + OldestAcceptable: oldestAcceptable, + TTL: ttl, + LimitBytes: limit, + }, + updateFunc: GetCacheUpdateProvider(mockAPI), + } +} diff --git a/orc8r/cloud/go/services/metricsd/prometheus/handlers/mocks/SeriesAPI.go b/orc8r/cloud/go/services/metricsd/prometheus/handlers/mocks/SeriesAPI.go new file mode 100644 index 000000000000..c4ea92ac1493 --- /dev/null +++ b/orc8r/cloud/go/services/metricsd/prometheus/handlers/mocks/SeriesAPI.go @@ -0,0 +1,52 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + model "github.com/prometheus/common/model" + + time "time" + + v1 "github.com/prometheus/client_golang/api/prometheus/v1" +) + +// SeriesAPI is an autogenerated mock type for the SeriesAPI type +type SeriesAPI struct { + mock.Mock +} + +// Series provides a mock function with given fields: ctx, matches, startTime, endTime +func (_m *SeriesAPI) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, v1.Warnings, error) { + ret := _m.Called(ctx, matches, startTime, endTime) + + var r0 []model.LabelSet + if rf, ok := ret.Get(0).(func(context.Context, []string, time.Time, time.Time) []model.LabelSet); ok { + r0 = rf(ctx, matches, startTime, endTime) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]model.LabelSet) + } + } + + var r1 v1.Warnings + if rf, ok := ret.Get(1).(func(context.Context, []string, time.Time, time.Time) v1.Warnings); ok { + r1 = rf(ctx, matches, startTime, endTime) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(v1.Warnings) + } + } + + var r2 error + if rf, ok := ret.Get(2).(func(context.Context, []string, time.Time, time.Time) error); ok { + r2 = rf(ctx, matches, startTime, endTime) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} diff --git a/orc8r/cloud/go/services/metricsd/prometheus/handlers/promo_handlers.go b/orc8r/cloud/go/services/metricsd/prometheus/handlers/promo_handlers.go index d3db1900a2e3..1cfe5624d57a 100644 --- a/orc8r/cloud/go/services/metricsd/prometheus/handlers/promo_handlers.go +++ b/orc8r/cloud/go/services/metricsd/prometheus/handlers/promo_handlers.go @@ -23,6 +23,7 @@ import ( "magma/orc8r/cloud/go/obsidian" "magma/orc8r/cloud/go/services/metricsd/obsidian/utils" + "magma/orc8r/cloud/go/services/metricsd/prometheus/handlers/cache" "magma/orc8r/cloud/go/services/metricsd/prometheus/restrictor" "magma/orc8r/cloud/go/services/orchestrator/obsidian/handlers" "magma/orc8r/cloud/go/services/tenants" @@ -30,7 +31,7 @@ import ( "magma/orc8r/lib/go/metrics" "github.com/labstack/echo" - v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" ) @@ -64,6 +65,8 @@ const ( TargetsMetadata = tenantH.TenantRootPath + obsidian.UrlSep + targetsMetadata defaultStepWidth = "15s" + + oneGB = 1024 * 1024 * 1024 ) func networkQueryRestrictorProvider(networkID string) restrictor.QueryRestrictor { @@ -226,6 +229,12 @@ type PromQLDataStruct struct { Result model.Value `json:"result"` } +// prometheusSeriesData is the struct the prometheus series api returns +type prometheusSeriesData struct { + Status string `json:"status"` + Data []model.LabelSet `json:"data"` +} + var ( minTime = time.Unix(math.MinInt64/1000+62135596801, 0).UTC() maxTime = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC() @@ -263,15 +272,38 @@ func TenantSeriesHandlerProvider(api v1.API) func(c echo.Context) error { if err != nil { return obsidian.HttpError(fmt.Errorf("Error parsing series matchers: %v", err), http.StatusBadRequest) } + series, err := prometheusSeries(c, seriesMatches, api) if err != nil { return err } - return c.JSON(http.StatusOK, series) + return c.JSON(http.StatusOK, struct { + Status string `json:"status"` + Data []model.LabelSet `json:"data"` + }{Status: "success", Data: series}) } } -func GetTenantPromSeriesHandler(api v1.API) func(c echo.Context) error { +// GetTenantPromSeriesHandler provides a handler for the /series endpoint +// spoofed to the same path as in prometheus proper. Used by Grafana only. +func GetTenantPromSeriesHandler(api v1.API, useCache bool) func(c echo.Context) error { + var seriesCache *cache.SeriesCache + if useCache { + seriesCache = cache.NewSeriesCache(cache.Params{ + Specs: cache.Specs{ + OldestAcceptable: 5 * time.Minute, + TTL: 30 * time.Minute, + LimitBytes: oneGB, + }, + Backfill: cache.BackfillSpecs{ + Lookback: 30 * 24 * time.Hour, + Width: 3 * time.Hour, + Steps: 30, + }, + UpdateFreq: 4 * time.Minute, + }, cache.GetCacheUpdateProvider(api)) + } + return func(c echo.Context) error { oID, oerr := obsidian.GetTenantID(c) if oerr != nil { @@ -285,14 +317,28 @@ func GetTenantPromSeriesHandler(api v1.API) func(c echo.Context) error { if err != nil { return obsidian.HttpError(fmt.Errorf("Error parsing series matchers: %v", err), http.StatusBadRequest) } - series, err := prometheusSeries(c, seriesMatches, api) + + // Check the cache for stored responses + if seriesCache != nil { + if resp, ok := seriesCache.Get(seriesMatches); ok { + return c.JSON(http.StatusOK, prometheusSeriesData{Status: "success", Data: resp}) + } + } + + // If cache miss, query the api and set response in the cache + defaultStartTime := time.Now().Add(-3 * time.Hour) + defaultEndTime := time.Now() + startTime, err := utils.ParseTime(c.QueryParam(utils.ParamRangeStart), &defaultStartTime) + endTime, err := utils.ParseTime(c.QueryParam(utils.ParamRangeEnd), &defaultEndTime) + + res, _, err := api.Series(context.Background(), seriesMatches, startTime, endTime) if err != nil { - return err + return obsidian.HttpError(err, http.StatusInternalServerError) } - return c.JSON(http.StatusOK, struct { - Status string `json:"status"` - Data []model.LabelSet `json:"data"` - }{Status: "success", Data: series}) + if seriesCache != nil { + seriesCache.Set(seriesMatches, res) + } + return c.JSON(http.StatusOK, prometheusSeriesData{Status: "success", Data: res}) } } @@ -339,11 +385,7 @@ func getSeriesMatches(c echo.Context, matchParam string, queryRestrictor restric return seriesMatchers, nil } -/* GetTenantPromV1ValuesHandler returns the values of a given label for a tenant. - * We can't just proxy the request to Prometheus since this endpoint has no way - * of restricting the query, so we have to simulate it by doing a series request - * and then manipulating the result - */ +// GetTenantPromV1ValuesHandler returns the values of a given label for a tenant. func GetTenantPromValuesHandler(api v1.API) func(c echo.Context) error { return func(c echo.Context) error { oID, oerr := obsidian.GetTenantID(c) @@ -373,16 +415,16 @@ func GetTenantPromValuesHandler(api v1.API) func(c echo.Context) error { startTime, err := utils.ParseTime(c.QueryParam(utils.ParamRangeStart), &defaultStartTime) endTime, err := utils.ParseTime(c.QueryParam(utils.ParamRangeEnd), &maxTime) - // TODO: catch the warnings replacing _ res, _, err := api.Series(context.Background(), seriesMatchers, startTime, endTime) if err != nil { return obsidian.HttpError(err, http.StatusInternalServerError) } - ret := prometheusValuesData{ - Status: "success", - Data: getSetOfValuesFromLabel(res, model.LabelName(labelName)), - } - return c.JSON(http.StatusOK, ret) + data := getSetOfValuesFromLabel(res, model.LabelName(labelName)) + + return c.JSON(http.StatusOK, prometheusValuesData{ + Status: "Success", + Data: data, + }) } } diff --git a/orc8r/cloud/go/services/metricsd/prometheus/handlers/promo_handlers_test.go b/orc8r/cloud/go/services/metricsd/prometheus/handlers/promo_handlers_test.go index bdf7d22277bc..18fb2d1e7515 100644 --- a/orc8r/cloud/go/services/metricsd/prometheus/handlers/promo_handlers_test.go +++ b/orc8r/cloud/go/services/metricsd/prometheus/handlers/promo_handlers_test.go @@ -95,3 +95,4 @@ func TestGetSetOfValuesFromLabel(t *testing.T) { sort.Strings(vals) assert.Equal(t, []string{"test", "test2"}, vals) } + diff --git a/orc8r/cloud/helm/orc8r/Chart.yaml b/orc8r/cloud/helm/orc8r/Chart.yaml index e07cf7faa8ab..2eea47609662 100644 --- a/orc8r/cloud/helm/orc8r/Chart.yaml +++ b/orc8r/cloud/helm/orc8r/Chart.yaml @@ -13,10 +13,31 @@ apiVersion: v1 appVersion: "1.0" description: A Helm chart for magma orchestrator name: orc8r -version: 1.4.43 +version: 1.5.2 engine: gotpl sources: - https://github.com/facebookincubator/magma keywords: - magma - or8cr + +dependencies: + - name: secrets + version: 0.1.9 + repository: "" + condition: secrets.create + - name: metrics + version: 1.4.20 + repository: "" + condition: metrics.enabled + - name: nms + version: 0.1.7 + repository: "" + condition: nms.enabled + - name: logging + version: 0.1.10 + repository: "" + condition: logging.enabled + - name: orc8rlib + version: 0.1.2 + repository: file://../orc8rlib diff --git a/orc8r/cloud/helm/orc8r/charts/metrics/Chart.yaml b/orc8r/cloud/helm/orc8r/charts/metrics/Chart.yaml index 683e4c87672c..1e3661e8a954 100644 --- a/orc8r/cloud/helm/orc8r/charts/metrics/Chart.yaml +++ b/orc8r/cloud/helm/orc8r/charts/metrics/Chart.yaml @@ -13,7 +13,7 @@ apiVersion: v1 appVersion: "1.0" description: A Helm chart for magma metrics name: metrics -version: 1.4.19 +version: 1.4.20 engine: gotpl sources: - https://github.com/facebookincubator/magma diff --git a/orc8r/cloud/helm/orc8r/charts/metrics/values.yaml b/orc8r/cloud/helm/orc8r/charts/metrics/values.yaml index 51505eef9841..6c0ce0e0ed1c 100644 --- a/orc8r/cloud/helm/orc8r/charts/metrics/values.yaml +++ b/orc8r/cloud/helm/orc8r/charts/metrics/values.yaml @@ -175,7 +175,7 @@ prometheusConfigurer: image: repository: docker.io/facebookincubator/prometheus-configurer - tag: 1.0.0 + tag: 1.0.3 pullPolicy: IfNotPresent resources: {} @@ -214,7 +214,7 @@ alertmanagerConfigurer: image: repository: docker.io/facebookincubator/alertmanager-configurer - tag: 1.0.0 + tag: 1.0.3 pullPolicy: IfNotPresent resources: {} diff --git a/orc8r/cloud/helm/orc8r/charts/secrets/Chart.yaml b/orc8r/cloud/helm/orc8r/charts/secrets/Chart.yaml index 3231b3bcb333..a0bf333e5574 100644 --- a/orc8r/cloud/helm/orc8r/charts/secrets/Chart.yaml +++ b/orc8r/cloud/helm/orc8r/charts/secrets/Chart.yaml @@ -13,7 +13,7 @@ apiVersion: v1 appVersion: "1.0" description: A Helm chart to create magma orchestrator secrets name: secrets -version: 0.1.8 +version: 0.1.9 engine: gotpl sources: - https://github.com/facebookincubator/magma diff --git a/orc8r/cloud/helm/orc8r/charts/secrets/templates/_helpers.tpl b/orc8r/cloud/helm/orc8r/charts/secrets/templates/_helpers.tpl index 9c0bbcd0fff3..4dc04b5b76f5 100644 --- a/orc8r/cloud/helm/orc8r/charts/secrets/templates/_helpers.tpl +++ b/orc8r/cloud/helm/orc8r/charts/secrets/templates/_helpers.tpl @@ -17,3 +17,28 @@ app.kubernetes.io/managed-by: helm app.kubernetes.io/part-of: magma helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} {{- end -}} + +{{- define "orchestrator-config-template" -}} +useGRPCExporter: true +prometheusGRPCPushAddress: "{{ .Release.Name }}-prometheus-cache:9092" +# Comment out above line, uncomment below, and set useGRPCExporter to false +# to switch to HTTP metric pushing (less efficient) +# prometheusPushAddresses: +# - "http://{{ .Release.Name }}-prometheus-cache:9091/metrics" +{{- end -}} + +{{- define "metricsd-thanos-config-template" -}} +profile: "prometheus" +prometheusQueryAddress: "http://{{ .Release.Name }}-thanos-query-http:10902" +alertmanagerApiURL: "http://{{ .Release.Name }}-alertmanager:9093/api/v2" +prometheusConfigServiceURL: "http://{{ .Release.Name }}-prometheus-configurer:9100/v1" +alertmanagerConfigServiceURL: "http://{{ .Release.Name }}-alertmanager-configurer:9101/v1" +{{- end -}} + +{{- define "metricsd-config-template" -}} +profile: "prometheus" +prometheusQueryAddress: "http://{{ .Release.Name }}-prometheus:9090" +alertmanagerApiURL: "http://{{ .Release.Name }}-alertmanager:9093/api/v2" +prometheusConfigServiceURL: "http://{{ .Release.Name }}-prometheus-configurer:9100/v1" +alertmanagerConfigServiceURL: "http://{{ .Release.Name }}-alertmanager-configurer:9101/v1" +{{- end -}} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/charts/secrets/templates/configs-orc8r.secret.yaml b/orc8r/cloud/helm/orc8r/charts/secrets/templates/configs-orc8r.secret.yaml index eb39073b2f44..6339575f9dba 100644 --- a/orc8r/cloud/helm/orc8r/charts/secrets/templates/configs-orc8r.secret.yaml +++ b/orc8r/cloud/helm/orc8r/charts/secrets/templates/configs-orc8r.secret.yaml @@ -21,21 +21,19 @@ metadata: {{ include "labels" . | indent 4 }} data: {{- if .Values.secret.configs.enabled }} -{{ $thanos_enabled := .Values.thanos_enabled}} -{{- range $key, $value := .Values.secret.configs.orc8r }} - # Set metricsd.yml explicitly if thanos is enabled - {{- if eq $key "metricsd.yml" }} - {{- if $thanos_enabled }} - {{ $value = ` - profile: "prometheus" - - prometheusQueryAddress: "http://{{ .Release.name }}-thanos-query-http:10902" +# Template the defaults (metrics, orchestrator) first, will be overriden if +# passed in through values file +{{ $orchestratorTemplate := include "orchestrator-config-template" .}} + orchestrator.yml: {{ $orchestratorTemplate | b64enc | quote }} +{{- if .Values.thanos_enabled }} +{{ $metricsdTemplate := include "metricsd-thanos-config-template" .}} + metricsd.yml: {{ $metricsdTemplate | b64enc | quote }} +{{- else}} +{{ $metricsdTemplate := include "metricsd-config-template" .}} + metricsd.yml: {{ $metricsdTemplate | b64enc | quote }} +{{- end}} - alertmanagerApiURL: "http://{{ .Release.name }}-alertmanager:9093/api/v2" - prometheusConfigServiceURL: "http://{{ .Release.name }}-prometheus-configurer:9100/v1" - alertmanagerConfigServiceURL: "http://{{ .Release.name }}-alertmanager-configurer:9101/v1"` }} - {{- end }} - {{- end }} +{{- range $key, $value := .Values.secret.configs.orc8r }} {{ $key }}: {{ $value | b64enc | quote }} {{- end }} {{- end }} diff --git a/orc8r/cloud/helm/orc8r/charts/secrets/values.yaml b/orc8r/cloud/helm/orc8r/charts/secrets/values.yaml index 49f29c8d32e1..a21418831140 100644 --- a/orc8r/cloud/helm/orc8r/charts/secrets/values.yaml +++ b/orc8r/cloud/helm/orc8r/charts/secrets/values.yaml @@ -50,27 +50,10 @@ secret: configs: enabled: true orc8r: - metricsd.yml: |- - profile: "prometheus" - - prometheusQueryAddress: "http://orc8r-prometheus:9090" - - alertmanagerApiURL: "http://orc8r-alertmanager:9093/api/v2" - prometheusConfigServiceURL: "http://orc8r-prometheus-configurer:9100/v1" - alertmanagerConfigServiceURL: "http://orc8r-alertmanager-configurer:9101/v1" - elastic.yml: |- elasticHost: "elasticsearch-master" elasticPort: 9200 - orchestrator.yml: |- - useGRPCExporter: true - prometheusGRPCPushAddress: "orc8r-prometheus-cache:9092" - # Comment out above line, uncomment below, and set useGRPCExporter to false - # to switch to HTTP metric pushing (less efficient) - # prometheusPushAddresses: - # - "http://orc8r-prometheus-cache:9091/metrics" - # cwf: # key: value diff --git a/orc8r/cloud/helm/orc8r/requirements.lock b/orc8r/cloud/helm/orc8r/requirements.lock index a6db0f9fbfd6..ec05fd80dcb2 100644 --- a/orc8r/cloud/helm/orc8r/requirements.lock +++ b/orc8r/cloud/helm/orc8r/requirements.lock @@ -1,10 +1,10 @@ dependencies: - name: secrets repository: "" - version: 0.1.8 + version: 0.1.9 - name: metrics repository: "" - version: 1.4.19 + version: 1.4.20 - name: nms repository: "" version: 0.1.7 @@ -13,6 +13,6 @@ dependencies: version: 0.1.10 - name: orc8rlib repository: file://../orc8rlib - version: 0.1.1 -digest: sha256:2b29530032b5da56f56e70b64a36594e01fd61f0074cc8e48a520bc5e7a79d58 -generated: "2020-10-28T16:07:03.37778-07:00" + version: 0.1.2 +digest: sha256:d7588766e39b87cce8272810870e5fc88538a1b65aa008b623eabe77a93c52c4 +generated: "2020-12-14T10:08:10.149396-08:00" diff --git a/orc8r/cloud/helm/orc8r/requirements.yaml b/orc8r/cloud/helm/orc8r/requirements.yaml index 1d2473a3cd37..6cbea88c00fc 100644 --- a/orc8r/cloud/helm/orc8r/requirements.yaml +++ b/orc8r/cloud/helm/orc8r/requirements.yaml @@ -11,7 +11,7 @@ dependencies: - name: secrets - version: 0.1.8 + version: 0.1.9 repository: "" condition: secrets.create - name: metrics diff --git a/orc8r/cloud/helm/orc8r/templates/accessd.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/accessd.deployment.yaml new file mode 100644 index 000000000000..d6dfec4c2ec6 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/accessd.deployment.yaml @@ -0,0 +1,49 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "accessd.deployment") -}} +{{- define "accessd.deployment" -}} +metadata: + name: orc8r-accessd + labels: + app.kubernetes.io/component: accessd +spec: + selector: + matchLabels: + app.kubernetes.io/component: accessd + template: + metadata: + labels: + app.kubernetes.io/component: accessd + spec: + containers: + - +{{ include "orc8rlib.container" (list . "accessd.container")}} +{{- end -}} +{{- define "accessd.container" -}} +name: accessd +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/accessd", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9091 +livenessProbe: + tcpSocket: + port: 9091 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9091 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/accessd.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/accessd.pdb.yaml new file mode 100644 index 000000000000..b96f4f0cc967 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/accessd.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "accessd.pdb") -}} +{{- define "accessd.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-accessd + labels: + app.kubernetes.io/component: accessd +spec: + selector: + matchLabels: + app.kubernetes.io/component: accessd +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/accessd.service.yaml b/orc8r/cloud/helm/orc8r/templates/accessd.service.yaml index de67b7b20f7e..c94c81aee84f 100644 --- a/orc8r/cloud/helm/orc8r/templates/accessd.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/accessd.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: accessd ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/bootstrapper.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/bootstrapper.deployment.yaml new file mode 100644 index 000000000000..041e1ee6df1f --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/bootstrapper.deployment.yaml @@ -0,0 +1,49 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "bootstrapper.deployment") -}} +{{- define "bootstrapper.deployment" -}} +metadata: + name: orc8r-bootstrapper + labels: + app.kubernetes.io/component: bootstrapper +spec: + selector: + matchLabels: + app.kubernetes.io/component: bootstrapper + template: + metadata: + labels: + app.kubernetes.io/component: bootstrapper + spec: + containers: + - +{{ include "orc8rlib.container" (list . "bootstrapper.container")}} +{{- end -}} +{{- define "bootstrapper.container" -}} +name: bootstrapper +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/bootstrapper", "-cak=/var/opt/magma/certs/bootstrapper.key", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9088 +livenessProbe: + tcpSocket: + port: 9088 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9088 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/bootstrapper.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/bootstrapper.pdb.yaml new file mode 100644 index 000000000000..59bbc55e5375 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/bootstrapper.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "bootstrapper.pdb") -}} +{{- define "bootstrapper.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-bootstrapper + labels: + app.kubernetes.io/component: bootstrapper +spec: + selector: + matchLabels: + app.kubernetes.io/component: bootstrapper +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/bootstrapper.service.yaml b/orc8r/cloud/helm/orc8r/templates/bootstrapper.service.yaml index f9bd8be9023f..0357592ac6e5 100644 --- a/orc8r/cloud/helm/orc8r/templates/bootstrapper.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/bootstrapper.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: bootstrapper ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/certifier.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/certifier.deployment.yaml new file mode 100644 index 000000000000..870025c0e6a3 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/certifier.deployment.yaml @@ -0,0 +1,51 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "certifier.deployment") -}} +{{- define "certifier.deployment" -}} +metadata: + name: orc8r-certifier + labels: + app.kubernetes.io/component: certifier +spec: + selector: + matchLabels: + app.kubernetes.io/component: certifier + template: + metadata: + labels: + app.kubernetes.io/component: certifier + spec: + containers: + - +{{ include "orc8rlib.container" (list . "certifier.container")}} +{{- end -}} +{{- define "certifier.container" -}} +name: certifier +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/certifier", "-cac=/var/opt/magma/certs/certifier.pem", + "-cak=/var/opt/magma/certs/certifier.key", "-vpnc=/var/opt/magma/certs/vpn_ca.crt", "-vpnk=/var/opt/magma/certs/vpn_ca.key", + "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9086 +livenessProbe: + tcpSocket: + port: 9086 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9086 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/certifier.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/certifier.pdb.yaml new file mode 100644 index 000000000000..60f342c247a8 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/certifier.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "certifier.pdb") -}} +{{- define "certifier.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-certifier + labels: + app.kubernetes.io/component: certifier +spec: + selector: + matchLabels: + app.kubernetes.io/component: certifier +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/certifier.service.yaml b/orc8r/cloud/helm/orc8r/templates/certifier.service.yaml index 9a7b3185f27a..8c544c718b3b 100644 --- a/orc8r/cloud/helm/orc8r/templates/certifier.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/certifier.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: certifier ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/configurator.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/configurator.deployment.yaml new file mode 100644 index 000000000000..d8a07ef3192f --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/configurator.deployment.yaml @@ -0,0 +1,49 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "configurator.deployment") -}} +{{- define "configurator.deployment" -}} +metadata: + name: orc8r-configurator + labels: + app.kubernetes.io/component: configurator +spec: + selector: + matchLabels: + app.kubernetes.io/component: configurator + template: + metadata: + labels: + app.kubernetes.io/component: configurator + spec: + containers: + - +{{ include "orc8rlib.container" (list . "configurator.container")}} +{{- end -}} +{{- define "configurator.container" -}} +name: configurator +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/configurator", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9108 +livenessProbe: + tcpSocket: + port: 9108 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9108 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/configurator.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/configurator.pdb.yaml new file mode 100644 index 000000000000..fceed06077b5 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/configurator.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "configurator.pdb") -}} +{{- define "configurator.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-configurator + labels: + app.kubernetes.io/component: configurator +spec: + selector: + matchLabels: + app.kubernetes.io/component: configurator +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/configurator.service.yaml b/orc8r/cloud/helm/orc8r/templates/configurator.service.yaml index cac1d1c28f3d..4f7efc67d963 100644 --- a/orc8r/cloud/helm/orc8r/templates/configurator.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/configurator.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: configurator ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/controller.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/controller.deployment.yaml deleted file mode 100644 index 957d2977f50a..000000000000 --- a/orc8r/cloud/helm/orc8r/templates/controller.deployment.yaml +++ /dev/null @@ -1,133 +0,0 @@ -{{/* -Copyright 2020 The Magma Authors. - -This source code is licensed under the BSD-style license found in the -LICENSE file in the root directory of this source tree. - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} -{{- $serviceName := print .Release.Name "-controller" -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $serviceName }} - labels: - app.kubernetes.io/component: controller -{{ include "labels" . | indent 4 }} -{{ include "orc8r-app-labels" . | indent 4 }} -spec: - replicas: {{ .Values.controller.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: controller - template: - metadata: - labels: - app.kubernetes.io/component: controller - annotations: - {{- with .Values.controller.podAnnotations }} -{{ toYaml . | indent 8 }} - {{- end }} - spec: - serviceAccountName: {{ .Release.Name }}-service-reader - {{- with .Values.controller.nodeSelector }} - nodeSelector: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.controller.tolerations }} - tolerations: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.controller.affinity }} - affinity: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: -{{ toYaml . | trimSuffix "\n" | indent 8 }} - {{- end }} - volumes: - - name: certs - secret: - secretName: {{ required "secret.certs must be provided" .Values.secret.certs }} - - name: envdir - secret: - secretName: {{ required "secret.envdir must be provided" .Values.secret.envdir }} - {{- if .Values.secret.configs }} - {{- range $module, $secretName := .Values.secret.configs }} - - name: {{ $secretName }}-{{ $module }} - secret: - secretName: {{ $secretName }} - {{- end }} - {{- else }} - - name: "empty-configs" - emptyDir: {} - {{- end }} - containers: - - name: {{ $serviceName }} - image: {{ required "controller.image.repository must be provided" .Values.controller.image.repository }}:{{ .Values.controller.image.tag }} - imagePullPolicy: {{ .Values.controller.image.pullPolicy }} - volumeMounts: - {{- range tuple "certs" "envdir" }} - - name: {{ . }} - mountPath: /var/opt/magma/{{ . }} - readOnly: true - {{- end }} - {{- if .Values.secret.configs }} - {{- range $module, $secretName := .Values.secret.configs }} - - name: {{ $secretName }}-{{ $module }} - mountPath: {{ print "/var/opt/magma/configs/" $module }} - readOnly: true - {{- end }} - {{- else }} - - name: "empty-configs" - mountPath: /var/opt/magma/configs - readOnly: true - {{- end }} - ports: - {{- with .Values.controller.service }} - - containerPort: {{ .targetPort }} - {{- range $_, $port := untilStep (.portStart | int) (.portEnd | add1 | int) 1 }} - - containerPort: {{ $port }} - {{- end }} - {{- end }} - - name: grpc - containerPort: 9180 - env: - - name: DATABASE_SOURCE - valueFrom: - secretKeyRef: - name: {{ $serviceName }} - key: {{ .Values.controller.spec.database.driver }}.connstr - - name: SQL_DRIVER - value: {{ .Values.controller.spec.database.driver }} - - name: SQL_DIALECT - value: {{ .Values.controller.spec.database.sql_dialect }} - - name: SERVICE_HOSTNAME - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: SERVICE_REGISTRY_MODE - value: {{ .Values.controller.spec.service_registry.mode }} - - name: HELM_RELEASE_NAME - value: {{ .Release.Name }} - - name: SERVICE_REGISTRY_NAMESPACE - value: {{ .Release.Namespace }} - livenessProbe: - tcpSocket: - port: 9081 - initialDelaySeconds: 10 - periodSeconds: 30 - # Readiness probe prevents service registry mode 'k8s' from working - # properly - #readinessProbe: - # tcpSocket: - # port: 9081 - # initialDelaySeconds: 5 - # periodSeconds: 10 - resources: -{{ toYaml .Values.controller.resources | indent 12 }} diff --git a/orc8r/cloud/helm/orc8r/templates/controller.service.yaml b/orc8r/cloud/helm/orc8r/templates/controller.service.yaml deleted file mode 100644 index 5448e0c21d48..000000000000 --- a/orc8r/cloud/helm/orc8r/templates/controller.service.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2020 The Magma Authors. - -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-controller - labels: - app.kubernetes.io/component: controller -{{ include "labels" . | indent 4 }} -{{ include "orc8r-labels" . | indent 4 }} - {{- with .Values.controller.service.labels }} -{{ toYaml . | indent 4}} - {{- end}} - {{- with .Values.controller.service.annotations }} - annotations: -{{ toYaml . | indent 4}} - {{- end }} -spec: - selector: - app.kubernetes.io/component: controller - type: {{ .Values.controller.service.type }} - ports: - {{- with .Values.controller.service }} - - name: web - port: {{ .port }} - targetPort: {{ .targetPort }} - {{- range $i, $port := untilStep (.portStart | int) (.portEnd | add1 | int) 1 }} - - name: port-{{ $i }} - port: {{ $port }} - targetPort: {{ $port }} - {{- end }} - {{- end }} diff --git a/orc8r/cloud/helm/orc8r/templates/device.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/device.deployment.yaml new file mode 100644 index 000000000000..4705894c6dcb --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/device.deployment.yaml @@ -0,0 +1,49 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "device.deployment") -}} +{{- define "device.deployment" -}} +metadata: + name: orc8r-device + labels: + app.kubernetes.io/component: device +spec: + selector: + matchLabels: + app.kubernetes.io/component: device + template: + metadata: + labels: + app.kubernetes.io/component: device + spec: + containers: + - +{{ include "orc8rlib.container" (list . "device.container")}} +{{- end -}} +{{- define "device.container" -}} +name: device +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/device", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9106 +livenessProbe: + tcpSocket: + port: 9106 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9106 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/device.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/device.pdb.yaml new file mode 100644 index 000000000000..dc7a8df7a4e1 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/device.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "device.pdb") -}} +{{- define "device.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-device + labels: + app.kubernetes.io/component: device +spec: + selector: + matchLabels: + app.kubernetes.io/component: device +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/device.service.yaml b/orc8r/cloud/helm/orc8r/templates/device.service.yaml index 7cc9857bf85a..76409984455f 100644 --- a/orc8r/cloud/helm/orc8r/templates/device.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/device.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: device ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/directoryd.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/directoryd.deployment.yaml new file mode 100644 index 000000000000..39893681e590 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/directoryd.deployment.yaml @@ -0,0 +1,49 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "directoryd.deployment") -}} +{{- define "directoryd.deployment" -}} +metadata: + name: orc8r-directoryd + labels: + app.kubernetes.io/component: directoryd +spec: + selector: + matchLabels: + app.kubernetes.io/component: directoryd + template: + metadata: + labels: + app.kubernetes.io/component: directoryd + spec: + containers: + - +{{ include "orc8rlib.container" (list . "directoryd.container")}} +{{- end -}} +{{- define "directoryd.container" -}} +name: directoryd +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/directoryd", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9100 +livenessProbe: + tcpSocket: + port: 9100 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9100 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/directoryd.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/directoryd.pdb.yaml new file mode 100644 index 000000000000..964393216eeb --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/directoryd.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "directoryd.pdb") -}} +{{- define "directoryd.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-directoryd + labels: + app.kubernetes.io/component: directoryd +spec: + selector: + matchLabels: + app.kubernetes.io/component: directoryd +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/directoryd.service.yaml b/orc8r/cloud/helm/orc8r/templates/directoryd.service.yaml index 7fc21f722a11..a70cd19eb57e 100644 --- a/orc8r/cloud/helm/orc8r/templates/directoryd.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/directoryd.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: directoryd ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/dispatcher.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/dispatcher.deployment.yaml new file mode 100644 index 000000000000..87d9555edf37 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/dispatcher.deployment.yaml @@ -0,0 +1,49 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "dispatcher.deployment") -}} +{{- define "dispatcher.deployment" -}} +metadata: + name: orc8r-dispatcher + labels: + app.kubernetes.io/component: dispatcher +spec: + selector: + matchLabels: + app.kubernetes.io/component: dispatcher + template: + metadata: + labels: + app.kubernetes.io/component: dispatcher + spec: + containers: + - +{{ include "orc8rlib.container" (list . "dispatcher.container")}} +{{- end -}} +{{- define "dispatcher.container" -}} +name: dispatcher +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/dispatcher", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9096 +livenessProbe: + tcpSocket: + port: 9096 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9096 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/dispatcher.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/dispatcher.pdb.yaml new file mode 100644 index 000000000000..b99a65bc7642 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/dispatcher.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "dispatcher.pdb") -}} +{{- define "dispatcher.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-dispatcher + labels: + app.kubernetes.io/component: dispatcher +spec: + selector: + matchLabels: + app.kubernetes.io/component: dispatcher +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/dispatcher.service.yaml b/orc8r/cloud/helm/orc8r/templates/dispatcher.service.yaml index 87a1afff6c19..c6d717fbd781 100644 --- a/orc8r/cloud/helm/orc8r/templates/dispatcher.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/dispatcher.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: dispatcher ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/metricsd.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/metricsd.deployment.yaml new file mode 100644 index 000000000000..e287318bd022 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/metricsd.deployment.yaml @@ -0,0 +1,51 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "metricsd.deployment") -}} +{{- define "metricsd.deployment" -}} +metadata: + name: orc8r-metricsd + labels: + app.kubernetes.io/component: metricsd +spec: + selector: + matchLabels: + app.kubernetes.io/component: metricsd + template: + metadata: + labels: + app.kubernetes.io/component: metricsd + spec: + containers: + - +{{ include "orc8rlib.container" (list . "metricsd.container")}} +{{- end -}} +{{- define "metricsd.container" -}} +name: metricsd +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/metricsd", "-run_echo_server=true", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9084 + - name: http + containerPort: 10084 +livenessProbe: + tcpSocket: + port: 9084 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9084 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/metricsd.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/metricsd.pdb.yaml new file mode 100644 index 000000000000..52fa9be539d5 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/metricsd.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "metricsd.pdb") -}} +{{- define "metricsd.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-metricsd + labels: + app.kubernetes.io/component: metricsd +spec: + selector: + matchLabels: + app.kubernetes.io/component: metricsd +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/metricsd.service.yaml b/orc8r/cloud/helm/orc8r/templates/metricsd.service.yaml index 1e34be133a41..b41b0579a33b 100644 --- a/orc8r/cloud/helm/orc8r/templates/metricsd.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/metricsd.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: metricsd ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/obsidian.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/obsidian.deployment.yaml new file mode 100644 index 000000000000..62b405fd031a --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/obsidian.deployment.yaml @@ -0,0 +1,51 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "obsidian.deployment") -}} +{{- define "obsidian.deployment" -}} +metadata: + name: orc8r-obsidian + labels: + app.kubernetes.io/component: obsidian +spec: + selector: + matchLabels: + app.kubernetes.io/component: obsidian + template: + metadata: + labels: + app.kubernetes.io/component: obsidian + spec: + containers: + - +{{ include "orc8rlib.container" (list . "obsidian.container")}} +{{- end -}} +{{- define "obsidian.container" -}} +name: obsidian +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/obsidian", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9093 + - name: http + containerPort: 9081 +livenessProbe: + tcpSocket: + port: 9081 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9081 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/obsidian.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/obsidian.pdb.yaml new file mode 100644 index 000000000000..9b24b8597abb --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/obsidian.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "obsidian.pdb") -}} +{{- define "obsidian.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-obsidian + labels: + app.kubernetes.io/component: obsidian +spec: + selector: + matchLabels: + app.kubernetes.io/component: obsidian +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/obsidian.service.yaml b/orc8r/cloud/helm/orc8r/templates/obsidian.service.yaml index ffa3841eb5cc..07b64804ab16 100644 --- a/orc8r/cloud/helm/orc8r/templates/obsidian.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/obsidian.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: obsidian ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/orchestrator.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/orchestrator.deployment.yaml new file mode 100644 index 000000000000..8f50f7aa5ed1 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/orchestrator.deployment.yaml @@ -0,0 +1,51 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "orchestrator.deployment") -}} +{{- define "orchestrator.deployment" -}} +metadata: + name: orc8r-orchestrator + labels: + app.kubernetes.io/component: orchestrator +spec: + selector: + matchLabels: + app.kubernetes.io/component: orchestrator + template: + metadata: + labels: + app.kubernetes.io/component: orchestrator + spec: + containers: + - +{{ include "orc8rlib.container" (list . "orchestrator.container")}} +{{- end -}} +{{- define "orchestrator.container" -}} +name: orchestrator +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/orchestrator","-run_echo_server=true", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9112 + - name: http + containerPort: 10112 +livenessProbe: + tcpSocket: + port: 9112 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9112 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/orchestrator.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/orchestrator.pdb.yaml new file mode 100644 index 000000000000..fe91f1e7773c --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/orchestrator.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "orchestrator.pdb") -}} +{{- define "orchestrator.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-orchestrator + labels: + app.kubernetes.io/component: orchestrator +spec: + selector: + matchLabels: + app.kubernetes.io/component: orchestrator +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/orchestrator.service.yaml b/orc8r/cloud/helm/orc8r/templates/orchestrator.service.yaml index 44a481d71174..25f4fd669aec 100644 --- a/orc8r/cloud/helm/orc8r/templates/orchestrator.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/orchestrator.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: orchestrator ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/service_registry.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/service_registry.deployment.yaml new file mode 100644 index 000000000000..8bb7463aeca6 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/service_registry.deployment.yaml @@ -0,0 +1,50 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "service_registry.deployment") -}} +{{- define "service_registry.deployment" -}} +metadata: + name: orc8r-service-registry + labels: + app.kubernetes.io/component: service_registry +spec: + selector: + matchLabels: + app.kubernetes.io/component: service_registry + template: + metadata: + labels: + app.kubernetes.io/component: service_registry + spec: + serviceAccountName: {{ .Release.Name }}-service-reader + containers: + - +{{ include "orc8rlib.container" (list . "service_registry.container")}} +{{- end -}} +{{- define "service_registry.container" -}} +name: service-registry +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/service_registry", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9180 +livenessProbe: + tcpSocket: + port: 9180 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9180 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/service_registry.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/service_registry.pdb.yaml new file mode 100644 index 000000000000..2aceee2d1d02 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/service_registry.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "service_registry.pdb") -}} +{{- define "service_registry.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-service-registry + labels: + app.kubernetes.io/component: service_registry +spec: + selector: + matchLabels: + app.kubernetes.io/component: service_registry +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/service_registry.service.yaml b/orc8r/cloud/helm/orc8r/templates/service_registry.service.yaml index 9df65263c9b7..9a143f3a5fae 100644 --- a/orc8r/cloud/helm/orc8r/templates/service_registry.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/service_registry.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: service_registry ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/state.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/state.deployment.yaml new file mode 100644 index 000000000000..78f2be454ecc --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/state.deployment.yaml @@ -0,0 +1,49 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "state.deployment") -}} +{{- define "state.deployment" -}} +metadata: + name: orc8r-state + labels: + app.kubernetes.io/component: state +spec: + selector: + matchLabels: + app.kubernetes.io/component: state + template: + metadata: + labels: + app.kubernetes.io/component: state + spec: + containers: + - +{{ include "orc8rlib.container" (list . "state.container")}} +{{- end -}} +{{- define "state.container" -}} +name: state +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/state", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9105 +livenessProbe: + tcpSocket: + port: 9105 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9105 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/state.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/state.pdb.yaml new file mode 100644 index 000000000000..bbf18691e1b1 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/state.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "state.pdb") -}} +{{- define "state.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-state + labels: + app.kubernetes.io/component: state +spec: + selector: + matchLabels: + app.kubernetes.io/component: state +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/state.service.yaml b/orc8r/cloud/helm/orc8r/templates/state.service.yaml index ccdd62c9d327..0c72051b4ec9 100644 --- a/orc8r/cloud/helm/orc8r/templates/state.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/state.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: state ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/streamer.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/streamer.deployment.yaml new file mode 100644 index 000000000000..4a289cf67524 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/streamer.deployment.yaml @@ -0,0 +1,49 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "streamer.deployment") -}} +{{- define "streamer.deployment" -}} +metadata: + name: orc8r-streamer + labels: + app.kubernetes.io/component: streamer +spec: + selector: + matchLabels: + app.kubernetes.io/component: streamer + template: + metadata: + labels: + app.kubernetes.io/component: streamer + spec: + containers: + - +{{ include "orc8rlib.container" (list . "streamer.container")}} +{{- end -}} +{{- define "streamer.container" -}} +name: streamer +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/streamer", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9082 +livenessProbe: + tcpSocket: + port: 9082 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9082 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/streamer.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/streamer.pdb.yaml new file mode 100644 index 000000000000..e3d7bd200921 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/streamer.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "streamer.pdb") -}} +{{- define "streamer.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-streamer + labels: + app.kubernetes.io/component: streamer +spec: + selector: + matchLabels: + app.kubernetes.io/component: streamer +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/streamer.service.yaml b/orc8r/cloud/helm/orc8r/templates/streamer.service.yaml index ecc268988511..da9f277cadbf 100644 --- a/orc8r/cloud/helm/orc8r/templates/streamer.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/streamer.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: streamer ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/templates/tenants.deployment.yaml b/orc8r/cloud/helm/orc8r/templates/tenants.deployment.yaml new file mode 100644 index 000000000000..95a4c5ff9955 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/tenants.deployment.yaml @@ -0,0 +1,51 @@ +{{/* +# Copyright 2020 The Magma Authors. + +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- include "orc8rlib.deployment" (list . "tenants.deployment") -}} +{{- define "tenants.deployment" -}} +metadata: + name: orc8r-tenants + labels: + app.kubernetes.io/component: tenants +spec: + selector: + matchLabels: + app.kubernetes.io/component: tenants + template: + metadata: + labels: + app.kubernetes.io/component: tenants + spec: + containers: + - +{{ include "orc8rlib.container" (list . "tenants.container")}} +{{- end -}} +{{- define "tenants.container" -}} +name: tenants +command: ["/usr/bin/envdir"] +args: ["/var/opt/magma/envdir", "/var/opt/magma/bin/tenants", "-run_echo_server=true", "-logtostderr=true", "-v=0"] +ports: + - name: grpc + containerPort: 9110 + - name: http + containerPort: 10110 +livenessProbe: + tcpSocket: + port: 9110 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9110 + initialDelaySeconds: 5 + periodSeconds: 10 +{{- end -}} diff --git a/orc8r/cloud/helm/orc8r/templates/tenants.pdb.yaml b/orc8r/cloud/helm/orc8r/templates/tenants.pdb.yaml new file mode 100644 index 000000000000..9c4002f2f842 --- /dev/null +++ b/orc8r/cloud/helm/orc8r/templates/tenants.pdb.yaml @@ -0,0 +1,25 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- include "orc8rlib.pdb" (list . "tenants.pdb") -}} +{{- define "tenants.pdb" -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: orc8r-tenants + labels: + app.kubernetes.io/component: tenants +spec: + selector: + matchLabels: + app.kubernetes.io/component: tenants +{{- end }} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/tenants.service.yaml b/orc8r/cloud/helm/orc8r/templates/tenants.service.yaml index 91f5e4a97beb..21f031bcdf67 100644 --- a/orc8r/cloud/helm/orc8r/templates/tenants.service.yaml +++ b/orc8r/cloud/helm/orc8r/templates/tenants.service.yaml @@ -24,6 +24,8 @@ metadata: {{ toYaml . | indent 4}} {{- end }} spec: + selector: + app.kubernetes.io/component: tenants ports: - name: grpc port: 9180 diff --git a/orc8r/cloud/helm/orc8r/values.yaml b/orc8r/cloud/helm/orc8r/values.yaml index 8e5b10652ee7..49dcc65d5f80 100644 --- a/orc8r/cloud/helm/orc8r/values.yaml +++ b/orc8r/cloud/helm/orc8r/values.yaml @@ -44,6 +44,7 @@ metrics: # secrets sub-chart configuration. secrets: create: false + fullReleaseName: orc8r # Define which secrets should be mounted by pods. secret: diff --git a/orc8r/cloud/helm/orc8rlib/Chart.yaml b/orc8r/cloud/helm/orc8rlib/Chart.yaml index 257e17415794..b0b7085005b4 100644 --- a/orc8r/cloud/helm/orc8rlib/Chart.yaml +++ b/orc8r/cloud/helm/orc8rlib/Chart.yaml @@ -13,7 +13,7 @@ apiVersion: v2 appVersion: 1.0.0 description: A Helm library chart for orchestrator modules name: orc8rlib -version: 0.1.1 +version: 0.1.2 type: library engine: gotpl sources: diff --git a/orc8r/cloud/helm/orc8rlib/templates/_container.yaml b/orc8r/cloud/helm/orc8rlib/templates/_container.yaml new file mode 100644 index 000000000000..1dbc2c2ec3c0 --- /dev/null +++ b/orc8r/cloud/helm/orc8rlib/templates/_container.yaml @@ -0,0 +1,74 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- define "orc8rlib.container.tpl" -}} +name: orc8r-container +image: {{ required "controller.image.repository must be provided" .Values.controller.image.repository }}:{{ .Values.controller.image.tag }} +imagePullPolicy: {{ .Values.controller.image.pullPolicy }} +volumeMounts: + {{- range tuple "certs" "envdir" }} + - name: {{ . }} + mountPath: /var/opt/magma/{{ . }} + readOnly: true + {{- end }} + {{- if .Values.secret.configs }} + {{- range $module, $secretName := .Values.secret.configs }} + - name: {{ $secretName }}-{{ $module }} + mountPath: {{ print "/var/opt/magma/configs/" $module }} + readOnly: true + {{- end }} + {{- else }} + - name: "empty-configs" + mountPath: /var/opt/magma/configs + readOnly: true + {{- end }} +ports: + - name: http + containerPort: 8080 + - name: grpc + containerPort: 9180 +env: + - name: DATABASE_SOURCE + valueFrom: + secretKeyRef: + name: orc8r-controller + key: {{ .Values.controller.spec.database.driver }}.connstr + - name: SQL_DRIVER + value: {{ .Values.controller.spec.database.driver }} + - name: SQL_DIALECT + value: {{ .Values.controller.spec.database.sql_dialect }} + - name: SERVICE_HOSTNAME + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: SERVICE_REGISTRY_MODE + value: {{ .Values.controller.spec.service_registry.mode }} + - name: HELM_RELEASE_NAME + value: {{ .Release.Name }} + - name: SERVICE_REGISTRY_NAMESPACE + value: {{ .Release.Namespace }} +livenessProbe: + tcpSocket: + port: 9180 + initialDelaySeconds: 10 + periodSeconds: 30 +readinessProbe: + tcpSocket: + port: 9180 + initialDelaySeconds: 5 + periodSeconds: 10 +resources: +{{ toYaml .Values.controller.resources | indent 2 }} +{{- end -}} +{{- define "orc8rlib.container" -}} +{{- include "orc8rlib.util.merge" (append . "orc8rlib.container.tpl") | indent 8 -}} +{{- end -}} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8rlib/templates/_deployment.yaml b/orc8r/cloud/helm/orc8rlib/templates/_deployment.yaml new file mode 100644 index 000000000000..452b2f6ed975 --- /dev/null +++ b/orc8r/cloud/helm/orc8rlib/templates/_deployment.yaml @@ -0,0 +1,76 @@ +{{/* +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- define "orc8rlib.deployment.tpl" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: orc8r + labels: + app.kubernetes.io/component: orc8r +{{ include "default-labels" . | indent 4 }} +spec: + replicas: {{ .Values.controller.replicas }} + selector: + matchLabels: + app.kubernetes.io/component: orc8r +{{ include "default-selector-labels" . | indent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: orc8r +{{ include "default-selector-labels" . | indent 8 }} + annotations: + {{- with .Values.controller.podAnnotations }} +{{ toYaml . | indent 8 }} + {{- end }} + spec: + {{- with .Values.controller.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.controller.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.controller.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml . | trimSuffix "\n" | indent 8 }} + {{- end }} + volumes: + - name: certs + secret: + secretName: {{ required "secret.certs must be provided" .Values.secret.certs }} + - name: envdir + secret: + secretName: {{ required "secret.envdir must be provided" .Values.secret.envdir }} + {{- if .Values.secret.configs }} + {{- range $module, $secretName := .Values.secret.configs }} + - name: {{ $secretName }}-{{ $module }} + secret: + secretName: {{ $secretName }} + {{- end }} + {{- else }} + - name: "empty-configs" + emptyDir: {} + {{- end }} + containers: + - +{{ include "orc8rlib.container.tpl" . | indent 8 }} +{{- end -}} +{{- define "orc8rlib.deployment" -}} +{{- include "orc8rlib.util.merge" (append . "orc8rlib.deployment.tpl") -}} +{{- end -}} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8rlib/templates/_helpers.tpl b/orc8r/cloud/helm/orc8rlib/templates/_helpers.tpl index 3ea4571f5219..bf563a26a678 100644 --- a/orc8r/cloud/helm/orc8rlib/templates/_helpers.tpl +++ b/orc8r/cloud/helm/orc8rlib/templates/_helpers.tpl @@ -21,7 +21,7 @@ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} {{- end -}} {{/* Generate selector labels */}} -{{/* TODO: Add back instance, name labels after the controller is split */}} {{- define "default-selector-labels" -}} -app.kubernetes.io/component: controller +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} \ No newline at end of file diff --git a/orc8r/cloud/helm/orc8r/templates/controller.pdb.yaml b/orc8r/cloud/helm/orc8rlib/templates/_pdb.yaml similarity index 72% rename from orc8r/cloud/helm/orc8r/templates/controller.pdb.yaml rename to orc8r/cloud/helm/orc8rlib/templates/_pdb.yaml index cc556fb39782..ac74193dcf70 100644 --- a/orc8r/cloud/helm/orc8r/templates/controller.pdb.yaml +++ b/orc8r/cloud/helm/orc8rlib/templates/_pdb.yaml @@ -10,15 +10,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "orc8rlib.pdb.tpl" -}} {{- if and .Values.controller.podDisruptionBudget.enabled (gt .Values.controller.replicas 1.0 )}} apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: - name: {{ .Release.Name }}-controller + name: orc8r labels: - app.kubernetes.io/component: controller -{{ include "labels" . | indent 4 }} -{{ include "orc8r-app-labels" . | indent 4 }} + app.kubernetes.io/component: orc8r +{{ include "default-labels" . | indent 4 }} spec: {{- with .Values.controller.podDisruptionBudget.minAvailable }} minAvailable: {{ . }} @@ -28,5 +28,10 @@ spec: {{- end }} selector: matchLabels: - app.kubernetes.io/component: controller + app.kubernetes.io/component: orc8r +{{ include "default-selector-labels" . | indent 6 }} {{- end }} +{{- end }} +{{- define "orc8rlib.pdb" -}} +{{- include "orc8rlib.util.merge" (append . "orc8rlib.pdb.tpl") -}} +{{- end -}} diff --git a/orc8r/gateway/configs/state.yml b/orc8r/gateway/configs/state.yml index 41081b65177d..53a2bb790f24 100644 --- a/orc8r/gateway/configs/state.yml +++ b/orc8r/gateway/configs/state.yml @@ -13,6 +13,8 @@ log_level: INFO +sync_interval: 60 + #state_protos: # - proto_file: - file to load proto from # proto_msg: - msg to load from proto file diff --git a/orc8r/gateway/python/magma/ctraced/command_builder.py b/orc8r/gateway/python/magma/ctraced/command_builder.py new file mode 100644 index 000000000000..621284ec7dfd --- /dev/null +++ b/orc8r/gateway/python/magma/ctraced/command_builder.py @@ -0,0 +1,172 @@ +""" +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from abc import ABC, abstractmethod +from typing import List + + +class TraceBuildException(Exception): + pass + + +class TraceBuilder(ABC): + """Builds commands to run call tracing. + + TODO(andreilee): Support tracing for subscriber + TODO(andreilee): Support tracing by 3gpp protocol + """ + def __init__(self): + super().__init__() + + @abstractmethod + def build_trace_command( + self, + interfaces: List[str], + max_filesize: int, + output_filename: str, + ) -> List[str]: + """Builds command to start a trace. Does not execute the command. + + Builds a command to start a trace using the specified arguments. + It is not guaranteed that the specific implementation of TraceBuilder + will be able to satisfy all the constraints placed by the specified + arguments. Check the docstring of the specific implementation for + additional details. + + Args: + interfaces: Interfaces to capture the trace on. Specify "any" to + capture on all interfaces. + max_filesize: Max capture filesize specified in KiB. Capture will + stop after reaching the specified size. Value of -1 specifies + no limit. + output_filename: Where the output file will be saved. + + Returns: + A command that can be run by subprocess.Popen to start the trace. + + Example: + ["tshark", "-i", "eth0", "-a", "filesize:4000", "-w", "out.pcap"] + + Raises: + TraceBuildException: Could not successfully build a command to + start a call trace with the specified arguments + """ + pass + + +class TSharkTraceBuilder(TraceBuilder): + """Builds commands to run call tracing with tshark. + + This is the most feature complete TraceBuilder. + """ + def __init__(self): + super().__init__() + + def build_trace_command( + self, + interfaces: List[str], + max_filesize: int, + output_filename: str, + ) -> List[str]: + command = ["tshark"] + + # Specify interfaces + if "any" in interfaces: + command.extend(["-ni", "any"]) + else: + for interface in interfaces: + command.extend(["-i", interface]) + + # Specify max filesize. tshark will terminate after it is reached. + if max_filesize != -1: + command.extend(["-a", "filesize:" + str(max_filesize)]) + + # Specify output file + command.extend(["-w", output_filename]) + + return command + + +class TcpdumpTraceBuilder(TraceBuilder): + """Builds commands to run call tracing with tcpdump. + + This is a less feature complete TraceBuilder, as tcpdump has less options + than a tool such as tshark. + """ + def __init__(self): + super().__init__() + + def build_trace_command( + self, + interfaces: List[str], + max_filesize: int, + output_filename: str, + ) -> List[str]: + """Builds command to start a trace using tcpdump. Does not execute it. + + SEE PARENT CLASS FOR DETAILS. + + Details here only specify argument restrictions. + + Argument Restrictions: + interfaces: Only one interface can be specified. + max_filesize: Only value of -1 supported (no limit). + output_filename: + + TODO(andreilee): Support max_filesize. + Add postrotate command to stop after max filesize + reached. + TODO(andreilee): Support multiple interfaces. + """ + command = ["tcpdump"] + + # Specify southbound interfaces + if len(interfaces) == 1: + command.extend(["-i", interfaces[0]]) + elif len(interfaces) > 1: + raise TraceBuildException("""Cannot start trace with tcpdump with + more than one interface specified""") + + # Currently unsupported. + # While a max filesize can be specified, tcpdump will rotate to new + # output files after the maximum is reached. + if max_filesize != -1: + raise TraceBuildException("""Cannot start trace with tcpdump with + max filesize""") + + # Specify output file + command.extend(["-w", self._trace_filename]) + + return command + + +def get_trace_builder(tool_name: str) -> TraceBuilder: + """Factory method for TraceBuilder. + + Args: + tool_name: Name of tool for capturing a call trace. + Only options allowed are ["tshark", "tcpdump"]. + + Returns: + The TraceBuilder implementation matching the specified tool name + + Raises: + TraceBuildException: If an unsupported tool name is specified. + """ + if tool_name == "tshark": + return TSharkTraceBuilder() + elif tool_name == "tcpdump": + return TcpdumpTraceBuilder() + raise TraceBuildException("Failed to create trace builder, " + "invalid tool name specified: {}" + .format(tool_name)) diff --git a/orc8r/gateway/python/magma/ctraced/main.py b/orc8r/gateway/python/magma/ctraced/main.py index f62002347e2c..d72f897aae7d 100644 --- a/orc8r/gateway/python/magma/ctraced/main.py +++ b/orc8r/gateway/python/magma/ctraced/main.py @@ -12,12 +12,19 @@ """ from magma.common.service import MagmaService +from .rpc_servicer import CtraceDRpcServicer +from .trace_manager import TraceManager from orc8r.protos.mconfig.mconfigs_pb2 import CtraceD def main(): """ main() for ctraced """ service = MagmaService('ctraced', CtraceD()) + trace_manager = TraceManager(service.config) + + ctraced_servicer = CtraceDRpcServicer(trace_manager) + ctraced_servicer.add_to_server(service.rpc_server) + # Run the service loop service.run() diff --git a/orc8r/gateway/python/magma/ctraced/rpc_servicer.py b/orc8r/gateway/python/magma/ctraced/rpc_servicer.py new file mode 100644 index 000000000000..5b013af179a6 --- /dev/null +++ b/orc8r/gateway/python/magma/ctraced/rpc_servicer.py @@ -0,0 +1,55 @@ +""" +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from orc8r.protos.ctraced_pb2_grpc import CallTraceServiceServicer,\ + add_CallTraceServiceServicer_to_server +from orc8r.protos.ctraced_pb2 import StartTraceRequest, StartTraceResponse,\ + EndTraceRequest, EndTraceResponse +from .trace_manager import TraceManager + + +class CtraceDRpcServicer(CallTraceServiceServicer): + """ + gRPC based server for CtraceD. + """ + + def __init__(self, trace_manager: TraceManager): + self._trace_mgr = trace_manager + pass + + def add_to_server(self, server): + """ + Add the servicer to a gRPC server + """ + add_CallTraceServiceServicer_to_server(self, server) + + def StartCallTrace( + self, + _request: StartTraceRequest, + _context, + ) -> StartTraceResponse: + success = self._trace_mgr.start_trace() + response = StartTraceResponse(success=success) + return response + + def EndCallTrace( + self, + _request: EndTraceRequest, + _context, + ) -> EndTraceResponse: + data = self._trace_mgr.end_trace() # type: bytes + response = EndTraceResponse( + success=True, + trace_content=data, + ) + return response diff --git a/orc8r/gateway/python/magma/ctraced/trace_manager.py b/orc8r/gateway/python/magma/ctraced/trace_manager.py new file mode 100644 index 000000000000..0bc2e1a26f2e --- /dev/null +++ b/orc8r/gateway/python/magma/ctraced/trace_manager.py @@ -0,0 +1,132 @@ +""" +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import errno +import logging +import os +import pathlib +import subprocess +import time +from subprocess import SubprocessError +from .command_builder import get_trace_builder + +_TRACE_FILE_NAME = "call_trace" +_TRACE_FILE_EXT = "pcap" +_MAX_FILESIZE = 4000 # ~ 4 MiB for a trace + + +class TraceManager: + """ + TraceManager is a wrapper for tshark/tcpdump specifically for starting and + stopping call/interface/subscriber traces. + + Only a single trace can be captured at a time. + """ + def __init__(self, config): + self._is_active = False # is call trace being captured + self._proc = None + self._trace_directory = config.get("trace_directory", + "/var/opt/magma/trace") # type: str + # Specify southbound interfaces + self._trace_interfaces = config.get("trace_interfaces", + ["eth0"]) # type: List[str] + + # Should specify absolute path of trace filename if trace is active + self._trace_filename = "" # type: str + + tool_name = config.get("trace_tool", "tshark") # type: str + self._trace_builder = get_trace_builder(tool_name) + + def start_trace(self) -> bool: + """Start a call trace. + + Captures all packets across the eth0 interface. + + Returns: + True if successfully started call trace + """ + if self._is_active: + logging.error("Failed to start trace: Trace already active") + return False + + # Example filename path: + # /var/opt/magma/trace/call_trace_1607358641.pcap + self._trace_filename = "{0}/{1}_{2}.{3}".format( + self._trace_directory, + _TRACE_FILE_NAME, + int(time.time()), + _TRACE_FILE_EXT) + + command = self._trace_builder.build_trace_command( + self._trace_interfaces, + _MAX_FILESIZE, + self._trace_filename) + + logging.info("Starting trace with tshark, command: [%s]", + ' '.join(command)) + + self._ensure_trace_directory_exists() + + # TODO(andreilee): Handle edge case where only one instance of the + # process can be running, and may have been started + # by something external as well. + try: + self._proc = subprocess.Popen(command) + except SubprocessError as e: + logging.error("Failed to start trace: %s", str(e)) + return False + + self._is_active = True + logging.info("Successfully started trace with tshark") + return True + + def end_trace(self) -> bytes: + """Ends call trace, if currently active. + + Returns: + Call trace file in bytes + """ + # If trace is active, then stop it + if self._is_active: + # If the process has ended, then _proc isn't None + self._proc.poll() + if self._proc.returncode is None: + self._proc.terminate() + + # Read trace data into bytes + with open(self._trace_filename, "rb") as trace_file: + data = trace_file.read() # type: bytes + + # Ensure the tmp trace file is deleted + self._ensure_tmp_file_deleted() + self._trace_filename = "" + + self._is_active = False + + # Everything cleaned up, return bytes + return data + + def _ensure_tmp_file_deleted(self): + """Ensure that tmp trace file is deleted. + + Uses exception handling rather than a check for file existence to avoid + TOCTTOU bug + """ + try: + os.remove(self._trace_filename) + except OSError as e: + if e.errno != errno.ENOENT: + logging.error("Error when deleting tmp trace file: %s", str(e)) + + def _ensure_trace_directory_exists(self) -> None: + pathlib.Path(self._trace_directory).mkdir(parents=True, exist_ok=True) diff --git a/orc8r/gateway/python/magma/state/state_replicator.py b/orc8r/gateway/python/magma/state/state_replicator.py index f57a8cd8fb15..4ced0bdc53b8 100644 --- a/orc8r/gateway/python/magma/state/state_replicator.py +++ b/orc8r/gateway/python/magma/state/state_replicator.py @@ -44,7 +44,8 @@ def __init__(self, service: MagmaService, garbage_collector: GarbageCollector, grpc_client_manager: GRPCClientManager): - super().__init__(DEFAULT_SYNC_INTERVAL, service.loop) + sync_interval = service.config.get('sync_interval', DEFAULT_SYNC_INTERVAL) + super().__init__(sync_interval, service.loop) self._service = service # Garbage collector to propagate deletions back to Orchestrator self._garbage_collector = garbage_collector @@ -69,6 +70,7 @@ def __init__(self, self._replication_iteration = 0 async def _run(self): + logging.debug("Check state") if not self._has_resync_completed: try: await self._resync() @@ -86,6 +88,7 @@ async def _run(self): GARBAGE_COLLECTION_ITERATION_INTERVAL: await self._garbage_collector.run_garbage_collection() self._replication_iteration = 0 + logging.debug("") async def _resync(self): states_to_sync = [] @@ -126,25 +129,40 @@ async def _collect_states_to_replicate(self): states_to_report = [] for redis_dict in self._redis_dicts: for key in redis_dict: + redis_state = redis_dict.get(key) device_id = make_scoped_device_id(key, redis_dict.state_scope) + in_mem_key = make_mem_key(device_id, redis_dict.redis_type) + if redis_state == None: + logging.debug("Content of key %s is empty, skipping", in_mem_key) + continue + redis_version = redis_dict.get_version(key) self._state_keys_from_current_iteration.add(in_mem_key) if in_mem_key in self._state_versions and \ self._state_versions[in_mem_key] == redis_version: + logging.debug("key %s already read on this iteration, skipping", in_mem_key) + continue + + try: + if redis_dict.state_format == PROTO_FORMAT: + state_to_serialize = MessageToDict(redis_state) + serialized_json_state = json.dumps(state_to_serialize) + else: + serialized_json_state = jsonpickle.encode(redis_state) + except Exception as e: # pylint: disable=broad-except + logging.error("Found bad state for %s for %s, not " + "replicating this state: %s", + key, device_id, e) continue - redis_state = redis_dict.get(key) - if redis_dict.state_format == PROTO_FORMAT: - state_to_serialize = MessageToDict(redis_state) - serialized_json_state = json.dumps(state_to_serialize) - else: - serialized_json_state = jsonpickle.encode(redis_state) state_proto = State(type=redis_dict.redis_type, deviceID=device_id, value=serialized_json_state.encode("utf-8"), version=redis_version) + logging.debug("key with version, %s contains: %s", in_mem_key, + serialized_json_state) states_to_report.append(state_proto) if len(states_to_report) == 0: diff --git a/orc8r/gateway/python/scripts/ctraced_cli.py b/orc8r/gateway/python/scripts/ctraced_cli.py new file mode 100644 index 000000000000..7a3a44b5b3e1 --- /dev/null +++ b/orc8r/gateway/python/scripts/ctraced_cli.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +""" +Copyright 2020 The Magma Authors. + +This source code is licensed under the BSD-style license found in the +LICENSE file in the root directory of this source tree. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import argparse +import textwrap + +from magma.common.rpc_utils import grpc_wrapper +from orc8r.protos import common_pb2 + +from orc8r.protos.ctraced_pb2 import StartTraceRequest, EndTraceRequest +from orc8r.protos.ctraced_pb2_grpc import CallTraceServiceStub + +@grpc_wrapper +def start_call_trace(client, args): + client.StartCallTrace(StartTraceRequest()) + + +@grpc_wrapper +def end_call_trace(client, args): + res = client.EndCallTrace(common_pb2.Void()) + print("Result of call trace: ", res) + + +def create_parser(): + """ + Creates the argparse parser with all the arguments. + """ + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description = textwrap.dedent('''\ + Management CLI for ctraced + -------------------------- + Use to start and end call traces. + Options are provided for the type of trace to record. + Only a single trace can be captured at a time. + ''')) + + # Add subcommands + subparsers = parser.add_subparsers(title='subcommands', dest='cmd') + + # Add StartCallTrace subcommand + parser_start_trace = subparsers.add_parser('start_call_trace', + help='Start a call trace') + trace_types = list(StartTraceRequest.TraceType.DESCRIPTOR.values_by_name) + supported_protocols =\ + list(StartTraceRequest.ProtocolName.DESCRIPTOR.values_by_name) + supported_interfaces =\ + list(StartTraceRequest.InterfaceName.DESCRIPTOR.values_by_name) + parser_start_trace.add_argument('--type', type=str, choices=trace_types, + help='Trace type', required=True) + parser_start_trace.add_argument('--imsi', type=str, choices=trace_types, + help='Trace type') + parser_start_trace.add_argument('--protocol', type=str, + choices=supported_protocols) + parser_start_trace.add_argument('--interface', type=str, + choices=supported_interfaces) + parser_start_trace.set_defaults(func=start_call_trace) + + # Add EndCallTrace subcommand + parser_end_trace = subparsers.add_parser('end_call_trace', + help='End a call trace') + parser_end_trace.set_defaults(func=end_call_trace) + + return parser + + +def main(): + parser = create_parser() + + # Parse the args + args = parser.parse_args() + if not args.cmd: + parser.print_usage() + exit(1) + + # Execute the subcommand function + args.func(args, CallTraceServiceStub, 'ctraced') + + +if __name__ == "__main__": + main() diff --git a/orc8r/gateway/python/setup.py b/orc8r/gateway/python/setup.py index c20cafe2a003..98c897c80006 100644 --- a/orc8r/gateway/python/setup.py +++ b/orc8r/gateway/python/setup.py @@ -42,6 +42,7 @@ ], scripts=[ 'scripts/checkin_cli.py', + 'scripts/ctraced_cli.py', 'scripts/directoryd_cli.py', 'scripts/generate_lighttpd_config.py', 'scripts/generate_nghttpx_config.py', diff --git a/orc8r/lib/go/registry/registry.go b/orc8r/lib/go/registry/registry.go index 7688e19240d3..44755786b2de 100644 --- a/orc8r/lib/go/registry/registry.go +++ b/orc8r/lib/go/registry/registry.go @@ -33,10 +33,10 @@ import ( const ( ServiceRegistryServiceName = "service_registry" - ServiceRegistryModeEnvVar = "SERVICE_REGISTRY_MODE" - DockerRegistryMode = "docker" - K8sRegistryMode = "k8s" - YamlRegistryMode = "yaml" + ServiceRegistryModeEnvVar = "SERVICE_REGISTRY_MODE" + DockerRegistryMode = "docker" + K8sRegistryMode = "k8s" + YamlRegistryMode = "yaml" HttpServerPort = 8080 GrpcServicePort = 9180 diff --git a/third_party/build/bin/asn1c_build.sh b/third_party/build/bin/asn1c_build.sh index ee9a0e0b56f3..650942a3e60f 100755 --- a/third_party/build/bin/asn1c_build.sh +++ b/third_party/build/bin/asn1c_build.sh @@ -22,10 +22,10 @@ set -e SCRIPT_DIR="$(dirname "$(realpath "$0")")" source "${SCRIPT_DIR}"/../lib/util.sh -COMMIT_DATE=20170324 +COMMIT_DATE=20190423 # index of the commit from a particular date, start from 0 COMMIT_INDEX=0 -COMMIT=224dc1f9 +COMMIT=f12568d6 # 0~ makes the version compatible with real version numbers # 0~20160721+c3~r43c4a295 < 0~20160721+c5~r43c4a295 < 0~20160722+c0~r43c4a295 < 0.1 ITERATION=0 @@ -49,7 +49,7 @@ WORK_DIR=/tmp/build-${PKGNAME} # The resulting package is placed in $OUTPUT_DIR # or in the cwd. if [ -z "$1" ]; then - OUTPUT_DIR=`pwd` + OUTPUT_DIR=$(pwd) else OUTPUT_DIR=$1 if [ ! -d "$OUTPUT_DIR" ]; then @@ -68,11 +68,10 @@ git clone https://gitlab.eurecom.fr/oai/asn1c.git cd asn1c git checkout ${COMMIT} . -# apply magma patches -# for p in ${OUTPUT_DIR}/*.patch; do patch -p1 < $p; done +autoreconf -iv -./configure $(configureopts) -make -j$(nproc) +./configure "$(configureopts)" +make -j"$(nproc)" make install DESTDIR=${WORK_DIR}/install/ # packaging @@ -80,26 +79,24 @@ PKGFILE="$(pkgfilename)" BUILD_PATH=${OUTPUT_DIR}/${PKGFILE} # remove old packages -if [ -f ${BUILD_PATH} ]; then - rm ${BUILD_PATH} +if [ -f "${BUILD_PATH}" ]; then + rm "${BUILD_PATH}" fi fpm \ -s dir \ - -t ${PKGFMT} \ - -a ${ARCH} \ - -n ${PKGNAME} \ - -v ${PKGVERSION} \ - --iteration ${ITERATION} \ - --provides ${PKGNAME} \ - --conflicts ${PKGNAME} \ - --replaces ${PKGNAME} \ - --package ${BUILD_PATH} \ + -t "${PKGFMT}" \ + -a "${ARCH}" \ + -n "${PKGNAME}" \ + -v "${PKGVERSION}" \ + --iteration "${ITERATION}" \ + --provides "${PKGNAME}" \ + --conflicts "${PKGNAME}" \ + --replaces "${PKGNAME}" \ + --package "${BUILD_PATH}" \ -C ${WORK_DIR}/install \ - --description "ASN.1 compiler with OpenAirInterface (OAI) specific changes. -ASN.1 to C compiler takes the ASN.1 module files (example) and generates the -C++ compatible C source code. That code can be used to serialize the native C -structures into compact and unambiguous BER/XER/PER-based data files, and -deserialize the files back." - - + --description "ASN.1 (Release 15) compiler with OpenAirInterface (OAI) + specific changes. ASN.1 to C compiler takes the ASN.1 module files (example) + and generates the C++ compatible C source code. That code can be used to + serialize the native C structures into compact and unambiguous + BER/XER/PER-based data files, and deserialize the files back." diff --git a/third_party/build/bin/asn1c_rel15_build.sh b/third_party/build/bin/asn1c_rel15_build.sh deleted file mode 100755 index 7272223ad386..000000000000 --- a/third_party/build/bin/asn1c_rel15_build.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -# -# Copyright 2020 The Magma Authors. - -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Generate the debian package from source for asn1c -# The source code is a forked version hosted on the openair repo, which comes -# with OpenAirInterface (OAI) specific changes. -# -# Example output: -# oai-asn1c_rel15_0~20160721+c3~r43c4a295-1_amd64.deb - -set -e -SCRIPT_DIR="$(dirname "$(realpath "$0")")" -source "${SCRIPT_DIR}"/../lib/util.sh - -COMMIT_DATE=20190423 -# index of the commit from a particular date, start from 0 -COMMIT_INDEX=0 -COMMIT=f12568d6 -# 0~ makes the version compatible with real version numbers -# 0~20160721+c3~r43c4a295 < 0~20160721+c5~r43c4a295 < 0~20160722+c0~r43c4a295 < 0.1 -ITERATION=0 -PKGVERSION=0~${COMMIT_DATE}+c${COMMIT_INDEX}~r${COMMIT} -VERSION="${PKGVERSION}"-"${ITERATION}" - -PKGNAME=oai-asn1c_rel15 - -function configureopts() { - if [ "${ARCH}" = "arm64" ]; then - echo --build=arm-linux-gnu - else - echo -n - fi -} - -WORK_DIR=/tmp/build-${PKGNAME} - -# The resulting package is placed in $OUTPUT_DIR -# or in the cwd. -if [ -z "$1" ]; then - OUTPUT_DIR=$(pwd) -else - OUTPUT_DIR=$1 - if [ ! -d "$OUTPUT_DIR" ]; then - echo "error: $OUTPUT_DIR is not a valid directory. Exiting..." - exit 1 - fi -fi - -# build from source -if [ -d ${WORK_DIR} ]; then - rm -rf ${WORK_DIR} -fi -mkdir ${WORK_DIR} -cd ${WORK_DIR} -git clone https://gitlab.eurecom.fr/oai/asn1c.git -cd asn1c -git checkout ${COMMIT} . - -autoreconf -iv - -./configure "$(configureopts)" -make -j"$(nproc)" -make install DESTDIR=${WORK_DIR}/install/ - -# packaging -PKGFILE="$(pkgfilename)" -BUILD_PATH=${OUTPUT_DIR}/${PKGFILE} - -# remove old packages -if [ -f "${BUILD_PATH}" ]; then - rm "${BUILD_PATH}" -fi - -fpm \ - -s dir \ - -t "${PKGFMT}" \ - -a "${ARCH}" \ - -n "${PKGNAME}" \ - -v "${PKGVERSION}" \ - --iteration "${ITERATION}" \ - --provides "${PKGNAME}" \ - --conflicts "${PKGNAME}" \ - --replaces "${PKGNAME}" \ - --package "${BUILD_PATH}" \ - -C ${WORK_DIR}/install \ - --description "ASN.1 (Release 15) compiler with OpenAirInterface (OAI) - specific changes. ASN.1 to C compiler takes the ASN.1 module files (example) - and generates the C++ compatible C source code. That code can be used to - serialize the native C structures into compact and unambiguous - BER/XER/PER-based data files, and deserialize the files back." -