Skip to content

Add error log and metrics to AI telemetry. #656

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Sep 4, 2020
Merged

Add error log and metrics to AI telemetry. #656

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
3d6ac5a
Accelerate metrics report from every 30 mins to every 5 mins.
csfmomo Aug 12, 2020
fd447c6
Add errCountTest metric.
csfmomo Aug 12, 2020
72b38fc
Refactor SendAiMetrics. AI initialization is in main routine while se…
csfmomo Aug 12, 2020
cbe0381
Add aiMetadata config.
csfmomo Aug 12, 2020
0cd35fd
Add SendErrorMetrics function in ai utils.
csfmomo Aug 13, 2020
f3dbffc
Going to push error log to AI telemetry.
csfmomo Aug 13, 2020
935c38f
Add error log to AI telemetry.
csfmomo Aug 13, 2020
4edb0d6
Change error message format.
csfmomo Aug 14, 2020
da7992b
Add error log and metrics to AI telemetry.
csfmomo Aug 14, 2020
3374d35
Remove unnecessary const.
csfmomo Aug 14, 2020
9ef997d
Change heartbeat back to every 30 mins.
csfmomo Aug 14, 2020
8a3239d
Seperate send log from SendErrorMetric function for better reuse.
csfmomo Aug 17, 2020
26283a7
Change a unit test set name to avoid kernel conflict.
csfmomo Aug 17, 2020
edd600b
Address comments. Make error log and metrics sending more generic.
csfmomo Sep 2, 2020
a7a298e
Fix typo.
csfmomo Sep 2, 2020
a5f65be
Fix indentation.
csfmomo Sep 2, 2020
f621d2e
Fix AI initialize issue.
csfmomo Sep 2, 2020
3a0dd40
Remove unnecessary log.
csfmomo Sep 2, 2020
b56c4cd
Use break in if condition.
csfmomo Sep 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 21 additions & 20 deletions npm/ipsm/ipsm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// Package ipsm focus on ip set operation
// Copyright 2018 Microsoft. All rights reserved.
// MIT License
package ipsm
Expand Down Expand Up @@ -95,7 +96,7 @@ func (ipsMgr *IpsetManager) CreateList(listName string) error {
}
log.Logf("Creating List: %+v", entry)
if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 {
log.Errorf("Error: failed to create ipset list %s.", listName)
metrics.SendErrorMetric(util.IpsmID, "Error: failed to create ipset list %s.", listName)
return err
}

Expand All @@ -116,7 +117,7 @@ func (ipsMgr *IpsetManager) DeleteList(listName string) error {
return nil
}

log.Errorf("Error: failed to delete ipset %s %+v", listName, entry)
metrics.SendErrorMetric(util.IpsmID, "Error: failed to delete ipset %s %+v", listName, entry)
return err
}

Expand Down Expand Up @@ -146,7 +147,7 @@ func (ipsMgr *IpsetManager) AddToList(listName string, setName string) error {
}

if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 {
log.Errorf("Error: failed to create ipset rules. rule: %+v", entry)
metrics.SendErrorMetric(util.IpsmID, "Error: failed to create ipset rules. rule: %+v", entry)
return err
}

Expand All @@ -158,7 +159,7 @@ func (ipsMgr *IpsetManager) AddToList(listName string, setName string) error {
// DeleteFromList removes an ipset to an ipset list.
func (ipsMgr *IpsetManager) DeleteFromList(listName string, setName string) error {
if _, exists := ipsMgr.listMap[listName]; !exists {
log.Logf("ipset list with name %s not found", listName)
metrics.SendErrorMetric(util.IpsmID, "ipset list with name %s not found", listName)
return nil
}

Expand All @@ -170,7 +171,7 @@ func (ipsMgr *IpsetManager) DeleteFromList(listName string, setName string) erro
}

if _, err := ipsMgr.Run(entry); err != nil {
log.Errorf("Error: failed to delete ipset entry. %+v", entry)
metrics.SendErrorMetric(util.IpsmID, "Error: failed to delete ipset entry. %+v", entry)
return err
}

Expand All @@ -181,7 +182,7 @@ func (ipsMgr *IpsetManager) DeleteFromList(listName string, setName string) erro

if len(ipsMgr.listMap[listName].elements) == 0 {
if err := ipsMgr.DeleteList(listName); err != nil {
log.Errorf("Error: failed to delete ipset list %s.", listName)
metrics.SendErrorMetric(util.IpsmID, "Error: failed to delete ipset list %s.", listName)
return err
}
}
Expand All @@ -206,7 +207,7 @@ func (ipsMgr *IpsetManager) CreateSet(setName string, spec []string) error {
}
log.Logf("Creating Set: %+v", entry)
if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 {
log.Errorf("Error: failed to create ipset.")
metrics.SendErrorMetric(util.IpsmID, "Error: failed to create ipset.")
return err
}

Expand All @@ -222,7 +223,7 @@ func (ipsMgr *IpsetManager) CreateSet(setName string, spec []string) error {
// DeleteSet removes a set from ipset.
func (ipsMgr *IpsetManager) DeleteSet(setName string) error {
if _, exists := ipsMgr.setMap[setName]; !exists {
log.Logf("ipset with name %s not found", setName)
metrics.SendErrorMetric(util.IpsmID, "ipset with name %s not found", setName)
return nil
}

Expand All @@ -236,7 +237,7 @@ func (ipsMgr *IpsetManager) DeleteSet(setName string) error {
return nil
}

log.Errorf("Error: failed to delete ipset %s. Entry: %+v", setName, entry)
metrics.SendErrorMetric(util.IpsmID, "Error: failed to delete ipset %s. Entry: %+v", setName, entry)
return err
}

Expand Down Expand Up @@ -285,7 +286,7 @@ func (ipsMgr *IpsetManager) AddToSet(setName, ip, spec, podUid string) error {
}

if errCode, err := ipsMgr.Run(entry); err != nil && errCode != 1 {
log.Logf("Error: failed to create ipset rules. %+v", entry)
metrics.SendErrorMetric(util.IpsmID, "Error: failed to create ipset rules. %+v", entry)
return err
}

Expand Down Expand Up @@ -329,7 +330,7 @@ func (ipsMgr *IpsetManager) DeleteFromSet(setName, ip, podUid string) error {
return nil
}

log.Errorf("Error: failed to delete ipset entry. Entry: %+v", entry)
metrics.SendErrorMetric(util.IpsmID, "Error: failed to delete ipset entry. Entry: %+v", entry)
return err
}

Expand All @@ -354,7 +355,7 @@ func (ipsMgr *IpsetManager) Clean() error {
}

if err := ipsMgr.DeleteSet(setName); err != nil {
log.Errorf("Error: failed to clean ipset")
metrics.SendErrorMetric(util.IpsmID, "Error: failed to clean ipset")
return err
}
}
Expand All @@ -365,7 +366,7 @@ func (ipsMgr *IpsetManager) Clean() error {
}

if err := ipsMgr.DeleteList(listName); err != nil {
log.Errorf("Error: failed to clean ipset list")
metrics.SendErrorMetric(util.IpsmID, "Error: failed to clean ipset list")
return err
}
}
Expand All @@ -379,13 +380,13 @@ func (ipsMgr *IpsetManager) Destroy() error {
operationFlag: util.IpsetFlushFlag,
}
if _, err := ipsMgr.Run(entry); err != nil {
log.Errorf("Error: failed to flush ipset")
metrics.SendErrorMetric(util.IpsmID, "Error: failed to flush ipset")
return err
}

entry.operationFlag = util.IpsetDestroyFlag
if _, err := ipsMgr.Run(entry); err != nil {
log.Errorf("Error: failed to destroy ipset")
metrics.SendErrorMetric(util.IpsmID, "Error: failed to destroy ipset")
return err
}

Expand All @@ -405,7 +406,7 @@ func (ipsMgr *IpsetManager) Run(entry *ipsEntry) (int, error) {
if msg, failed := err.(*exec.ExitError); failed {
errCode := msg.Sys().(syscall.WaitStatus).ExitStatus()
if errCode > 0 {
log.Errorf("Error: There was an error running command: [%s %v] Stderr: [%v, %s]", cmdName, strings.Join(cmdArgs, " "), err, strings.TrimSuffix(string(msg.Stderr), "\n"))
metrics.SendErrorMetric(util.IpsmID, "Error: There was an error running command: [%s %v] Stderr: [%v, %s]", cmdName, strings.Join(cmdArgs, " "), err, strings.TrimSuffix(string(msg.Stderr), "\n"))
}

return errCode, err
Expand All @@ -422,7 +423,7 @@ func (ipsMgr *IpsetManager) Save(configFile string) error {

cmd := exec.Command(util.Ipset, util.IpsetSaveFlag, util.IpsetFileFlag, configFile)
if err := cmd.Start(); err != nil {
log.Errorf("Error: failed to save ipset to file.")
metrics.SendErrorMetric(util.IpsmID, "Error: failed to save ipset to file.")
return err
}
cmd.Wait()
Expand All @@ -438,7 +439,7 @@ func (ipsMgr *IpsetManager) Restore(configFile string) error {

f, err := os.Stat(configFile)
if err != nil {
log.Errorf("Error: failed to get file %s stat from ipsm.Restore", configFile)
metrics.SendErrorMetric(util.IpsmID, "Error: failed to get file %s stat from ipsm.Restore", configFile)
return err
}

Expand All @@ -450,12 +451,12 @@ func (ipsMgr *IpsetManager) Restore(configFile string) error {

cmd := exec.Command(util.Ipset, util.IpsetRestoreFlag, util.IpsetFileFlag, configFile)
if err := cmd.Start(); err != nil {
log.Errorf("Error: failed to restore ipset from file.")
metrics.SendErrorMetric(util.IpsmID, "Error: failed to to restore ipset from file.")
return err
}
cmd.Wait()

//TODO based on the set name and number of entries in the config file, update IPSetInventory

return nil
}
}
4 changes: 2 additions & 2 deletions npm/ipsm/ipsm_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ func TestDeleteFromList(t *testing.T) {
}

if _, err := ipsMgr.Run(entry); err == nil {
t.Errorf("TestDeleteFromList failed @ ipsMgr.CreateSet since %s still exist in kernel", setName)
t.Errorf("TestDeleteFromList failed @ ipsMgr.DeleteSet since %s still exist in kernel", setName)
}
}

Expand Down Expand Up @@ -449,7 +449,7 @@ func TestDestroy(t *testing.T) {
}
}()

if err := ipsMgr.AddToSet("test-set", "1.2.3.4", util.IpsetNetHashFlag, ""); err != nil {
if err := ipsMgr.AddToSet("test-destroy-set", "1.2.3.4", util.IpsetNetHashFlag, ""); err != nil {
t.Errorf("TestDestroy failed @ ipsMgr.AddToSet")
}

Expand Down
46 changes: 21 additions & 25 deletions npm/iptm/iptm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
/*
// Part of this file is modified from iptables package from Kuberenetes.
// https://github.com/kubernetes/kubernetes/blob/master/pkg/util/iptables

Part of this file is modified from iptables package from Kuberenetes.
https://github.com/kubernetes/kubernetes/blob/master/pkg/util/iptables

*/
package iptm

import (
Expand All @@ -15,7 +12,6 @@ import (
"time"

"golang.org/x/sys/unix"

"github.com/Azure/azure-container-networking/log"
"github.com/Azure/azure-container-networking/npm/metrics"
"github.com/Azure/azure-container-networking/npm/util"
Expand Down Expand Up @@ -92,7 +88,7 @@ func (iptMgr *IptablesManager) InitNpmChains() error {
iptMgr.OperationFlag = util.IptablesInsertionFlag
entry.Specs = append([]string{index}, entry.Specs...)
if _, err = iptMgr.Run(entry); err != nil {
log.Errorf("Error: failed to add AZURE-NPM chain to FORWARD chain.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to add AZURE-NPM chain to FORWARD chain.")
return err
}
}
Expand All @@ -113,7 +109,7 @@ func (iptMgr *IptablesManager) InitNpmChains() error {
if !exists {
iptMgr.OperationFlag = util.IptablesAppendFlag
if _, err := iptMgr.Run(entry); err != nil {
log.Errorf("Error: failed to add AZURE-NPM-INGRESS-PORT chain to AZURE-NPM chain.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to add AZURE-NPM-INGRESS-PORT chain to AZURE-NPM chain.")
return err
}
}
Expand All @@ -139,7 +135,7 @@ func (iptMgr *IptablesManager) InitNpmChains() error {
if !exists {
iptMgr.OperationFlag = util.IptablesAppendFlag
if _, err := iptMgr.Run(entry); err != nil {
log.Errorf("Error: failed to add AZURE-NPM-EGRESS-PORT chain to AZURE-NPM chain.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to add AZURE-NPM-INGRESS-PORT chain to AZURE-NPM chain.")
return err
}
}
Expand All @@ -165,7 +161,7 @@ func (iptMgr *IptablesManager) InitNpmChains() error {
if !exists {
iptMgr.OperationFlag = util.IptablesAppendFlag
if _, err := iptMgr.Run(entry); err != nil {
log.Errorf("Error: failed to add AZURE-NPM-TARGET-SETS chain to AZURE-NPM chain.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to add AZURE-NPM-TARGET-SETS chain to AZURE-NPM chain.")
return err
}
}
Expand All @@ -188,7 +184,7 @@ func (iptMgr *IptablesManager) InitNpmChains() error {
if !exists {
iptMgr.OperationFlag = util.IptablesAppendFlag
if _, err = iptMgr.Run(entry); err != nil {
log.Logf("Error: failed to add default allow CONNECTED/RELATED rule to AZURE-NPM chain.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to add default allow CONNECTED/RELATED rule to AZURE-NPM chain.")
return err
}
}
Expand Down Expand Up @@ -218,7 +214,7 @@ func (iptMgr *IptablesManager) UninitNpmChains() error {
iptMgr.OperationFlag = util.IptablesDeletionFlag
errCode, err := iptMgr.Run(entry)
if errCode != iptablesErrDoesNotExist && err != nil {
log.Errorf("Error: failed to remove default rule from FORWARD chain.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to add default allow CONNECTED/RELATED rule to AZURE-NPM chain.")
return err
}

Expand All @@ -229,7 +225,7 @@ func (iptMgr *IptablesManager) UninitNpmChains() error {
}
errCode, err := iptMgr.Run(entry)
if errCode != iptablesErrDoesNotExist && err != nil {
log.Errorf("Error: failed to flush iptables chain %s.", chain)
metrics.SendErrorMetric(util.IptmID, "Error: failed to flush iptables chain %s.", chain)
}
}

Expand Down Expand Up @@ -270,7 +266,7 @@ func (iptMgr *IptablesManager) AddChain(chain string) error {
return nil
}

log.Errorf("Error: failed to create iptables chain %s.", entry.Chain)
metrics.SendErrorMetric(util.IptmID, "Error: failed to create iptables chain %s.", entry.Chain)
return err
}

Expand All @@ -290,7 +286,7 @@ func (iptMgr *IptablesManager) DeleteChain(chain string) error {
return nil
}

log.Errorf("Error: failed to delete iptables chain %s.", entry.Chain)
metrics.SendErrorMetric(util.IptmID, "Error: failed to delete iptables chain %s.", entry.Chain)
return err
}

Expand All @@ -309,7 +305,7 @@ func (iptMgr *IptablesManager) Add(entry *IptEntry) error {
iptMgr.OperationFlag = util.IptablesInsertionFlag
}
if _, err := iptMgr.Run(entry); err != nil {
log.Errorf("Error: failed to create iptables rules.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to create iptables rules.")
return err
}

Expand All @@ -334,7 +330,7 @@ func (iptMgr *IptablesManager) Delete(entry *IptEntry) error {

iptMgr.OperationFlag = util.IptablesDeletionFlag
if _, err := iptMgr.Run(entry); err != nil {
log.Errorf("Error: failed to delete iptables rules.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to delete iptables rules.")
return err
}

Expand Down Expand Up @@ -364,7 +360,7 @@ func (iptMgr *IptablesManager) Run(entry *IptEntry) (int, error) {
if msg, failed := err.(*exec.ExitError); failed {
errCode := msg.Sys().(syscall.WaitStatus).ExitStatus()
if errCode > 0 && iptMgr.OperationFlag != util.IptablesCheckFlag {
log.Errorf("Error: There was an error running command: [%s %v] Stderr: [%v, %s]", cmdName, strings.Join(cmdArgs, " "), err, strings.TrimSuffix(string(msg.Stderr), "\n"))
metrics.SendErrorMetric(util.IptmID, "Error: There was an error running command: [%s %v] Stderr: [%v, %s]", cmdName, strings.Join(cmdArgs, " "), err, strings.TrimSuffix(string(msg.Stderr), "\n"))
}

return errCode, err
Expand Down Expand Up @@ -393,15 +389,15 @@ func (iptMgr *IptablesManager) Save(configFile string) error {
// create the config file for writing
f, err := os.Create(configFile)
if err != nil {
log.Errorf("Error: failed to open file: %s.", configFile)
metrics.SendErrorMetric(util.IptmID, "Error: failed to open file: %s.", configFile)
return err
}
defer f.Close()

cmd := exec.Command(util.IptablesSave)
cmd.Stdout = f
if err := cmd.Start(); err != nil {
log.Errorf("Error: failed to run iptables-save.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to run iptables-save.")
return err
}
cmd.Wait()
Expand Down Expand Up @@ -429,15 +425,15 @@ func (iptMgr *IptablesManager) Restore(configFile string) error {
// open the config file for reading
f, err := os.Open(configFile)
if err != nil {
log.Errorf("Error: failed to open file: %s.", configFile)
metrics.SendErrorMetric(util.IptmID, "Error: failed to open file: %s.", configFile)
return err
}
defer f.Close()

cmd := exec.Command(util.IptablesRestore)
cmd.Stdin = f
if err := cmd.Start(); err != nil {
log.Errorf("Error: failed to run iptables-restore.")
metrics.SendErrorMetric(util.IptmID, "Error: failed to run iptables-restore.")
return err
}
cmd.Wait()
Expand All @@ -460,7 +456,7 @@ func grabIptablesLocks() (*os.File, error) {
// Grab 1.6.x style lock.
l, err := os.OpenFile(util.IptablesLockFile, os.O_CREATE, 0600)
if err != nil {
log.Logf("Error: failed to open iptables lock file %s.", util.IptablesLockFile)
metrics.SendErrorMetric(util.IptmID, "Error: failed to open iptables lock file %s.", util.IptablesLockFile)
return nil, err
}

Expand All @@ -471,7 +467,7 @@ func grabIptablesLocks() (*os.File, error) {

return true, nil
}); err != nil {
log.Logf("Error: failed to acquire new iptables lock: %v.", err)
metrics.SendErrorMetric(util.IptmID, "Error: failed to acquire new iptables lock: %v.", err)
return nil, err
}

Expand Down Expand Up @@ -507,4 +503,4 @@ func grabIptablesFileLock(f *os.File) error {
// // Write table headers.
// writeLine(filterChains, "*filter")

// }
// }
Loading