Skip to content

Commit

Permalink
Support L7 NetworkPolicy Logging (antrea-io#4625)
Browse files Browse the repository at this point in the history 
Antrea-native policy now supports layer 7 NetworkPolicy. To provide
more information for users, logging for this feature is introduced.

Antrea-native policy is not accurate enough in reporting packet status
before sending to l7 engine. Logs are fixed to reflect "Redirect" action.
Audit logging UT are updated to cover more cases.

L7 engine provides its own logs. Currently, Suricata is used as L7 engine.
Configuration is updated to generate two log files, fast.log and eve.json
Both files locates at /var/log/antrea/networkpolicy/. Documentation is updated.

Signed-off-by: Qiyue Yao <yaoq@vmware.com>
  • Loading branch information
@qiyueyao
qiyueyao authored and Pulkit Jain committed Apr 28, 2023
1 parent 9272252 commit cb78eae
Show file tree
Hide file tree
Showing 11 changed files with 326 additions and 59 deletions.
75 changes: 75 additions & 0 deletions docs/antrea-l7-network-policy.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
- [Usage](#usage)
- [HTTP](#http)
- [More examples](#more-examples)
- [Logs](#logs)
- [Limitations](#limitations)
<!-- /toc -->

Expand Down Expand Up @@ -200,6 +201,80 @@ spec:
- http: {} # automatically dropped, and subsequent rules will not be considered.
```

### Logs

Layer 7 traffic that matches the NetworkPolicy will be logged in an event
triggered log file (`/var/log/antrea/networkpolicy/l7engine/eve-YEAR-MONTH-DAY.json`).
The event type for this log is `alert`. If `enableLogging` is set for the rule,
packets that match the rule will also be logged in addition to the event with
event type `packet`. Below is an example of the two event types.

Deny ingress from client (10.10.1.5) to web (10.10.1.4/admin)

```json
{
"timestamp": "2023-03-09T20:00:28.210821+0000",
"flow_id": 627175734391745,
"in_iface": "antrea-l7-tap0",
"event_type": "alert",
"vlan": [
1
],
"src_ip": "10.10.1.5",
"src_port": 43352,
"dest_ip": "10.10.1.4",
"dest_port": 80,
"proto": "TCP",
"alert": {
"action": "blocked",
"gid": 1,
"signature_id": 1,
"rev": 0,
"signature": "Reject by AntreaClusterNetworkPolicy:test-l7-ingress",
"category": "",
"severity": 3,
"tenant_id": 1
},
"http": {
"hostname": "10.10.1.4",
"url": "/admin",
"http_user_agent": "curl/7.74.0",
"http_method": "GET",
"protocol": "HTTP/1.1",
"length": 0
},
"app_proto": "http",
"flow": {
"pkts_toserver": 3,
"pkts_toclient": 1,
"bytes_toserver": 284,
"bytes_toclient": 74,
"start": "2023-03-09T20:00:28.209857+0000"
}
}
```

```json
{
"timestamp": "2023-03-09T20:00:28.225016+0000",
"flow_id": 627175734391745,
"in_iface": "antrea-l7-tap0",
"event_type": "packet",
"vlan": [
1
],
"src_ip": "10.10.1.4",
"src_port": 80,
"dest_ip": "10.10.1.5",
"dest_port": 43352,
"proto": "TCP",
"packet": "/lhtPRglzmQvxnJoCABFAAAoUGYAAEAGFE4KCgEECgoBBQBQqVhIGzbi/odenlAUAfsR7QAA",
"packet_info": {
"linktype": 1
}
}
```

## Limitations

This feature is currently only supported for Nodes running Linux.
4 changes: 3 additions & 1 deletion docs/antrea-network-policy.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -706,7 +706,9 @@ traffic that matches the "DropToThirdParty" egress rule, while the rule
"AllowFromFrontend" is not logged. Specifically for drop and reject rules,
deduplication is applied to reduce duplicated logs, and duplication buffer
length is set to 1 second. If a rule name is not provided, an identifiable
name will be generated for the rule and displayed in the log.
name will be generated for the rule and displayed in the log. For rules in layer
7 NetworkPolicy, packets are logged with action `Redirect` prior to analysis by
the layer 7 engine, more details are available in the corresponding engine logs.
The rules are logged in the following format:

```text
Expand Down
11 changes: 11 additions & 0 deletions pkg/agent/controller/networkpolicy/audit_logging.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,17 @@ func getNetworkPolicyInfo(pktIn *ofctrl.PacketIn, c *Controller, ob *logInfo) er
}
ob.disposition = openflow.DispositionToString[disposition]

// Get layer 7 NetworkPolicy redirect action, if traffic is redirected, disposition log should be overwritten.
if match = getMatchRegField(matchers, openflow.L7NPRegField); match != nil {
l7NPRegVal, err := getInfoInReg(match, openflow.L7NPRegField.GetRange().ToNXRange())
if err != nil {
return fmt.Errorf("received error while unloading l7 NP redirect value from reg: %v", err)
}
if l7NPRegVal == openflow.DispositionL7NPRedirect {
ob.disposition = "Redirect"
}
}

// Set match to corresponding ingress/egress reg according to disposition.
match = getMatch(matchers, tableID, disposition)

Expand Down
189 changes: 157 additions & 32 deletions pkg/agent/controller/networkpolicy/audit_logging_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ import (

"antrea.io/libOpenflow/openflow15"
"antrea.io/ofnet/ofctrl"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"antrea.io/antrea/pkg/agent/openflow"
openflowtest "antrea.io/antrea/pkg/agent/openflow/testing"
"antrea.io/antrea/pkg/apis/controlplane/v1beta2"
binding "antrea.io/antrea/pkg/ovs/openflow"
"antrea.io/antrea/pkg/util/ip"
)
Expand All @@ -41,6 +41,12 @@ const (
testBufferLength time.Duration = 100 * time.Millisecond
)

var (
actionAllow = openflow.DispositionToString[openflow.DispositionAllow]
actionDrop = openflow.DispositionToString[openflow.DispositionDrop]
actionRedirect = "Redirect"
)

// mockLogger implements io.Writer.
type mockLogger struct {
mu sync.Mutex
Expand Down Expand Up @@ -122,7 +128,7 @@ func newTestAntreaPolicyLogger(bufferLength time.Duration, clock Clock) (*Antrea

func newLogInfo(disposition string) (*logInfo, string) {
testLogInfo := &logInfo{
tableName: "AntreaPolicyIngressRule",
tableName: openflow.AntreaPolicyIngressRuleTable.GetName(),
npRef: "AntreaNetworkPolicy:default/test",
ruleName: "test-rule",
ofPriority: "0",
Expand All @@ -146,7 +152,7 @@ func expectedLogWithCount(msg string, count int) string {

func TestAllowPacketLog(t *testing.T) {
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, &realClock{})
ob, expected := newLogInfo("Allow")
ob, expected := newLogInfo(actionAllow)

antreaLogger.LogDedupPacket(ob)
actual := <-mockAnpLogger.logged
Expand All @@ -155,7 +161,7 @@ func TestAllowPacketLog(t *testing.T) {

func TestDropPacketLog(t *testing.T) {
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, &realClock{})
ob, expected := newLogInfo("Drop")
ob, expected := newLogInfo(actionDrop)

antreaLogger.LogDedupPacket(ob)
actual := <-mockAnpLogger.logged
Expand All @@ -166,7 +172,7 @@ func TestDropPacketDedupLog(t *testing.T) {
clock := NewVirtualClock(time.Now())
defer clock.Stop()
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, clock)
ob, expected := newLogInfo("Drop")
ob, expected := newLogInfo(actionDrop)
// Add the additional log info for duplicate packets.
expected = expectedLogWithCount(expected, 2)

Expand All @@ -187,7 +193,7 @@ func TestDropPacketMultiDedupLog(t *testing.T) {
clock := NewVirtualClock(time.Now())
defer clock.Stop()
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, clock)
ob, expected := newLogInfo("Drop")
ob, expected := newLogInfo(actionDrop)

consumeLog := func() (int, error) {
select {
Expand Down Expand Up @@ -231,33 +237,148 @@ func TestDropPacketMultiDedupLog(t *testing.T) {
assert.Equal(t, 1, c2)
}

func TestRedirectPacketLog(t *testing.T) {
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, &realClock{})
ob, expected := newLogInfo(actionRedirect)

antreaLogger.LogDedupPacket(ob)
actual := <-mockAnpLogger.logged
assert.Contains(t, actual, expected)
}

func TestGetNetworkPolicyInfo(t *testing.T) {
openflow.InitMockTables(
map[*openflow.Table]uint8{
openflow.AntreaPolicyEgressRuleTable: uint8(5),
openflow.EgressRuleTable: uint8(6),
openflow.EgressDefaultTable: uint8(7),
openflow.AntreaPolicyIngressRuleTable: uint8(12),
openflow.IngressRuleTable: uint8(13),
openflow.IngressDefaultTable: uint8(14),
})
c := &Controller{ofClient: &openflowtest.MockClient{}}
ob := new(logInfo)
regID := openflow.APDispositionField.GetRegID()
dispositionMatch := openflow15.MatchField{
Class: openflow15.OXM_CLASS_PACKET_REGS,
Field: uint8(regID / 2),
HasMask: false,
Value: &openflow15.ByteArrayField{Data: []byte{1, 1, 1, 1}},
prepareMockOFTablesWithCache()
generateMatch := func(regID int, data []byte) openflow15.MatchField {
return openflow15.MatchField{
Class: openflow15.OXM_CLASS_PACKET_REGS,
Field: uint8(regID / 2),
HasMask: false,
Value: &openflow15.ByteArrayField{Data: data},
}
}
testANPRef := "AntreaNetworkPolicy:default/test-anp"
testK8sRef := "K8sNetworkPolicy:default/test-anp"
testPriority, testRule := "61800", "test-rule"
allowDispositionData := []byte{0x11, 0x00, 0x00, 0x11}
dropDispositionData := []byte{0x11, 0x00, 0x08, 0x11}
redirectDispositionData := []byte{0x11, 0x08, 0x00, 0x11}
ingressData := []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}
tests := []struct {
name string
tableID uint8
expectedCalls func(mockClient *openflowtest.MockClientMockRecorder)
dispositionData []byte
ob *logInfo
wantOb *logInfo
wantErr error
}{
{
name: "ANP Allow",
tableID: openflow.AntreaPolicyIngressRuleTable.GetID(),
expectedCalls: func(mockClient *openflowtest.MockClientMockRecorder) {
mockClient.GetPolicyInfoFromConjunction(gomock.Any()).Return(
testANPRef, testPriority, testRule)
},
dispositionData: allowDispositionData,
wantOb: &logInfo{
tableName: openflow.AntreaPolicyIngressRuleTable.GetName(),
disposition: actionAllow,
npRef: testANPRef,
ofPriority: testPriority,
ruleName: testRule,
},
},
{
name: "K8s Allow",
tableID: openflow.IngressRuleTable.GetID(),
expectedCalls: func(mockClient *openflowtest.MockClientMockRecorder) {
mockClient.GetPolicyInfoFromConjunction(gomock.Any()).Return(
testK8sRef, testPriority, "")
},
dispositionData: allowDispositionData,
wantOb: &logInfo{
tableName: openflow.IngressRuleTable.GetName(),
disposition: actionAllow,
npRef: testK8sRef,
ofPriority: testPriority,
ruleName: "<nil>",
},
},
{
name: "ANP Drop",
tableID: openflow.AntreaPolicyIngressRuleTable.GetID(),
expectedCalls: func(mockClient *openflowtest.MockClientMockRecorder) {
mockClient.GetPolicyInfoFromConjunction(gomock.Any()).Return(
testANPRef, testPriority, testRule)
},
dispositionData: dropDispositionData,
wantOb: &logInfo{
tableName: openflow.AntreaPolicyIngressRuleTable.GetName(),
disposition: actionDrop,
npRef: testANPRef,
ofPriority: testPriority,
ruleName: testRule,
},
},
{
name: "K8s Drop",
tableID: openflow.IngressDefaultTable.GetID(),
dispositionData: dropDispositionData,
wantOb: &logInfo{
tableName: openflow.IngressDefaultTable.GetName(),
disposition: actionDrop,
npRef: "K8sNetworkPolicy",
ofPriority: "<nil>",
ruleName: "<nil>",
},
},
{
name: "ANP Redirect",
tableID: openflow.AntreaPolicyIngressRuleTable.GetID(),
expectedCalls: func(mockClient *openflowtest.MockClientMockRecorder) {
mockClient.GetPolicyInfoFromConjunction(gomock.Any()).Return(
testANPRef, testPriority, testRule)
},
dispositionData: redirectDispositionData,
wantOb: &logInfo{
tableName: openflow.AntreaPolicyIngressRuleTable.GetName(),
disposition: actionRedirect,
npRef: testANPRef,
ofPriority: testPriority,
ruleName: testRule,
},
},
}
matchers := []openflow15.MatchField{dispositionMatch}
pktIn := &ofctrl.PacketIn{TableId: 17, Match: openflow15.Match{Fields: matchers}}

err := getNetworkPolicyInfo(pktIn, c, ob)
assert.Equal(t, string(v1beta2.K8sNetworkPolicy), ob.npRef)
assert.Equal(t, "<nil>", ob.ofPriority)
assert.Equal(t, "<nil>", ob.ruleName)
require.NoError(t, err)
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
// Inject disposition and redirect match.
dispositionMatch := generateMatch(openflow.APDispositionField.GetRegID(), tc.dispositionData)
matchers := []openflow15.MatchField{dispositionMatch}
// Inject ingress/egress match when case is not K8s default drop.
if tc.expectedCalls != nil {
regID := openflow.TFIngressConjIDField.GetRegID()
if tc.wantOb.disposition == actionDrop {
regID = openflow.CNPConjIDField.GetRegID()
}
ingressMatch := generateMatch(regID, ingressData)
matchers = append(matchers, ingressMatch)
}
pktIn := &ofctrl.PacketIn{TableId: tc.tableID, Match: openflow15.Match{Fields: matchers}}

ctrl := gomock.NewController(t)
defer ctrl.Finish()
testClientInterface := openflowtest.NewMockClient(ctrl)
if tc.expectedCalls != nil {
tc.expectedCalls(testClientInterface.EXPECT())
}
c := &Controller{ofClient: testClientInterface}
tc.ob = new(logInfo)
gotErr := getNetworkPolicyInfo(pktIn, c, tc.ob)
assert.Equal(t, tc.wantOb, tc.ob)
assert.Equal(t, tc.wantErr, gotErr)
})
}
}

func TestGetPacketInfo(t *testing.T) {
Expand All @@ -277,7 +398,6 @@ func TestGetPacketInfo(t *testing.T) {
SourcePort: 35402,
DestinationPort: 80,
},
ob: new(logInfo),
wantOb: &logInfo{
srcIP: "0.0.0.0",
srcPort: "35402",
Expand All @@ -295,7 +415,6 @@ func TestGetPacketInfo(t *testing.T) {
IPLength: 60,
IPProto: ip.ICMPProtocol,
},
ob: new(logInfo),
wantOb: &logInfo{
srcIP: "0.0.0.0",
srcPort: "<nil>",
Expand All @@ -308,7 +427,13 @@ func TestGetPacketInfo(t *testing.T) {
}

for _, tc := range tests {
tc.ob = new(logInfo)
getPacketInfo(tc.packet, tc.ob)
assert.Equal(t, tc.wantOb, tc.ob)
}
}

func prepareMockOFTablesWithCache() {
openflow.InitMockTables(mockOFTables)
openflow.InitOFTableCache(mockOFTables)
}

0 comments on commit cb78eae

Please sign in to comment.