feat: event logs clean #256

Xvv-v · 2022-10-13T02:34:11Z

Signed-off-by: xu.jingwen xu.jingwen@99cloud.net

What type of PR is this?

/kind feature

What this PR does / why we need it:

Which issue(s) this PR fixes:

Fixes #255

Special notes for reviewers:

Does this PR introduced a user-facing change?

Additional documentation, usage docs, etc.:

Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

kubeclipper-bot · 2022-10-13T02:34:14Z

@Xvv-v: Adding the "do-not-merge/release-note-label-needed" label because no release-note block was detected, please follow our release note process to remove it.

Instructions for interacting with me using PR comments are available here. If you have questions or suggestions related to my behavior, please file an issue against the kubernetes/test-infra repository.

Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

codecov-commenter · 2022-10-13T02:43:13Z

Codecov Report

Merging #256 (19e848c) into master (f1cb534) will decrease coverage by 0.00%.
The diff coverage is 61.11%.

@@            Coverage Diff             @@
##           master     #256      +/-   ##
==========================================
- Coverage   13.12%   13.11%   -0.01%     
==========================================
  Files          98       98              
  Lines       15341    15359      +18     
==========================================
+ Hits         2014     2015       +1     
- Misses      13088    13105      +17     
  Partials      239      239

Impacted Files	Coverage Δ
pkg/cli/deploy/deploy.go	`5.69% <0.00%> (-0.03%)`	⬇️
pkg/auditing/audit.go	`28.50% <78.57%> (+0.77%)`	⬆️
pkg/scheme/core/v1/k8s/cluster.go	`0.12% <0.00%> (-0.01%)`	⬇️
pkg/component/nfs/nfs.go	`0.56% <0.00%> (-0.01%)`	⬇️
pkg/apis/core/v1/handler.go	`0.33% <0.00%> (-0.01%)`	⬇️

Xvv-v · 2022-10-13T02:47:03Z

/cc @x893675

x893675 · 2022-10-13T06:03:16Z

/cc @lixd

x893675 · 2022-10-13T06:03:30Z

/milestone v1.3.1

x893675 · 2022-10-13T06:42:46Z

pkg/models/models.go

@@ -168,6 +169,14 @@ func ObjectMetaFilter(obj metav1.ObjectMeta, key, value string) bool {
 		if !strings.Contains(obj.Annotations[common.AnnotationDescription], value) {
 			return false
 		}
+	case "time":


rename create_before

x893675 · 2022-10-13T06:48:51Z

pkg/models/platform/interface.go

@@ -57,6 +57,7 @@ type EventReader interface {
 type EventReaderEx interface {
 	GetEventEx(ctx context.Context, name string, resourceVersion string) (*v1.Event, error)
 	ListEventsEx(ctx context.Context, query *query.Query) (*models.PageableResponse, error)
+	ListEventsWithTimeEx(ctx context.Context, query *query.Query) (*models.PageableResponse, error)


x893675 · 2022-10-13T06:49:03Z

pkg/models/platform/platformsetting.go

@@ -138,6 +138,30 @@ func (p *platformOperator) ListEventsEx(ctx context.Context, query *query.Query)
 	return models.ListExV2(ctx, p.eventStorage, query, p.eventFilter, nil, nil)
 }

+func (p *platformOperator) ListEventsWithTimeEx(ctx context.Context, query *query.Query) (*models.PageableResponse, error) {


x893675 · 2022-10-13T06:52:35Z

pkg/controller/audit_status.go

+	q := &query.Query{
+		FieldSelector: fieldSelector,
+		Reverse:       false,
+		Limit:         -1,


remove Limit

x893675 · 2022-10-13T06:52:49Z

pkg/controller/audit_status.go

+		FieldSelector: fieldSelector,
+		Reverse:       false,
+		Limit:         -1,
+		Pagination: &query.Pagination{


use query.NoPagination()

x893675 · 2022-10-13T06:58:30Z

pkg/controller/audit_status.go

+			Offset: 0,
+		},
+	}
+	loginLogs, err := s.AuditOperator.ListEvents(context.TODO(), q)


query with timestamp, then check entry size, if not allowd, skip direct

x893675 · 2022-10-13T06:58:45Z

pkg/controller/audit_status.go

+		q.FuzzySearch = map[string]string{
+			"time": timestamp,
+		}
+		eventList, err := s.AuditOperator.ListEventsWithTimeEx(context.TODO(), q)


use ListEventEx instead

x893675 · 2022-10-13T07:01:21Z

pkg/auditing/option/options.go

+)
+
+type AuditOptions struct {
+	EventLogHistoryRetentionPeriod time.Duration `json:"eventLogHistoryRetentionPeriod" yaml:"eventLogHistoryRetentionPeriod"`


rename RetentionPeriod
rename MaximumEntries

x893675 · 2022-10-13T07:03:38Z

pkg/controller/audit_status.go

+
+func (s *AuditStatusMon) monitorAuditStatus() {
+	timestamp := time.Now().Add(-s.AuditOptions.EventLogHistoryRetentionPeriod).Format(time.RFC3339)
+	s.cleanLogs("type", timestamp)


add annotate
// clean login / logout audit envent
s.cleanLogs("type", timestamp)

lixd · 2022-10-13T06:48:33Z

pkg/auditing/option/options.go

+		default:
+			return errors.New("[--period] format error, only support [h, d, w]")
+		}
+	} else {


return error first

if !m.MatchString(o.period){ return errors.New("only support [ None ｜ Metadata | Request | RequestResponse ]") } // do parse

lixd · 2022-10-13T07:02:37Z

pkg/controller/audit_status.go

+
+func (s *AuditStatusMon) monitorAuditStatus() {
+	timestamp := time.Now().Add(-s.AuditOptions.EventLogHistoryRetentionPeriod).Format(time.RFC3339)
+	s.cleanLogs("type", timestamp)


wrong FieldSelector format

Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

x893675 · 2022-10-14T05:33:28Z

cmd/kubeclipper-server/app/options/options.go

 	var errors []error
 	errors = append(errors, s.GenericServerRunOptions.Validate()...)
 	errors = append(errors, s.EtcdOptions.Validate()...)
 	errors = append(errors, s.MQOptions.Validate()...)
 	errors = append(errors, s.LogOptions.Validate()...)
 	errors = append(errors, s.AuthenticationOptions.Validate()...)
+	if err = s.AuditOptions.Validate(); err != nil {


rewrite to append(errors, s.AuditOptions.Validate())

lixd · 2022-10-14T05:28:05Z

pkg/server/config/config.go

@@ -56,6 +58,7 @@ type Config struct {
 	MQOptions               *natsio.NatsOptions                `json:"mq,omitempty" yaml:"mq,omitempty"  mapstructure:"mq"`
 	LogOptions              *logger.Options                    `json:"log,omitempty" yaml:"log,omitempty" mapstructure:"log"`
 	AuthenticationOptions   *authoptions.AuthenticationOptions `json:"authentication,omitempty" yaml:"authentication,omitempty" mapstructure:"authentication"`
+	AuditOptions            *auditoptions.AuditOptions         `json:"auditOptions,omitempty" yaml:"auditOptions,omitempty" mapstructure:"authentication"`


audit tag not same with kubeclipper-server.yaml

Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

lixd

/lgtm

kubeclipper-bot · 2022-10-14T07:08:29Z

LGTM label has been added.

Git tree hash: 85035ebb996ea51c7bfd8faa5c1b94678c4a3645

x893675 · 2022-10-14T07:54:29Z

/approve

kubeclipper-bot · 2022-10-14T07:54:37Z

[APPROVALNOTIFIER] This PR is APPROVED

This pull-request has been approved by: lixd, x893675, Xvv-v

The full list of commands accepted by this bot can be found here.

The pull request process is described here

Needs approval from an approver in each of these files:

~~OWNERS~~ [x893675]

Approvers can indicate their approval by writing /approve in a comment
Approvers can cancel approval by writing /approve cancel in a comment

feat: event logs clean

e1c7656

Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

kubeclipper-bot added kind/feature Categorizes issue or PR as related to a new feature. dco-signoff: yes do-not-merge/release-note-label-needed size/L Denotes a PR that changes 100-499 lines, ignoring generated files. labels Oct 13, 2022

code formatting

6b1755c

Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

kubeclipper-bot requested a review from x893675 October 13, 2022 02:47

kubeclipper-bot requested a review from lixd October 13, 2022 06:03

kubeclipper-bot added this to the v1.3.1 milestone Oct 13, 2022

x893675 reviewed Oct 13, 2022

View reviewed changes

lixd reviewed Oct 13, 2022

View reviewed changes

Xvv-v added 2 commits October 14, 2022 10:52

first change

1f92c4c

Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

modify code checkfmt timeout

09c1010

Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

x893675 reviewed Oct 14, 2022

View reviewed changes

lixd reviewed Oct 14, 2022

View reviewed changes

second modify

19e848c

Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

lixd approved these changes Oct 14, 2022

View reviewed changes

kubeclipper-bot assigned lixd Oct 14, 2022

kubeclipper-bot added the lgtm Indicates that a PR is ready to be merged. label Oct 14, 2022

kubeclipper-bot added the approved Indicates a PR has been approved by an approver from all required OWNERS files. label Oct 14, 2022

kubeclipper-bot merged commit 17bba1d into kubeclipper:master Oct 14, 2022

Xvv-v deleted the clean branch October 21, 2022 01:59

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat: event logs clean #256

feat: event logs clean #256

Xvv-v commented Oct 13, 2022

kubeclipper-bot commented Oct 13, 2022

codecov-commenter commented Oct 13, 2022 •

edited

Xvv-v commented Oct 13, 2022

x893675 commented Oct 13, 2022

x893675 commented Oct 13, 2022

x893675 Oct 13, 2022

x893675 Oct 13, 2022

x893675 Oct 13, 2022

x893675 Oct 13, 2022

x893675 Oct 13, 2022

x893675 Oct 13, 2022

x893675 Oct 13, 2022

x893675 Oct 13, 2022

x893675 Oct 13, 2022

lixd Oct 13, 2022

lixd Oct 13, 2022

x893675 Oct 14, 2022

lixd Oct 14, 2022

lixd left a comment

kubeclipper-bot commented Oct 14, 2022

x893675 commented Oct 14, 2022

kubeclipper-bot commented Oct 14, 2022

feat: event logs clean #256

feat: event logs clean #256

Conversation

Xvv-v commented Oct 13, 2022

What type of PR is this?

What this PR does / why we need it:

Which issue(s) this PR fixes:

Special notes for reviewers:

Does this PR introduced a user-facing change?

Additional documentation, usage docs, etc.:

kubeclipper-bot commented Oct 13, 2022

codecov-commenter commented Oct 13, 2022 • edited

Codecov Report

Xvv-v commented Oct 13, 2022

x893675 commented Oct 13, 2022

x893675 commented Oct 13, 2022

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

lixd left a comment

Choose a reason for hiding this comment

kubeclipper-bot commented Oct 14, 2022

x893675 commented Oct 14, 2022

kubeclipper-bot commented Oct 14, 2022

codecov-commenter commented Oct 13, 2022 •

edited