Skip to content

Commit

Permalink
Merge branch 'main' into perf/query-accessory
Browse files Browse the repository at this point in the history
  • Loading branch information
wy65701436 committed Nov 22, 2023
2 parents 9446ce3 + 17c17ff commit 5650e27
Show file tree
Hide file tree
Showing 58 changed files with 803 additions and 172 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/CI.yml
Expand Up @@ -44,7 +44,7 @@ jobs:
- name: Set up Go 1.21
uses: actions/setup-go@v1
with:
go-version: 1.21.3
go-version: 1.21.4
id: go
- uses: actions/checkout@v3
with:
Expand Down Expand Up @@ -105,7 +105,7 @@ jobs:
- name: Set up Go 1.21
uses: actions/setup-go@v1
with:
go-version: 1.21.3
go-version: 1.21.4
id: go
- uses: actions/checkout@v3
with:
Expand Down Expand Up @@ -160,7 +160,7 @@ jobs:
- name: Set up Go 1.21
uses: actions/setup-go@v1
with:
go-version: 1.21.3
go-version: 1.21.4
id: go
- uses: actions/checkout@v3
with:
Expand Down Expand Up @@ -215,7 +215,7 @@ jobs:
- name: Set up Go 1.21
uses: actions/setup-go@v1
with:
go-version: 1.21.3
go-version: 1.21.4
id: go
- uses: actions/checkout@v3
with:
Expand Down Expand Up @@ -268,7 +268,7 @@ jobs:
- name: Set up Go 1.21
uses: actions/setup-go@v1
with:
go-version: 1.21.3
go-version: 1.21.4
id: go
- uses: actions/checkout@v3
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build-package.yml
Expand Up @@ -26,7 +26,7 @@ jobs:
- name: Set up Go 1.21
uses: actions/setup-go@v1
with:
go-version: 1.21.3
go-version: 1.21.4
id: go
- name: Setup Docker
uses: docker-practice/actions-setup-docker@master
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/conformance_test.yml
Expand Up @@ -28,7 +28,7 @@ jobs:
- name: Set up Go 1.21
uses: actions/setup-go@v1
with:
go-version: 1.21.3
go-version: 1.21.4
id: go
- uses: actions/checkout@v3
with:
Expand Down
2 changes: 2 additions & 0 deletions CONTRIBUTING.md
Expand Up @@ -164,6 +164,8 @@ Harbor backend is written in [Go](http://golang.org/). If you don't have a Harbo
| 2.7 | 1.19.4 |
| 2.8 | 1.20.6 |
| 2.9 | 1.21.3 |
| 2.10 | 1.21.4 |


Ensure your GOPATH and PATH have been configured in accordance with the Go environment instructions.

Expand Down
2 changes: 1 addition & 1 deletion Makefile
Expand Up @@ -140,7 +140,7 @@ GOINSTALL=$(GOCMD) install
GOTEST=$(GOCMD) test
GODEP=$(GOTEST) -i
GOFMT=gofmt -w
GOBUILDIMAGE=golang:1.21.3
GOBUILDIMAGE=golang:1.21.4
GOBUILDPATHINCONTAINER=/harbor

# go build
Expand Down
5 changes: 5 additions & 0 deletions api/v2.0/swagger.yaml
Expand Up @@ -7867,6 +7867,11 @@ definitions:
x-nullable: true
x-omitempty: true
$ref: '#/definitions/AuthproxySetting'
oidc_provider_name:
type: string
x-nullable: true
x-omitempty: true
description: The OIDC provider name, empty if current auth is not OIDC_auth or OIDC provider is not configured.
AuthproxySetting:
type: object
properties:
Expand Down
2 changes: 1 addition & 1 deletion make/photon/registry/Dockerfile.binary
@@ -1,4 +1,4 @@
FROM golang:1.21.3
FROM golang:1.21.4

ENV DISTRIBUTION_DIR /go/src/github.com/docker/distribution
ENV BUILDTAGS include_oss include_gcs
Expand Down
2 changes: 1 addition & 1 deletion make/photon/trivy-adapter/Dockerfile.binary
@@ -1,4 +1,4 @@
FROM golang:1.21.3
FROM golang:1.21.4

ADD . /go/src/github.com/aquasecurity/harbor-scanner-trivy/
WORKDIR /go/src/github.com/aquasecurity/harbor-scanner-trivy/
Expand Down
2 changes: 1 addition & 1 deletion make/photon/trivy-adapter/builder.sh
Expand Up @@ -19,7 +19,7 @@ TEMP=$(mktemp -d ${TMPDIR-/tmp}/trivy-adapter.XXXXXX)
git clone https://github.com/aquasecurity/harbor-scanner-trivy.git $TEMP
cd $TEMP; git checkout $VERSION; cd -

echo "Building Trivy adapter binary based on golang:1.21.3..."
echo "Building Trivy adapter binary based on golang:1.21.4..."
cp Dockerfile.binary $TEMP
docker build -f $TEMP/Dockerfile.binary -t trivy-adapter-golang $TEMP

Expand Down
11 changes: 11 additions & 0 deletions src/common/const.go
Expand Up @@ -184,6 +184,7 @@ const (
TraceOtelTimeout = "trace_otel_timeout"

GDPRDeleteUser = "gdpr_delete_user"
GDPRAuditLogs = "gdpr_audit_logs"

// These variables are temporary solution for issue: https://github.com/goharbor/harbor/issues/16039
// When user disable the pull count/time/audit log, it will decrease the database access, especially in large concurrency pull scenarios.
Expand Down Expand Up @@ -230,4 +231,14 @@ const (
QuotaUpdateProvider = "quota_update_provider"
// IllegalCharsInUsername is the illegal chars in username
IllegalCharsInUsername = `,"~#%$`

// Beego web config
// BeegoMaxMemoryBytes is the max memory(bytes) of the beego web config
BeegoMaxMemoryBytes = "beego_max_memory_bytes"
// DefaultBeegoMaxMemoryBytes sets default max memory to 128GB
DefaultBeegoMaxMemoryBytes = 1 << 37
// BeegoMaxUploadSizeBytes is the max upload size(bytes) of the beego web config
BeegoMaxUploadSizeBytes = "beego_max_upload_size_bytes"
// DefaultBeegoMaxUploadSizeBytes sets default max upload size to 128GB
DefaultBeegoMaxUploadSizeBytes = 1 << 37
)
4 changes: 2 additions & 2 deletions src/controller/securityhub/controller.go
Expand Up @@ -86,8 +86,8 @@ func NewController() Controller {
func (c *controller) SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error) {
opts := newOptions(options...)
scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx)
if err != nil {
return nil, err
if len(scannerUUID) == 0 || err != nil {
return &secHubModel.Summary{}, nil
}
sum, err := c.secHubMgr.Summary(ctx, scannerUUID, projectID, nil)
if err != nil {
Expand Down
10 changes: 10 additions & 0 deletions src/controller/systeminfo/controller.go
Expand Up @@ -50,6 +50,7 @@ type Data struct {
BannerMessage string
AuthProxySettings *models.HTTPAuthProxy
Protected *protectedData
OIDCProviderName string
}

type protectedData struct {
Expand Down Expand Up @@ -103,6 +104,7 @@ func (c *controller) GetInfo(ctx context.Context, opt Options) (*Data, error) {
SelfRegistration: utils.SafeCastBool(cfg[common.SelfRegistration]),
HarborVersion: fmt.Sprintf("%s-%s", version.ReleaseVersion, version.GitCommit),
BannerMessage: utils.SafeCastString(mgr.Get(ctx, common.BannerMessage).GetString()),
OIDCProviderName: OIDCProviderName(cfg),
}
if res.AuthMode == common.HTTPAuth {
if s, err := config.HTTPAuthProxySetting(ctx); err == nil {
Expand Down Expand Up @@ -137,6 +139,14 @@ func (c *controller) GetInfo(ctx context.Context, opt Options) (*Data, error) {
return res, nil
}

func OIDCProviderName(cfg map[string]interface{}) string {
authMode := utils.SafeCastString(cfg[common.AUTHMode])
if authMode != common.OIDCAuth {
return ""
}
return utils.SafeCastString(cfg[common.OIDCName])
}

func (c *controller) GetCapacity(ctx context.Context) (*imagestorage.Capacity, error) {
systeminfo.Init()
return imagestorage.GlobalDriver.Cap()
Expand Down
22 changes: 22 additions & 0 deletions src/controller/systeminfo/controller_test.go
Expand Up @@ -105,3 +105,25 @@ func (s *sysInfoCtlTestSuite) TestGetInfo() {
func TestControllerSuite(t *testing.T) {
suite.Run(t, &sysInfoCtlTestSuite{})
}

func TestOIDCProviderName(t *testing.T) {
type args struct {
cfg map[string]interface{}
}
tests := []struct {
name string
args args
want string
}{
{"normal testing", args{map[string]interface{}{common.AUTHMode: common.OIDCAuth, common.OIDCName: "test"}}, "test"},
{"not oidc", args{map[string]interface{}{common.AUTHMode: common.DBAuth, common.OIDCName: "test"}}, ""},
{"empty provider", args{map[string]interface{}{common.AUTHMode: common.OIDCAuth, common.OIDCName: ""}}, ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := OIDCProviderName(tt.args.cfg); got != tt.want {
t.Errorf("OIDCProviderName() = %v, want %v", got, tt.want)
}
})
}
}
37 changes: 35 additions & 2 deletions src/controller/user/controller.go
Expand Up @@ -21,12 +21,15 @@ import (
commonmodels "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/security/local"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/job/impl/gdpr"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/member"
"github.com/goharbor/harbor/src/pkg/oidc"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/pkg/user"
"github.com/goharbor/harbor/src/pkg/user/models"
)
Expand Down Expand Up @@ -76,6 +79,8 @@ func NewController() Controller {
mgr: user.New(),
oidcMetaMgr: oidc.NewMetaMgr(),
memberMgr: member.Mgr,
taskMgr: task.NewManager(),
exeMgr: task.NewExecutionManager(),
}
}

Expand All @@ -88,6 +93,8 @@ type controller struct {
mgr user.Manager
oidcMetaMgr oidc.MetaManager
memberMgr member.Manager
taskMgr task.Manager
exeMgr task.ExecutionManager
}

func (c *controller) UpdateOIDCMeta(ctx context.Context, ou *commonmodels.OIDCUser, cols ...string) error {
Expand Down Expand Up @@ -183,10 +190,36 @@ func (c *controller) Delete(ctx context.Context, id int) error {
if err != nil {
return errors.UnknownError(err).WithMessage("failed to load GDPR setting: %v", err)
}

if gdprSetting.AuditLogs {
userDb, err := c.mgr.Get(ctx, id)
if err != nil {
return errors.Wrap(err, "unable to get user information")
}
params := map[string]interface{}{
gdpr.UserNameParam: userDb.Username,
}
execID, err := c.exeMgr.Create(ctx, job.AuditLogsGDPRCompliantVendorType, -1, task.ExecutionTriggerEvent, params)
if err != nil {
return err
}
_, err = c.taskMgr.Create(ctx, execID, &task.Job{
Name: job.AuditLogsGDPRCompliantVendorType,
Metadata: &job.Metadata{
JobKind: job.KindGeneric,
},
Parameters: params,
})
if err != nil {
return err
}
}
if gdprSetting.DeleteUser {
return c.mgr.DeleteGDPR(ctx, id)
err = c.mgr.DeleteGDPR(ctx, id)
} else {
err = c.mgr.Delete(ctx, id)
}
return c.mgr.Delete(ctx, id)
return err
}

func (c *controller) List(ctx context.Context, query *q.Query, options ...models.Option) ([]*commonmodels.User, error) {
Expand Down
8 changes: 6 additions & 2 deletions src/core/main.go
Expand Up @@ -127,8 +127,6 @@ func main() {

web.BConfig.WebConfig.Session.SessionOn = true
web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName
web.BConfig.MaxMemory = 1 << 35 // (32GB)
web.BConfig.MaxUploadSize = 1 << 35 // (32GB)
// the core db used for beego session
redisCoreURL := os.Getenv("_REDIS_URL_CORE")
if len(redisCoreURL) > 0 {
Expand Down Expand Up @@ -163,6 +161,12 @@ func main() {
log.Info("initializing configurations...")
config.Init()
log.Info("configurations initialization completed")

// default beego max memory and max upload size is 128GB, consider from some AI related image would be large,
// also support customize it from the environment variables if the default value cannot satisfy some scenarios.
web.BConfig.MaxMemory = config.GetBeegoMaxMemoryBytes()
web.BConfig.MaxUploadSize = config.GetBeegoMaxUploadSizeBytes()

metricCfg := config.Metric()
if metricCfg.Enabled {
metric.RegisterCollectors()
Expand Down
85 changes: 85 additions & 0 deletions src/jobservice/job/impl/gdpr/audit_logs_data_masking.go
@@ -0,0 +1,85 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gdpr

import (
"fmt"

"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/audit"
"github.com/goharbor/harbor/src/pkg/user"
)

const UserNameParam = "username"

type AuditLogsDataMasking struct {
manager audit.Manager
userManager user.Manager
}

func (a AuditLogsDataMasking) MaxFails() uint {
return 3
}

func (a AuditLogsDataMasking) MaxCurrency() uint {
return 1
}

func (a AuditLogsDataMasking) ShouldRetry() bool {
return true
}

func (a AuditLogsDataMasking) Validate(params job.Parameters) error {
if params == nil {
// Params are required
return errors.New("missing job parameters")
}
_, err := a.parseParams(params)
return err
}

func (a *AuditLogsDataMasking) init() {
if a.manager == nil {
a.manager = audit.New()
}
if a.userManager == nil {
a.userManager = user.New()
}
}

func (a AuditLogsDataMasking) Run(ctx job.Context, params job.Parameters) error {
logger := ctx.GetLogger()
logger.Info("GDPR audit logs data masking job started")
a.init()
username, err := a.parseParams(params)
if err != nil {
return err
}
logger.Infof("Masking log entries for a user: %s", username)
return a.manager.UpdateUsername(ctx.SystemContext(), username, a.userManager.GenerateCheckSum(username))
}

func (a AuditLogsDataMasking) parseParams(params job.Parameters) (string, error) {
value, exist := params[UserNameParam]
if !exist {
return "", fmt.Errorf("param %s not found", UserNameParam)
}
str, ok := value.(string)
if !ok {
return "", fmt.Errorf("the value of %s isn't string", UserNameParam)
}
return str, nil
}

0 comments on commit 5650e27

Please sign in to comment.