228 changes: 228 additions & 0 deletions modules/git/foreachref/parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package foreachref_test

import (
"errors"
"fmt"
"io"
"strings"
"testing"

"code.gitea.io/gitea/modules/git/foreachref"
"code.gitea.io/gitea/modules/json"

"github.com/stretchr/testify/require"
)

type refSlice = []map[string]string

func TestParser(t *testing.T) {
tests := []struct {
name string

givenFormat foreachref.Format
givenInput io.Reader

wantRefs refSlice
wantErr bool
expectedErr error
}{
// this would, for example, be the result when running `git
// for-each-ref refs/tags` on a repo without tags.
{
name: "no references on empty input",

givenFormat: foreachref.NewFormat("refname:short"),
givenInput: strings.NewReader(``),

wantRefs: []map[string]string{},
},

// note: `git for-each-ref` will add a newline between every
// reference (in addition to the ref-delimiter we've chosen)
{
name: "single field requested, single reference in output",

givenFormat: foreachref.NewFormat("refname:short"),
givenInput: strings.NewReader("refname:short v0.0.1\x00\x00" + "\n"),

wantRefs: []map[string]string{
{"refname:short": "v0.0.1"},
},
},
{
name: "single field requested, multiple references in output",

givenFormat: foreachref.NewFormat("refname:short"),
givenInput: strings.NewReader(
"refname:short v0.0.1\x00\x00" + "\n" +
"refname:short v0.0.2\x00\x00" + "\n" +
"refname:short v0.0.3\x00\x00" + "\n"),

wantRefs: []map[string]string{
{"refname:short": "v0.0.1"},
{"refname:short": "v0.0.2"},
{"refname:short": "v0.0.3"},
},
},

{
name: "multiple fields requested for each reference",

givenFormat: foreachref.NewFormat("refname:short", "objecttype", "objectname"),
givenInput: strings.NewReader(

"refname:short v0.0.1\x00objecttype commit\x00objectname 7b2c5ac9fc04fc5efafb60700713d4fa609b777b\x00\x00" + "\n" +
"refname:short v0.0.2\x00objecttype commit\x00objectname a1f051bc3eba734da4772d60e2d677f47cf93ef4\x00\x00" + "\n" +
"refname:short v0.0.3\x00objecttype commit\x00objectname ef82de70bb3f60c65fb8eebacbb2d122ef517385\x00\x00" + "\n",
),

wantRefs: []map[string]string{
{
"refname:short": "v0.0.1",
"objecttype": "commit",
"objectname": "7b2c5ac9fc04fc5efafb60700713d4fa609b777b",
},
{
"refname:short": "v0.0.2",
"objecttype": "commit",
"objectname": "a1f051bc3eba734da4772d60e2d677f47cf93ef4",
},
{
"refname:short": "v0.0.3",
"objecttype": "commit",
"objectname": "ef82de70bb3f60c65fb8eebacbb2d122ef517385",
},
},
},

{
name: "must handle multi-line fields such as 'content'",

givenFormat: foreachref.NewFormat("refname:short", "contents", "author"),
givenInput: strings.NewReader(
"refname:short v0.0.1\x00contents Create new buffer if not present yet (#549)\n\nFixes a nil dereference when ProcessFoo is used\nwith multiple commands.\x00author Foo Bar <foo@bar.com> 1507832733 +0200\x00\x00" + "\n" +
"refname:short v0.0.2\x00contents Update CI config (#651)\n\n\x00author John Doe <john.doe@foo.com> 1521643174 +0000\x00\x00" + "\n" +
"refname:short v0.0.3\x00contents Fixed code sample for bash completion (#687)\n\n\x00author Foo Baz <foo@baz.com> 1524836750 +0200\x00\x00" + "\n",
),

wantRefs: []map[string]string{
{
"refname:short": "v0.0.1",
"contents": "Create new buffer if not present yet (#549)\n\nFixes a nil dereference when ProcessFoo is used\nwith multiple commands.",
"author": "Foo Bar <foo@bar.com> 1507832733 +0200",
},
{
"refname:short": "v0.0.2",
"contents": "Update CI config (#651)",
"author": "John Doe <john.doe@foo.com> 1521643174 +0000",
},
{
"refname:short": "v0.0.3",
"contents": "Fixed code sample for bash completion (#687)",
"author": "Foo Baz <foo@baz.com> 1524836750 +0200",
},
},
},

{
name: "must handle fields without values",

givenFormat: foreachref.NewFormat("refname:short", "object", "objecttype"),
givenInput: strings.NewReader(
"refname:short v0.0.1\x00object \x00objecttype commit\x00\x00" + "\n" +
"refname:short v0.0.2\x00object \x00objecttype commit\x00\x00" + "\n" +
"refname:short v0.0.3\x00object \x00objecttype commit\x00\x00" + "\n",
),

wantRefs: []map[string]string{
{
"refname:short": "v0.0.1",
"object": "",
"objecttype": "commit",
},
{
"refname:short": "v0.0.2",
"object": "",
"objecttype": "commit",
},
{
"refname:short": "v0.0.3",
"object": "",
"objecttype": "commit",
},
},
},

{
name: "must fail when the number of fields in the input doesn't match expected format",

givenFormat: foreachref.NewFormat("refname:short", "objecttype", "objectname"),
givenInput: strings.NewReader(
"refname:short v0.0.1\x00objecttype commit\x00\x00" + "\n" +
"refname:short v0.0.2\x00objecttype commit\x00\x00" + "\n" +
"refname:short v0.0.3\x00objecttype commit\x00\x00" + "\n",
),

wantErr: true,
expectedErr: errors.New("unexpected number of reference fields: wanted 2, was 3"),
},

{
name: "must fail input fields don't match expected format",

givenFormat: foreachref.NewFormat("refname:short", "objectname"),
givenInput: strings.NewReader(
"refname:short v0.0.1\x00objecttype commit\x00\x00" + "\n" +
"refname:short v0.0.2\x00objecttype commit\x00\x00" + "\n" +
"refname:short v0.0.3\x00objecttype commit\x00\x00" + "\n",
),

wantErr: true,
expectedErr: errors.New("unexpected field name at position 1: wanted: 'objectname', was: 'objecttype'"),
},
}

for _, test := range tests {
tc := test // don't close over loop variable
t.Run(tc.name, func(t *testing.T) {
parser := tc.givenFormat.Parser(tc.givenInput)

//
// parse references from input
//
gotRefs := make([]map[string]string, 0)
for {
ref := parser.Next()
if ref == nil {
break
}
gotRefs = append(gotRefs, ref)
}
err := parser.Err()

//
// verify expectations
//
if tc.wantErr {
require.Error(t, err)
require.EqualError(t, err, tc.expectedErr.Error())
} else {
require.NoError(t, err, "for-each-ref parser unexpectedly failed with: %v", err)
require.Equal(t, tc.wantRefs, gotRefs, "for-each-ref parser produced unexpected reference set. wanted: %v, got: %v", pretty(tc.wantRefs), pretty(gotRefs))
}
})
}
}

func pretty(v interface{}) string {
data, err := json.MarshalIndent(v, "", " ")
if err != nil {
// shouldn't happen
panic(fmt.Sprintf("json-marshalling failed: %v", err))
}
return string(data)
}
101 changes: 82 additions & 19 deletions modules/git/repo_tag.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ package git
import (
"context"
"fmt"
"io"
"strings"

"code.gitea.io/gitea/modules/git/foreachref"
"code.gitea.io/gitea/modules/util"
)

Expand Down Expand Up @@ -111,37 +113,98 @@ func (repo *Repository) GetTagWithID(idStr, name string) (*Tag, error) {

// GetTagInfos returns all tag infos of the repository.
func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) {
// TODO this a slow implementation, makes one git command per tag
stdout, err := NewCommand(repo.Ctx, "tag").RunInDir(repo.Path)
if err != nil {
return nil, 0, err
}
forEachRefFmt := foreachref.NewFormat("objecttype", "refname:short", "object", "objectname", "creator", "contents", "contents:signature")

tagNames := strings.Split(strings.TrimRight(stdout, "\n"), "\n")
tagsTotal := len(tagNames)

if page != 0 {
tagNames = util.PaginateSlice(tagNames, page, pageSize).([]string)
}
stdoutReader, stdoutWriter := io.Pipe()
defer stdoutReader.Close()
defer stdoutWriter.Close()
stderr := strings.Builder{}
rc := &RunContext{Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr, Timeout: -1}

tags := make([]*Tag, 0, len(tagNames))
for _, tagName := range tagNames {
tagName = strings.TrimSpace(tagName)
if len(tagName) == 0 {
continue
go func() {
err := NewCommand(repo.Ctx, "for-each-ref", "--format", forEachRefFmt.Flag(), "--sort", "-*creatordate", "refs/tags").RunWithContext(rc)
if err != nil {
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String()))
} else {
_ = stdoutWriter.Close()
}
}()

var tags []*Tag
parser := forEachRefFmt.Parser(stdoutReader)
for {
ref := parser.Next()
if ref == nil {
break
}

tag, err := repo.GetTag(tagName)
tag, err := parseTagRef(ref)
if err != nil {
return nil, tagsTotal, err
return nil, 0, fmt.Errorf("GetTagInfos: parse tag: %w", err)
}
tag.Name = tagName
tags = append(tags, tag)
}
if err := parser.Err(); err != nil {
return nil, 0, fmt.Errorf("GetTagInfos: parse output: %w", err)
}

sortTagsByTime(tags)
tagsTotal := len(tags)
if page != 0 {
tags = util.PaginateSlice(tags, page, pageSize).([]*Tag)
}

return tags, tagsTotal, nil
}

// parseTagRef parses a tag from a 'git for-each-ref'-produced reference.
func parseTagRef(ref map[string]string) (tag *Tag, err error) {
tag = &Tag{
Type: ref["objecttype"],
Name: ref["refname:short"],
}

tag.ID, err = NewIDFromString(ref["objectname"])
if err != nil {
return nil, fmt.Errorf("parse objectname '%s': %v", ref["objectname"], err)
}

if tag.Type == "commit" {
// lightweight tag
tag.Object = tag.ID
} else {
// annotated tag
tag.Object, err = NewIDFromString(ref["object"])
if err != nil {
return nil, fmt.Errorf("parse object '%s': %v", ref["object"], err)
}
}

tag.Tagger, err = newSignatureFromCommitline([]byte(ref["creator"]))
if err != nil {
return nil, fmt.Errorf("parse tagger: %w", err)
}

tag.Message = ref["contents"]
// strip PGP signature if present in contents field
pgpStart := strings.Index(tag.Message, beginpgp)
if pgpStart >= 0 {
tag.Message = tag.Message[0:pgpStart]
}

// annotated tag with GPG signature
if tag.Type == "tag" && ref["contents:signature"] != "" {
payload := fmt.Sprintf("object %s\ntype commit\ntag %s\ntagger %s\n\n%s\n",
tag.Object, tag.Name, ref["creator"], strings.TrimSpace(tag.Message))
tag.Signature = &CommitGPGSignature{
Signature: ref["contents:signature"],
Payload: payload,
}
}

return tag, nil
}

// GetAnnotatedTag returns a Git tag by its SHA, must be an annotated tag
func (repo *Repository) GetAnnotatedTag(sha string) (*Tag, error) {
id, err := NewIDFromString(sha)
Expand Down
182 changes: 182 additions & 0 deletions modules/git/repo_tag_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"code.gitea.io/gitea/modules/util"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestRepository_GetTags(t *testing.T) {
Expand Down Expand Up @@ -195,3 +196,184 @@ func TestRepository_GetAnnotatedTag(t *testing.T) {
assert.True(t, IsErrNotExist(err))
assert.Nil(t, tag4)
}

func TestRepository_parseTagRef(t *testing.T) {
tests := []struct {
name string

givenRef map[string]string

want *Tag
wantErr bool
expectedErr error
}{
{
name: "lightweight tag",

givenRef: map[string]string{
"objecttype": "commit",
"refname:short": "v1.9.1",
// object will be empty for lightweight tags
"object": "",
"objectname": "ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889",
"creator": "Foo Bar <foo@bar.com> 1565789218 +0300",
"contents": `Add changelog of v1.9.1 (#7859)
* add changelog of v1.9.1
* Update CHANGELOG.md
`,
"contents:signature": "",
},

want: &Tag{
Name: "v1.9.1",
ID: MustIDFromString("ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889"),
Object: MustIDFromString("ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889"),
Type: "commit",
Tagger: parseAuthorLine(t, "Foo Bar <foo@bar.com> 1565789218 +0300"),
Message: "Add changelog of v1.9.1 (#7859)\n\n* add changelog of v1.9.1\n* Update CHANGELOG.md\n",
Signature: nil,
},
},

{
name: "annotated tag",

givenRef: map[string]string{
"objecttype": "tag",
"refname:short": "v0.0.1",
// object will refer to commit hash for annotated tag
"object": "3325fd8a973321fd59455492976c042dde3fd1ca",
"objectname": "8c68a1f06fc59c655b7e3905b159d761e91c53c9",
"creator": "Foo Bar <foo@bar.com> 1565789218 +0300",
"contents": `Add changelog of v1.9.1 (#7859)
* add changelog of v1.9.1
* Update CHANGELOG.md
`,
"contents:signature": "",
},

want: &Tag{
Name: "v0.0.1",
ID: MustIDFromString("8c68a1f06fc59c655b7e3905b159d761e91c53c9"),
Object: MustIDFromString("3325fd8a973321fd59455492976c042dde3fd1ca"),
Type: "tag",
Tagger: parseAuthorLine(t, "Foo Bar <foo@bar.com> 1565789218 +0300"),
Message: "Add changelog of v1.9.1 (#7859)\n\n* add changelog of v1.9.1\n* Update CHANGELOG.md\n",
Signature: nil,
},
},

{
name: "annotated tag with signature",

givenRef: map[string]string{
"objecttype": "tag",
"refname:short": "v0.0.1",
"object": "3325fd8a973321fd59455492976c042dde3fd1ca",
"objectname": "8c68a1f06fc59c655b7e3905b159d761e91c53c9",
"creator": "Foo Bar <foo@bar.com> 1565789218 +0300",
"contents": `Add changelog of v1.9.1 (#7859)
* add changelog of v1.9.1
* Update CHANGELOG.md
-----BEGIN PGP SIGNATURE-----
aBCGzBAABCgAdFiEEyWRwv/q1Q6IjSv+D4IPOwzt33PoFAmI8jbIACgkQ4IPOwzt3
3PoRuAv9FVSbPBXvzECubls9KQd7urwEvcfG20Uf79iBwifQJUv+egNQojrs6APT
T4CdIXeGRpwJZaGTUX9RWnoDO1SLXAWnc82CypWraNwrHq8Go2YeoVu0Iy3vb0EU
REdob/tXYZecMuP8AjhUR0XfdYaERYAvJ2dYsH/UkFrqDjM3V4kPXWG+R5DCaZiE
slB5U01i4Dwb/zm/ckzhUGEcOgcnpOKX8SnY5kYRVDY47dl/yJZ1u2XWir3mu60G
1geIitH7StBddHi/8rz+sJwTfcVaLjn2p59p/Dr9aGbk17GIaKq1j0pZA2lKT0Xt
f9jDqU+9vCxnKgjSDhrwN69LF2jT47ZFjEMGV/wFPOa1EBxVWpgQ/CfEolBlbUqx
yVpbxi/6AOK2lmG130e9jEZJcu+WeZUeq851WgKSEkf2d5f/JpwtSTEOlOedu6V6
kl845zu5oE2nKM4zMQ7XrYQn538I31ps+VGQ0H8R07WrZP8WKUWugL2cU8KmXFwg
qbHDASXl
=2yGi
-----END PGP SIGNATURE-----
`,
"contents:signature": `-----BEGIN PGP SIGNATURE-----
aBCGzBAABCgAdFiEEyWRwv/q1Q6IjSv+D4IPOwzt33PoFAmI8jbIACgkQ4IPOwzt3
3PoRuAv9FVSbPBXvzECubls9KQd7urwEvcfG20Uf79iBwifQJUv+egNQojrs6APT
T4CdIXeGRpwJZaGTUX9RWnoDO1SLXAWnc82CypWraNwrHq8Go2YeoVu0Iy3vb0EU
REdob/tXYZecMuP8AjhUR0XfdYaERYAvJ2dYsH/UkFrqDjM3V4kPXWG+R5DCaZiE
slB5U01i4Dwb/zm/ckzhUGEcOgcnpOKX8SnY5kYRVDY47dl/yJZ1u2XWir3mu60G
1geIitH7StBddHi/8rz+sJwTfcVaLjn2p59p/Dr9aGbk17GIaKq1j0pZA2lKT0Xt
f9jDqU+9vCxnKgjSDhrwN69LF2jT47ZFjEMGV/wFPOa1EBxVWpgQ/CfEolBlbUqx
yVpbxi/6AOK2lmG130e9jEZJcu+WeZUeq851WgKSEkf2d5f/JpwtSTEOlOedu6V6
kl845zu5oE2nKM4zMQ7XrYQn538I31ps+VGQ0H8R07WrZP8WKUWugL2cU8KmXFwg
qbHDASXl
=2yGi
-----END PGP SIGNATURE-----
`,
},

want: &Tag{
Name: "v0.0.1",
ID: MustIDFromString("8c68a1f06fc59c655b7e3905b159d761e91c53c9"),
Object: MustIDFromString("3325fd8a973321fd59455492976c042dde3fd1ca"),
Type: "tag",
Tagger: parseAuthorLine(t, "Foo Bar <foo@bar.com> 1565789218 +0300"),
Message: "Add changelog of v1.9.1 (#7859)\n\n* add changelog of v1.9.1\n* Update CHANGELOG.md",
Signature: &CommitGPGSignature{
Signature: `-----BEGIN PGP SIGNATURE-----
aBCGzBAABCgAdFiEEyWRwv/q1Q6IjSv+D4IPOwzt33PoFAmI8jbIACgkQ4IPOwzt3
3PoRuAv9FVSbPBXvzECubls9KQd7urwEvcfG20Uf79iBwifQJUv+egNQojrs6APT
T4CdIXeGRpwJZaGTUX9RWnoDO1SLXAWnc82CypWraNwrHq8Go2YeoVu0Iy3vb0EU
REdob/tXYZecMuP8AjhUR0XfdYaERYAvJ2dYsH/UkFrqDjM3V4kPXWG+R5DCaZiE
slB5U01i4Dwb/zm/ckzhUGEcOgcnpOKX8SnY5kYRVDY47dl/yJZ1u2XWir3mu60G
1geIitH7StBddHi/8rz+sJwTfcVaLjn2p59p/Dr9aGbk17GIaKq1j0pZA2lKT0Xt
f9jDqU+9vCxnKgjSDhrwN69LF2jT47ZFjEMGV/wFPOa1EBxVWpgQ/CfEolBlbUqx
yVpbxi/6AOK2lmG130e9jEZJcu+WeZUeq851WgKSEkf2d5f/JpwtSTEOlOedu6V6
kl845zu5oE2nKM4zMQ7XrYQn538I31ps+VGQ0H8R07WrZP8WKUWugL2cU8KmXFwg
qbHDASXl
=2yGi
-----END PGP SIGNATURE-----
`,
Payload: `object 3325fd8a973321fd59455492976c042dde3fd1ca
type commit
tag v0.0.1
tagger Foo Bar <foo@bar.com> 1565789218 +0300
Add changelog of v1.9.1 (#7859)
* add changelog of v1.9.1
* Update CHANGELOG.md
`,
},
},
},
}

for _, test := range tests {
tc := test // don't close over loop variable
t.Run(tc.name, func(t *testing.T) {
got, err := parseTagRef(tc.givenRef)

if tc.wantErr {
require.Error(t, err)
require.ErrorIs(t, err, tc.expectedErr)
} else {
require.NoError(t, err)
require.Equal(t, tc.want, got)
}
})
}
}

func parseAuthorLine(t *testing.T, committer string) *Signature {
t.Helper()

sig, err := newSignatureFromCommitline([]byte(committer))
if err != nil {
t.Fatalf("parse author line '%s': %v", committer, err)
}

return sig
}
60 changes: 60 additions & 0 deletions modules/repository/repo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
}

if !opts.Releases {
// note: this will greatly improve release (tag) sync
// for pull-mirrors with many tags
repo.IsMirror = opts.Mirror
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Error("Failed to synchronize tags to releases for repository: %v", err)
}
Expand Down Expand Up @@ -254,6 +257,14 @@ func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo

// SyncReleasesWithTags synchronizes release table with repository tags
func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) error {
log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)

// optimized procedure for pull-mirrors which saves a lot of time (in
// particular for repos with many tags).
if repo.IsMirror {
return pullMirrorReleaseSync(repo, gitRepo)
}

existingRelTags := make(map[string]struct{})
opts := models.FindReleasesOptions{
IncludeDrafts: true,
Expand Down Expand Up @@ -450,3 +461,52 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re

return nil
}

// pullMirrorReleaseSync is a pull-mirror specific tag<->release table
// synchronization which overwrites all Releases from the repository tags. This
// can be relied on since a pull-mirror is always identical to its
// upstream. Hence, after each sync we want the pull-mirror release set to be
// identical to the upstream tag set. This is much more efficient for
// repositories like https://github.com/vim/vim (with over 13000 tags).
func pullMirrorReleaseSync(repo *repo_model.Repository, gitRepo *git.Repository) error {
log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
tags, numTags, err := gitRepo.GetTagInfos(0, 0)
if err != nil {
return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
}
err = db.WithTx(func(ctx context.Context) error {
//
// clear out existing releases
//
if _, err := db.DeleteByBean(ctx, &models.Release{RepoID: repo.ID}); err != nil {
return fmt.Errorf("unable to clear releases for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
}
//
// make release set identical to upstream tags
//
for _, tag := range tags {
release := models.Release{
RepoID: repo.ID,
TagName: tag.Name,
LowerTagName: strings.ToLower(tag.Name),
Sha1: tag.Object.String(),
// NOTE: ignored, since NumCommits are unused
// for pull-mirrors (only relevant when
// displaying releases, IsTag: false)
NumCommits: -1,
CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()),
IsTag: true,
}
if err := db.Insert(ctx, release); err != nil {
return fmt.Errorf("unable insert tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err)
}
}
return nil
})
if err != nil {
return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
}

log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags)
return nil
}