Permalink
Browse files

Write per-source-package index pages for tracker.debian.org

fixes #94
  • Loading branch information...
stapelberg committed Jan 13, 2018
1 parent 01e1c55 commit af8eedcb2c884e6c201b6bd651d4f37bfda4ede8
Showing with 156 additions and 21 deletions.
  1. +23 −0 assets/srcpkgindex.tmpl
  2. +1 −1 bundle.go
  3. +1 −0 cmd/debiman/main.go
  4. +62 −0 cmd/debiman/render.go
  5. +47 −0 cmd/debiman/renderpkgindex.go
  6. +22 −20 internal/bundled/GENERATED_bundled.go
@@ -0,0 +1,23 @@
{{ template "header" . }}

<div class="maincontents">

<h1>Manpages of <a href="https://tracker.debian.org/pkg/{{ .Src }}">src:{{ .Src }}</a> in Debian {{ .First.Package.Suite }}</h1>

<ul>
{{ range $idx, $fn := .Mans }}
{{ with $m := index $.ManpageByName $fn }}
<li>
<a href="{{ BaseURLPath }}/{{ $m.ServingPath }}.html">{{ $m.Name }}({{ $m.Section }})
{{ if ne $m.Language "en" }}
(<span title="{{ EnglishLang $m.LanguageTag }} ({{ $m.Language }})">{{ DisplayLang $m.LanguageTag }}</span>)
{{ end }}
</a>
</li>
{{ end }}
{{ end }}
</ul>

</div>

{{ template "footer" . }}
@@ -1,3 +1,3 @@
package bundle

//go:generate sh -c "go run goembed.go -package bundled -var assets assets/header.tmpl assets/footer.tmpl assets/style.css assets/manpage.tmpl assets/manpageerror.tmpl assets/manpagefooterextra.tmpl assets/contents.tmpl assets/pkgindex.tmpl assets/index.tmpl assets/faq.tmpl assets/notfound.tmpl assets/Inconsolata.woff assets/Inconsolata.woff2 assets/opensearch.xml assets/Roboto-Bold.woff assets/Roboto-Bold.woff2 assets/Roboto-Regular.woff assets/Roboto-Regular.woff2 > internal/bundled/GENERATED_bundled.go"
//go:generate sh -c "go run goembed.go -package bundled -var assets assets/header.tmpl assets/footer.tmpl assets/style.css assets/manpage.tmpl assets/manpageerror.tmpl assets/manpagefooterextra.tmpl assets/contents.tmpl assets/pkgindex.tmpl assets/srcpkgindex.tmpl assets/index.tmpl assets/faq.tmpl assets/notfound.tmpl assets/Inconsolata.woff assets/Inconsolata.woff2 assets/opensearch.xml assets/Roboto-Bold.woff assets/Roboto-Bold.woff2 assets/Roboto-Regular.woff assets/Roboto-Regular.woff2 > internal/bundled/GENERATED_bundled.go"
@@ -160,6 +160,7 @@ func main() {
commonTmpls = commontmpl.MustParseCommonTmpls()
contentsTmpl = mustParseContentsTmpl()
pkgindexTmpl = mustParsePkgindexTmpl()
srcpkgindexTmpl = mustParseSrcPkgindexTmpl()
indexTmpl = mustParseIndexTmpl()
faqTmpl = mustParseFaqTmpl()
aboutTmpl = mustParseAboutTmpl()
@@ -441,7 +441,65 @@ func walkContents(ctx context.Context, renderChan chan<- renderJob, whitelist ma
})
}

func writeSourceIndex(gv globalView, newestForSource map[string]time.Time) error {
// Partition by suite for reduced memory usage and better locality of file
// system access
for suite := range gv.suites {
binariesBySource := make(map[string][]string)
for _, p := range gv.pkgs {
if p.suite != suite {
continue
}
binariesBySource[p.source] = append(binariesBySource[p.source], p.binarypkg)
}

for src, binaries := range binariesBySource {
srcDir := filepath.Join(*servingDir, suite, "src:"+src)
// skip if current index file is more recent than newestForSource
st, err := os.Stat(filepath.Join(srcDir, "index.html.gz"))
if !*forceRerender && err == nil && st.ModTime().After(newestForSource[src]) {
continue
}

// Aggregate manpages of all binary packages for this source package
manpages := make(map[string]*manpage.Meta)
for _, binary := range binaries {
m, err := listManpages(filepath.Join(*servingDir, suite, binary))
if err != nil {
if os.IsNotExist(err) {
continue // The package might not contain any manpages.
}
return err
}
for k, v := range m {
manpages[k] = v
}
}
if len(manpages) == 0 {
continue // The entire source package does not contain any manpages.
}

if err := os.MkdirAll(srcDir, 0755); err != nil {
return err
}
if err := renderSrcPkgindex(filepath.Join(srcDir, "index.html.gz"), src, manpages); err != nil {
return err
}
}
}
return nil
}

func renderAll(gv globalView) error {
log.Printf("Preparing inverted maps")
sourceByBinary := make(map[string]string, len(gv.pkgs))
newestForSource := make(map[string]time.Time)
for _, p := range gv.pkgs {
sourceByBinary[p.suite+"/"+p.binarypkg] = p.source
newestForSource[p.source] = time.Time{}
}
log.Printf("%d sourceByBinary entries, %d newestForSource entries", len(sourceByBinary), len(newestForSource))

eg, ctx := errgroup.WithContext(context.Background())
renderChan := make(chan renderJob)
for i := 0; i < *renderConcurrency; i++ {
@@ -496,6 +554,10 @@ func renderAll(gv globalView) error {
return err
}

if err := writeSourceIndex(gv, newestForSource); err != nil {
return fmt.Errorf("writing source index: %v", err)
}

suitedirs, err := ioutil.ReadDir(*servingDir)
if err != nil {
return err
@@ -12,11 +12,16 @@ import (
)

var pkgindexTmpl = mustParsePkgindexTmpl()
var srcpkgindexTmpl = mustParseSrcPkgindexTmpl()

func mustParsePkgindexTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("pkgindex").Parse(bundled.Asset("pkgindex.tmpl")))
}

func mustParseSrcPkgindexTmpl() *template.Template {
return template.Must(template.Must(commonTmpls.Clone()).New("srcpkgindex").Parse(bundled.Asset("srcpkgindex.tmpl")))
}

func renderPkgindex(dest string, manpageByName map[string]*manpage.Meta) error {
var first *manpage.Meta
for _, m := range manpageByName {
@@ -56,3 +61,45 @@ func renderPkgindex(dest string, manpageByName map[string]*manpage.Meta) error {
})
})
}

func renderSrcPkgindex(dest string, src string, manpageByName map[string]*manpage.Meta) error {
var first *manpage.Meta
for _, m := range manpageByName {
first = m
break
}

mans := make([]string, 0, len(manpageByName))
for n := range manpageByName {
mans = append(mans, n)
}
sort.Strings(mans)

return write.Atomically(dest, true, func(w io.Writer) error {
return srcpkgindexTmpl.Execute(w, struct {
Title string
DebimanVersion string
Breadcrumbs breadcrumbs
FooterExtra string
First *manpage.Meta
Meta *manpage.Meta
ManpageByName map[string]*manpage.Meta
Mans []string
HrefLangs []*manpage.Meta
Src string
}{
Title: fmt.Sprintf("Manpages of src:%s in Debian %s", src, first.Package.Suite),
DebimanVersion: debimanVersion,
Breadcrumbs: breadcrumbs{
{fmt.Sprintf("/contents-%s.html", first.Package.Suite), first.Package.Suite},
{fmt.Sprintf("/%s/%s/index.html", first.Package.Suite, first.Package.Binarypkg), first.Package.Binarypkg},
{"", "Contents"},
},
First: first,
Meta: first,
ManpageByName: manpageByName,
Mans: mans,
Src: src,
})
})
}
Oops, something went wrong.

0 comments on commit af8eedc

Please sign in to comment.