Write per-source-package index pages for tracker.debian.org

fixes #94

assets/srcpkgindex.tmpl

@@ -0,0 +1,23 @@
+{{ template "header" . }}
+
+<div class="maincontents">
+
+<h1>Manpages of <a href="https://tracker.debian.org/pkg/{{ .Src }}">src:{{ .Src }}</a> in Debian {{ .First.Package.Suite }}</h1>
+
+<ul>
+{{ range $idx, $fn := .Mans }}
+  {{ with $m := index $.ManpageByName $fn }}
+<li>
+  <a href="{{ BaseURLPath }}/{{ $m.ServingPath }}.html">{{ $m.Name }}({{ $m.Section }})
+    {{ if ne $m.Language "en" }}
+      (<span title="{{ EnglishLang $m.LanguageTag }} ({{ $m.Language }})">{{ DisplayLang $m.LanguageTag }}</span>)
+    {{ end }}
+  </a>
+</li>
+  {{ end }}
+{{ end }}
+</ul>
+
+</div>
+
+{{ template "footer" . }}

bundle.go

@@ -1,3 +1,3 @@
 package bundle
 
-//go:generate sh -c "go run goembed.go -package bundled -var assets assets/header.tmpl assets/footer.tmpl assets/style.css assets/manpage.tmpl assets/manpageerror.tmpl assets/manpagefooterextra.tmpl assets/contents.tmpl assets/pkgindex.tmpl assets/index.tmpl assets/faq.tmpl assets/notfound.tmpl assets/Inconsolata.woff assets/Inconsolata.woff2 assets/opensearch.xml assets/Roboto-Bold.woff assets/Roboto-Bold.woff2 assets/Roboto-Regular.woff assets/Roboto-Regular.woff2 > internal/bundled/GENERATED_bundled.go"
+//go:generate sh -c "go run goembed.go -package bundled -var assets assets/header.tmpl assets/footer.tmpl assets/style.css assets/manpage.tmpl assets/manpageerror.tmpl assets/manpagefooterextra.tmpl assets/contents.tmpl assets/pkgindex.tmpl assets/srcpkgindex.tmpl assets/index.tmpl assets/faq.tmpl assets/notfound.tmpl assets/Inconsolata.woff assets/Inconsolata.woff2 assets/opensearch.xml assets/Roboto-Bold.woff assets/Roboto-Bold.woff2 assets/Roboto-Regular.woff assets/Roboto-Regular.woff2 > internal/bundled/GENERATED_bundled.go"

cmd/debiman/main.go

@@ -160,6 +160,7 @@ func main() {
 		commonTmpls = commontmpl.MustParseCommonTmpls()
 		contentsTmpl = mustParseContentsTmpl()
 		pkgindexTmpl = mustParsePkgindexTmpl()
+		srcpkgindexTmpl = mustParseSrcPkgindexTmpl()
 		indexTmpl = mustParseIndexTmpl()
 		faqTmpl = mustParseFaqTmpl()
 		aboutTmpl = mustParseAboutTmpl()

cmd/debiman/render.go

@@ -441,7 +441,65 @@ func walkContents(ctx context.Context, renderChan chan<- renderJob, whitelist ma
 	})
 }
 
+func writeSourceIndex(gv globalView, newestForSource map[string]time.Time) error {
+	// Partition by suite for reduced memory usage and better locality of file
+	// system access
+	for suite := range gv.suites {
+		binariesBySource := make(map[string][]string)
+		for _, p := range gv.pkgs {
+			if p.suite != suite {
+				continue
+			}
+			binariesBySource[p.source] = append(binariesBySource[p.source], p.binarypkg)
+		}
+
+		for src, binaries := range binariesBySource {
+			srcDir := filepath.Join(*servingDir, suite, "src:"+src)
+			// skip if current index file is more recent than newestForSource
+			st, err := os.Stat(filepath.Join(srcDir, "index.html.gz"))
+			if !*forceRerender && err == nil && st.ModTime().After(newestForSource[src]) {
+				continue
+			}
+
+			// Aggregate manpages of all binary packages for this source package
+			manpages := make(map[string]*manpage.Meta)
+			for _, binary := range binaries {
+				m, err := listManpages(filepath.Join(*servingDir, suite, binary))
+				if err != nil {
+					if os.IsNotExist(err) {
+						continue // The package might not contain any manpages.
+					}
+					return err
+				}
+				for k, v := range m {
+					manpages[k] = v
+				}
+			}
+			if len(manpages) == 0 {
+				continue // The entire source package does not contain any manpages.
+			}
+
+			if err := os.MkdirAll(srcDir, 0755); err != nil {
+				return err
+			}
+			if err := renderSrcPkgindex(filepath.Join(srcDir, "index.html.gz"), src, manpages); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func renderAll(gv globalView) error {
+	log.Printf("Preparing inverted maps")
+	sourceByBinary := make(map[string]string, len(gv.pkgs))
+	newestForSource := make(map[string]time.Time)
+	for _, p := range gv.pkgs {
+		sourceByBinary[p.suite+"/"+p.binarypkg] = p.source
+		newestForSource[p.source] = time.Time{}
+	}
+	log.Printf("%d sourceByBinary entries, %d newestForSource entries", len(sourceByBinary), len(newestForSource))
+
 	eg, ctx := errgroup.WithContext(context.Background())
 	renderChan := make(chan renderJob)
 	for i := 0; i < *renderConcurrency; i++ {
@@ -496,6 +554,10 @@ func renderAll(gv globalView) error {
 		return err
 	}
 
+	if err := writeSourceIndex(gv, newestForSource); err != nil {
+		return fmt.Errorf("writing source index: %v", err)
+	}
+
 	suitedirs, err := ioutil.ReadDir(*servingDir)
 	if err != nil {
 		return err

cmd/debiman/renderpkgindex.go

@@ -12,11 +12,16 @@ import (
 )
 
 var pkgindexTmpl = mustParsePkgindexTmpl()
+var srcpkgindexTmpl = mustParseSrcPkgindexTmpl()
 
 func mustParsePkgindexTmpl() *template.Template {
 	return template.Must(template.Must(commonTmpls.Clone()).New("pkgindex").Parse(bundled.Asset("pkgindex.tmpl")))
 }
 
+func mustParseSrcPkgindexTmpl() *template.Template {
+	return template.Must(template.Must(commonTmpls.Clone()).New("srcpkgindex").Parse(bundled.Asset("srcpkgindex.tmpl")))
+}
+
 func renderPkgindex(dest string, manpageByName map[string]*manpage.Meta) error {
 	var first *manpage.Meta
 	for _, m := range manpageByName {
@@ -56,3 +61,45 @@ func renderPkgindex(dest string, manpageByName map[string]*manpage.Meta) error {
 		})
 	})
 }
+
+func renderSrcPkgindex(dest string, src string, manpageByName map[string]*manpage.Meta) error {
+	var first *manpage.Meta
+	for _, m := range manpageByName {
+		first = m
+		break
+	}
+
+	mans := make([]string, 0, len(manpageByName))
+	for n := range manpageByName {
+		mans = append(mans, n)
+	}
+	sort.Strings(mans)
+
+	return write.Atomically(dest, true, func(w io.Writer) error {
+		return srcpkgindexTmpl.Execute(w, struct {
+			Title          string
+			DebimanVersion string
+			Breadcrumbs    breadcrumbs
+			FooterExtra    string
+			First          *manpage.Meta
+			Meta           *manpage.Meta
+			ManpageByName  map[string]*manpage.Meta
+			Mans           []string
+			HrefLangs      []*manpage.Meta
+			Src            string
+		}{
+			Title:          fmt.Sprintf("Manpages of src:%s in Debian %s", src, first.Package.Suite),
+			DebimanVersion: debimanVersion,
+			Breadcrumbs: breadcrumbs{
+				{fmt.Sprintf("/contents-%s.html", first.Package.Suite), first.Package.Suite},
+				{fmt.Sprintf("/%s/%s/index.html", first.Package.Suite, first.Package.Binarypkg), first.Package.Binarypkg},
+				{"", "Contents"},
+			},
+			First:         first,
+			Meta:          first,
+			ManpageByName: manpageByName,
+			Mans:          mans,
+			Src:           src,
+		})
+	})
+}