Write per-source-package index pages for tracker.debian.org

fixes #94
Debian · Jan 13, 2018 · af8eedcb2c884e6c201b6bd651d4f37bfda4ede8 · af8eedc
1 parent 01e1c55
commit af8eedcb2c884e6c201b6bd651d4f37bfda4ede8
Unified Split

Showing with 156 additions and 21 deletions.

+23 −0 assets/srcpkgindex.tmpl

+1 −1 bundle.go

+1 −0 cmd/debiman/main.go

+62 −0 cmd/debiman/render.go

+47 −0 cmd/debiman/renderpkgindex.go

+22 −20 internal/bundled/GENERATED_bundled.go
diff --git a/assets/srcpkgindex.tmpl b/assets/srcpkgindex.tmpl
@@ -0,0 +1,23 @@
+  
+{{ template "header" . }}
+  
+
+  
+<div class="maincontents">
+  
+
+  
+<h1>Manpages of <a href="https://tracker.debian.org/pkg/{{ .Src }}">src:{{ .Src }}</a> in Debian {{ .First.Package.Suite }}</h1>
+  
+
+  
+<ul>
+  
+{{ range $idx, $fn := .Mans }}
+  
+  {{ with $m := index $.ManpageByName $fn }}
+  
+<li>
+  
+  <a href="{{ BaseURLPath }}/{{ $m.ServingPath }}.html">{{ $m.Name }}({{ $m.Section }})
+  
+    {{ if ne $m.Language "en" }}
+  
+      (<span title="{{ EnglishLang $m.LanguageTag }} ({{ $m.Language }})">{{ DisplayLang $m.LanguageTag }}</span>)
+  
+    {{ end }}
+  
+  </a>
+  
+</li>
+  
+  {{ end }}
+  
+{{ end }}
+  
+</ul>
+  
+
+  
+</div>
+  
+
+  
+{{ template "footer" . }}
diff --git a/bundle.go b/bundle.go
@@ -1,3 +1,3 @@
 package bundle
-  
-//go:generate sh -c "go run goembed.go -package bundled -var assets assets/header.tmpl assets/footer.tmpl assets/style.css assets/manpage.tmpl assets/manpageerror.tmpl assets/manpagefooterextra.tmpl assets/contents.tmpl assets/pkgindex.tmpl assets/index.tmpl assets/faq.tmpl assets/notfound.tmpl assets/Inconsolata.woff assets/Inconsolata.woff2 assets/opensearch.xml assets/Roboto-Bold.woff assets/Roboto-Bold.woff2 assets/Roboto-Regular.woff assets/Roboto-Regular.woff2 > internal/bundled/GENERATED_bundled.go"
+  
+//go:generate sh -c "go run goembed.go -package bundled -var assets assets/header.tmpl assets/footer.tmpl assets/style.css assets/manpage.tmpl assets/manpageerror.tmpl assets/manpagefooterextra.tmpl assets/contents.tmpl assets/pkgindex.tmpl assets/srcpkgindex.tmpl assets/index.tmpl assets/faq.tmpl assets/notfound.tmpl assets/Inconsolata.woff assets/Inconsolata.woff2 assets/opensearch.xml assets/Roboto-Bold.woff assets/Roboto-Bold.woff2 assets/Roboto-Regular.woff assets/Roboto-Regular.woff2 > internal/bundled/GENERATED_bundled.go"
diff --git a/cmd/debiman/main.go b/cmd/debiman/main.go
@@ -160,6 +160,7 @@ func main() {
 		commonTmpls = commontmpl.MustParseCommonTmpls()
 		contentsTmpl = mustParseContentsTmpl()
 		pkgindexTmpl = mustParsePkgindexTmpl()
+  
+		srcpkgindexTmpl = mustParseSrcPkgindexTmpl()
 		indexTmpl = mustParseIndexTmpl()
 		faqTmpl = mustParseFaqTmpl()
 		aboutTmpl = mustParseAboutTmpl()
diff --git a/cmd/debiman/render.go b/cmd/debiman/render.go
@@ -441,7 +441,65 @@ func walkContents(ctx context.Context, renderChan chan<- renderJob, whitelist ma
 	})
 }
+  
+func writeSourceIndex(gv globalView, newestForSource map[string]time.Time) error {
+  
+	// Partition by suite for reduced memory usage and better locality of file
+  
+	// system access
+  
+	for suite := range gv.suites {
+  
+		binariesBySource := make(map[string][]string)
+  
+		for _, p := range gv.pkgs {
+  
+			if p.suite != suite {
+  
+				continue
+  
+			}
+  
+			binariesBySource[p.source] = append(binariesBySource[p.source], p.binarypkg)
+  
+		}
+  
+
+  
+		for src, binaries := range binariesBySource {
+  
+			srcDir := filepath.Join(*servingDir, suite, "src:"+src)
+  
+			// skip if current index file is more recent than newestForSource
+  
+			st, err := os.Stat(filepath.Join(srcDir, "index.html.gz"))
+  
+			if !*forceRerender && err == nil && st.ModTime().After(newestForSource[src]) {
+  
+				continue
+  
+			}
+  
+
+  
+			// Aggregate manpages of all binary packages for this source package
+  
+			manpages := make(map[string]*manpage.Meta)
+  
+			for _, binary := range binaries {
+  
+				m, err := listManpages(filepath.Join(*servingDir, suite, binary))
+  
+				if err != nil {
+  
+					if os.IsNotExist(err) {
+  
+						continue // The package might not contain any manpages.
+  
+					}
+  
+					return err
+  
+				}
+  
+				for k, v := range m {
+  
+					manpages[k] = v
+  
+				}
+  
+			}
+  
+			if len(manpages) == 0 {
+  
+				continue // The entire source package does not contain any manpages.
+  
+			}
+  
+
+  
+			if err := os.MkdirAll(srcDir, 0755); err != nil {
+  
+				return err
+  
+			}
+  
+			if err := renderSrcPkgindex(filepath.Join(srcDir, "index.html.gz"), src, manpages); err != nil {
+  
+				return err
+  
+			}
+  
+		}
+  
+	}
+  
+	return nil
+  
+}
+  
+
 func renderAll(gv globalView) error {
+  
+	log.Printf("Preparing inverted maps")
+  
+	sourceByBinary := make(map[string]string, len(gv.pkgs))
+  
+	newestForSource := make(map[string]time.Time)
+  
+	for _, p := range gv.pkgs {
+  
+		sourceByBinary[p.suite+"/"+p.binarypkg] = p.source
+  
+		newestForSource[p.source] = time.Time{}
+  
+	}
+  
+	log.Printf("%d sourceByBinary entries, %d newestForSource entries", len(sourceByBinary), len(newestForSource))
+  
+
 	eg, ctx := errgroup.WithContext(context.Background())
 	renderChan := make(chan renderJob)
 	for i := 0; i < *renderConcurrency; i++ {
@@ -496,6 +554,10 @@ func renderAll(gv globalView) error {
 		return err
 	}
+  
+	if err := writeSourceIndex(gv, newestForSource); err != nil {
+  
+		return fmt.Errorf("writing source index: %v", err)
+  
+	}
+  
+
 	suitedirs, err := ioutil.ReadDir(*servingDir)
 	if err != nil {
 		return err
diff --git a/cmd/debiman/renderpkgindex.go b/cmd/debiman/renderpkgindex.go
@@ -12,11 +12,16 @@ import (
 )
 var pkgindexTmpl = mustParsePkgindexTmpl()
+  
+var srcpkgindexTmpl = mustParseSrcPkgindexTmpl()
 func mustParsePkgindexTmpl() *template.Template {
 	return template.Must(template.Must(commonTmpls.Clone()).New("pkgindex").Parse(bundled.Asset("pkgindex.tmpl")))
 }
+  
+func mustParseSrcPkgindexTmpl() *template.Template {
+  
+	return template.Must(template.Must(commonTmpls.Clone()).New("srcpkgindex").Parse(bundled.Asset("srcpkgindex.tmpl")))
+  
+}
+  
+
 func renderPkgindex(dest string, manpageByName map[string]*manpage.Meta) error {
 	var first *manpage.Meta
 	for _, m := range manpageByName {
@@ -56,3 +61,45 @@ func renderPkgindex(dest string, manpageByName map[string]*manpage.Meta) error {
 		})
 	})
 }
+  
+
+  
+func renderSrcPkgindex(dest string, src string, manpageByName map[string]*manpage.Meta) error {
+  
+	var first *manpage.Meta
+  
+	for _, m := range manpageByName {
+  
+		first = m
+  
+		break
+  
+	}
+  
+
+  
+	mans := make([]string, 0, len(manpageByName))
+  
+	for n := range manpageByName {
+  
+		mans = append(mans, n)
+  
+	}
+  
+	sort.Strings(mans)
+  
+
+  
+	return write.Atomically(dest, true, func(w io.Writer) error {
+  
+		return srcpkgindexTmpl.Execute(w, struct {
+  
+			Title          string
+  
+			DebimanVersion string
+  
+			Breadcrumbs    breadcrumbs
+  
+			FooterExtra    string
+  
+			First          *manpage.Meta
+  
+			Meta           *manpage.Meta
+  
+			ManpageByName  map[string]*manpage.Meta
+  
+			Mans           []string
+  
+			HrefLangs      []*manpage.Meta
+  
+			Src            string
+  
+		}{
+  
+			Title:          fmt.Sprintf("Manpages of src:%s in Debian %s", src, first.Package.Suite),
+  
+			DebimanVersion: debimanVersion,
+  
+			Breadcrumbs: breadcrumbs{
+  
+				{fmt.Sprintf("/contents-%s.html", first.Package.Suite), first.Package.Suite},
+  
+				{fmt.Sprintf("/%s/%s/index.html", first.Package.Suite, first.Package.Binarypkg), first.Package.Binarypkg},
+  
+				{"", "Contents"},
+  
+			},
+  
+			First:         first,
+  
+			Meta:          first,
+  
+			ManpageByName: manpageByName,
+  
+			Mans:          mans,
+  
+			Src:           src,
+  
+		})
+  
+	})
+  
+}