Permalink
Browse files

Implement Page bundling and image handling

This commit is not the smallest in Hugo's history.

Some hightlights include:

* Page bundles (for complete articles, keeping images and content together etc.).
* Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`.
* Processed images are cached inside `resources/_gen/images` (default) in your project.
* Symbolic links (both files and dirs) are now allowed anywhere inside /content
* A new table based build summary
* The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below).

A site building  benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory:

```bash
▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render"

benchmark                                                                                                         old ns/op     new ns/op     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      101785785     78067944      -23.30%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     185481057     149159919     -19.58%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      103149918     85679409      -16.94%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     203515478     169208775     -16.86%

benchmark                                                                                                         old allocs     new allocs     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      532464         391539         -26.47%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1056549        772702         -26.87%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      555974         406630         -26.86%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1086545        789922         -27.30%

benchmark                                                                                                         old bytes     new bytes     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      53243246      43598155      -18.12%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     105811617     86087116      -18.64%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      54558852      44545097      -18.35%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     106903858     86978413      -18.64%
```

Fixes #3651
Closes #3158
Fixes #1014
Closes #2021
Fixes #1240
Updates #3757
  • Loading branch information...
bep committed Jul 24, 2017
1 parent 02f2735 commit 3cdf19e9b7e46c57a9bb43ff02199177feb55768
Showing with 5,747 additions and 3,243 deletions.
  1. +102 −17 Gopkg.lock
  2. +6 −2 Gopkg.toml
  3. +2 −7 commands/benchmark.go
  4. +21 −13 commands/commandeer.go
  5. +80 −55 commands/convert.go
  6. +333 −129 commands/hugo.go
  7. +0 −1 commands/import_jekyll.go
  8. +15 −27 commands/list.go
  9. +1 −1 commands/list_config.go
  10. +18 −17 commands/new.go
  11. +70 −81 commands/server.go
  12. +11 −5 commands/static_syncer.go
  13. +3 −1 commands/undraft.go
  14. +2 −2 create/content_template_handler.go
  15. +10 −0 deps/deps.go
  16. +7 −0 helpers/content.go
  17. +22 −0 helpers/content_test.go
  18. +2 −2 helpers/general.go
  19. +10 −12 helpers/language.go
  20. +9 −37 helpers/path.go
  21. +0 −34 helpers/path_test.go
  22. +36 −0 helpers/pathspec.go
  23. +116 −0 helpers/processing_stats.go
  24. +1 −1 hugolib/alias.go
  25. +3 −1 hugolib/alias_test.go
  26. +6 −6 hugolib/config.go
  27. +15 −16 hugolib/datafiles_test.go
  28. +109 −0 hugolib/fileInfo.go
  29. +61 −0 hugolib/fileInfo_test.go
  30. +0 −65 hugolib/handler_base.go
  31. +0 −59 hugolib/handler_file.go
  32. +0 −128 hugolib/handler_meta.go
  33. +0 −157 hugolib/handler_page.go
  34. +0 −77 hugolib/handler_test.go
  35. +208 −129 hugolib/hugo_sites.go
  36. +7 −27 hugolib/hugo_sites_build.go
  37. +49 −51 hugolib/hugo_sites_build_test.go
  38. +2 −3 hugolib/hugo_sites_multihost_test.go
  39. BIN hugolib/hugolib.debug
  40. +0 −654 hugolib/menu_old_test.go
  41. +125 −31 hugolib/page.go
  42. +188 −0 hugolib/page_bundler.go
  43. +683 −0 hugolib/page_bundler_capture.go
  44. +255 −0 hugolib/page_bundler_capture_test.go
  45. +346 −0 hugolib/page_bundler_handlers.go
  46. +379 −0 hugolib/page_bundler_test.go
  47. +13 −0 hugolib/page_collections.go
  48. +1 −1 hugolib/page_collections_test.go
  49. +70 −4 hugolib/page_output.go
  50. +34 −7 hugolib/page_paths.go
  51. +0 −1 hugolib/page_paths_test.go
  52. +23 −0 hugolib/page_resource.go
  53. +2 −2 hugolib/page_test.go
  54. +4 −2 hugolib/pagination.go
  55. +8 −3 hugolib/permalinks.go
  56. +84 −0 hugolib/prune_resources.go
  57. +1 −1 hugolib/rss_test.go
  58. +2 −3 hugolib/shortcode_test.go
  59. +280 −449 hugolib/site.go
  60. +6 −4 hugolib/siteJSONEncode_test.go
  61. +26 −5 hugolib/site_render.go
  62. +3 −2 hugolib/site_sections_test.go
  63. +101 −0 hugolib/site_stats_test.go
  64. +93 −110 hugolib/site_test.go
  65. +3 −4 hugolib/site_url_test.go
  66. +15 −3 hugolib/testhelpers_test.go
  67. +17 −3 i18n/translationProvider.go
  68. +551 −0 resource/image.go
  69. +112 −0 resource/image_cache.go
  70. +134 −0 resource/image_test.go
  71. +275 −0 resource/resource.go
  72. +108 −0 resource/resource_test.go
  73. BIN resource/testdata/sunset.jpg
  74. +78 −0 resource/testhelpers_test.go
  75. +6 −5 source/content_directory_test.go
  76. +7 −4 source/dirs.go
  77. +0 −172 source/file.go
  78. +213 −0 source/fileInfo.go
  79. +6 −9 source/{inmemory.go → fileInfo_test.go}
  80. +0 −62 source/file_test.go
  81. +28 −80 source/filesystem.go
  82. +13 −53 source/filesystem_test.go
  83. +0 −170 source/lazy_file_reader.go
  84. +0 −236 source/lazy_file_reader_test.go
  85. +117 −0 source/sourceSpec.go
View

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.
View
@@ -21,8 +21,8 @@
revision = "v1.1.0"
[[constraint]]
branch = "master"
name = "github.com/dchest/cssmin"
name = "github.com/disintegration/imaging"
revision = "v1.2.4"
[[constraint]]
name = "github.com/magefile/mage"
@@ -116,6 +116,10 @@
name = "github.com/stretchr/testify"
version = "1.1.4"
[[constraint]]
branch = "master"
name = "github.com/olekukonko/tablewriter"
[[constraint]]
name = "github.com/yosssi/ace"
version = "0.0.5"
View
@@ -48,12 +48,7 @@ func init() {
}
func benchmark(cmd *cobra.Command, args []string) error {
cfg, err := InitializeConfig(benchmarkCmd)
if err != nil {
return err
}
c, err := newCommandeer(cfg)
c, err := InitializeConfig(false, nil, benchmarkCmd)
if err != nil {
return err
}
@@ -84,7 +79,7 @@ func benchmark(cmd *cobra.Command, args []string) error {
t := time.Now()
for i := 0; i < benchmarkTimes; i++ {
if err = c.resetAndBuildSites(false); err != nil {
if err = c.resetAndBuildSites(); err != nil {
return err
}
}
View
@@ -18,14 +18,18 @@ import (
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
src "github.com/gohugoio/hugo/source"
)
type commandeer struct {
*deps.DepsCfg
pathSpec *helpers.PathSpec
visitedURLs *types.EvictingStringQueue
staticDirsConfig []*src.Dirs
serverPorts []int
languages helpers.Languages
configured bool
}
@@ -44,29 +48,33 @@ func (c *commandeer) PathSpec() *helpers.PathSpec {
return c.pathSpec
}
func (c *commandeer) languages() helpers.Languages {
return c.Cfg.Get("languagesSorted").(helpers.Languages)
}
func (c *commandeer) initFs(fs *hugofs.Fs) error {
c.DepsCfg.Fs = fs
ps, err := helpers.NewPathSpec(fs, c.Cfg)
if err != nil {
return err
}
c.pathSpec = ps
dirsConfig, err := c.createStaticDirsConfig()
if err != nil {
return err
}
c.staticDirsConfig = dirsConfig
return nil
}
func newCommandeer(cfg *deps.DepsCfg) (*commandeer, error) {
l := cfg.Language
if l == nil {
l = helpers.NewDefaultLanguage(cfg.Cfg)
}
ps, err := helpers.NewPathSpec(cfg.Fs, l)
if err != nil {
return nil, err
func newCommandeer(cfg *deps.DepsCfg, running bool) (*commandeer, error) {
cfg.Running = running
var languages helpers.Languages
if l, ok := cfg.Cfg.Get("languagesSorted").(helpers.Languages); ok {
languages = l
}
return &commandeer{DepsCfg: cfg, pathSpec: ps, visitedURLs: types.NewEvictingStringQueue(10)}, nil
c := &commandeer{DepsCfg: cfg, languages: languages, visitedURLs: types.NewEvictingStringQueue(10)}
return c, nil
}
Oops, something went wrong.

0 comments on commit 3cdf19e

Please sign in to comment.