diff --git a/Makefile b/Makefile index c12f2bb48..d55c11d61 100644 --- a/Makefile +++ b/Makefile @@ -9,15 +9,16 @@ SELINUX_OPT := $(shell [ $(DOCKER_SELINUX_LABEL) -eq 1 ] && echo "$(COL .PHONY: html init-highlights html-nohighlight sparse assets webdev-build \ bigpage test xtest ctest help run clean-html clean-images \ clean-search clean test-links push \ + gen-pod6-source clean-build \ docker-image docker-htmlify docker-test docker-xtest docker-ctest docker-testall docker-run -html: bigpage htmlify +html: gen-pod6-source bigpage htmlify -htmlify: init-highlights assets gen-pod6-source +htmlify: init-highlights assets perl6 htmlify.p6 gen-pod6-source: - perl6 manage-page-order.p6 update + perl6 util/manage-page-order.p6 update init-highlights: ATOMDIR="./highlights/atom-language-perl6"; \ @@ -38,7 +39,7 @@ webdev-build: perl6 htmlify.p6 --no-highlight --sparse=200 bigpage: - pod2onepage --html -v --source-path=./doc --exclude=404.pod6 > html/perl6.html + pod2onepage --html -v --source-path=./build --exclude=404.pod6 > html/perl6.html # Common tests that are run by travis with every commit test: @@ -75,6 +76,8 @@ help: @echo "docker-testall: run all tests (in container)" @echo "docker-run: run the development webserver (in container)" +start: run + run: @echo "Starting local server…" ./app-start @@ -121,6 +124,9 @@ clean-images: clean-search: rm -f html/js/search.js +clean-build: + find build -name "*.pod6" -exec rm -f {} \; + clean: clean-html clean-images clean-search distclean: clean diff --git a/doc/Language/0-html-source/README b/doc/Language/0-html-source/README deleted file mode 100644 index ed975183a..000000000 --- a/doc/Language/0-html-source/README +++ /dev/null @@ -1 +0,0 @@ -This is a place holder to preserve this dir in the doc repo. diff --git a/htmlify.p6 b/htmlify.p6 index 0ab05a760..8e8732735 100755 --- a/htmlify.p6 +++ b/htmlify.p6 @@ -195,9 +195,9 @@ sub MAIN( my %h = $type-graph.sorted.kv.flat.reverse; write-type-graph-images(:force($typegraph), :$parallel); - process-pod-dir 'Programs', :$sparse, :$parallel; - process-pod-dir 'Language', :$sparse, :$parallel; - process-pod-dir 'Type', :sorted-by{ %h{.key} // -1 }, :$sparse, :$parallel; + process-pod-dir :topdir('build'), :dir('Programs'), :$sparse, :$parallel; + process-pod-dir :topdir('build'), :dir('Language'), :$sparse, :$parallel; + process-pod-dir :topdir('build'), :dir('Type'), :sorted-by{ %h{.key} // -1 }, :$sparse, :$parallel; highlight-code-blocks unless $no-highlight; @@ -236,8 +236,9 @@ sub MAIN( spurt('links.txt', $url-log.URLS.sort.unique.join("\n")); } -sub process-pod-dir($dir, :&sorted-by = &[cmp], :$sparse, :$parallel) { - say "Reading doc/$dir ..."; +sub process-pod-dir(:$topdir, :$dir, :&sorted-by = &[cmp], :$sparse, :$parallel) { + #say "Reading doc/$dir ..."; + say "Reading $topdir/$dir ..."; # What does this array look like? # @@ -251,35 +252,27 @@ sub process-pod-dir($dir, :&sorted-by = &[cmp], :$sparse, :$parallel) { # value: filename relative to the "doc/$dir" directory my @pod-sources; - if $dir eq 'Language' { - # uses a special sort order by :page-order as a %config hash entry - # TODO treat the Programs directory the same way - # use the target files auto-generated: - @pod-sources = get-pod6-page-order(:dir("$dir/0-html-source")); - } - else { - # default sort is by name {%hash{.key} => file basename w/o extension - @pod-sources = - recursive-dir("doc/$dir/") - .grep({.path ~~ / '.pod6' $/}) - .map({ - .path.subst("doc/$dir/", '') - .subst(rx{\.pod6$}, '') - .subst(:g, '/', '::') - => $_ + # default sort is by name {%hash{.key} => file basename w/o extension + @pod-sources = + recursive-dir("$topdir/$dir/") + .grep({.path ~~ / '.pod6' $/}) + .map({ + .path.subst("$topdir/$dir/", '') + .subst(rx{\.pod6$}, '') + .subst(:g, '/', '::') + => $_ }).sort(&sorted-by); - } =begin comment # PLEASE LEAVE THIS DEBUG CODE IN UNTIL WE'RE HAPPY # WITH LANGUAGE PAGE SORTING AND DISPLAY - if 0 && $dir eq 'Language' { + #if 1 && $dir eq 'Language' { #if 1 && $dir eq 'Programs' { say "\@pod-sources:"; for @pod-sources.kv -> $num, (:key($filename), :value($file)) { say "num: $num; key: |$filename|; value : |$file|"; } - #die "debug exit"; + die "debug exit"; } =end comment @@ -288,7 +281,7 @@ sub process-pod-dir($dir, :&sorted-by = &[cmp], :$sparse, :$parallel) { @pod-sources = @pod-sources[^(@pod-sources / $sparse).ceiling]; } - say "Processing $dir Pod files ..."; + say "Processing $topdir/$dir Pod files ..."; my $total = +@pod-sources; my $kind = $dir.lc; for @pod-sources.kv -> $num, (:key($filename), :value($file)) { diff --git a/lib/Pod/Htmlify.pm6 b/lib/Pod/Htmlify.pm6 index 36c3324f8..1775aaf44 100644 --- a/lib/Pod/Htmlify.pm6 +++ b/lib/Pod/Htmlify.pm6 @@ -127,56 +127,4 @@ sub get-page-order-value($fname) is export { return $sortid; } -#| Return an array of pod6 page file name data sorted by :page-order value -sub get-pod6-page-order(:$dir) is export { - # caller: process-pod-dir - # - # Return the .pod6 files found in $dir with :page-order values - # as an array of pairs of file basename (stripped of 'pod6') - # as key, and file names (relative to 'doc') as values. - # The array of such pairs is sorted in :page-order value - # alphabetical order. - # - # Note any .pod6 not having a :page-order entry is not - # included in the outout array. - # - # An exception is thrown if a duplicate :page-order key is found. - # - - my $d = "doc/$dir"; - die "FATAL: '$d' is NOT a known directory'" if !$d.IO.d; - - my %h; - for (dir $d) -> $f { - next if !$f.IO.f; - next if $f !~~ /'.pod6' $/; - - my $sortid = get-page-order-value $f; - next if !$sortid; - - # the sortid must be unique for each file in this directory - die "FATAL: Duplicate :page-order key '$sortid' in file '$f'" - if %h{$sortid}; - - my $key = $f.basename; - # remove the pod6 extension - $key ~~ s/'.pod6'//; - my $value = $f; - # pack it all up - %h{$sortid} = ($key, $value); - } - - # turn the hash into a sorted array - my @keys = %h.keys.sort; - my @arr; - # TODO below can be improved after all else is satisfactory - for @keys -> $k { - my @a = @(%h{$k}); - my $hk = @a.shift; - my $hv = @a.shift; - push @arr, ($hk => $hv); - } - return @arr; -} - # vim: expandtab shiftwidth=4 ft=perl6 diff --git a/manage-page-order.p6 b/util/manage-page-order.p6 similarity index 71% rename from manage-page-order.p6 rename to util/manage-page-order.p6 index 43068c816..48aa9e80b 100755 --- a/manage-page-order.p6 +++ b/util/manage-page-order.p6 @@ -1,6 +1,10 @@ #!/usr/bin/env perl6 -# pod6 directory of interest for auto-gen handling: +# NOTE: This file is called by the Makefile (or Sakefile) from the +# top level of the doc repo and all files are referenced +# relative to it. + +# pod6 source directory of interest for auto-gen handling: my $lang-dir = 'doc/Language'; # pod6 source d#irectory my $fromdir = $lang-dir; @@ -8,9 +12,10 @@ my $cfil = '00-POD6-CONTROL'; # base filename my $cloc = "$lang-dir/$cfil"; # location for placing the generated files -# htmlify.p6 must use this location for building: -my $tgt-dir = '0-html-source'; -my $todir = "$lang-dir/$tgt-dir"; +# htmlify.p6 AND pod2bigpage (see Makefile) must use this location for building: +my $build-dir = 'build'; +my $tgt-dir = 'Language'; +my $todir = "$build-dir/$tgt-dir"; constant $TITLE = '=TITLE'; constant $SUBTITLE = '=SUBTITLE'; @@ -48,20 +53,89 @@ given $mode { } } -sub do-update() { +sub do-update () { say "DEBUG: in sub {&?ROUTINE.name}" if $debug eq 'r'; + + # FIRST + # write the auto-generated pod6 files used for the Language page + write-Language-files(); + + # THEN + # copy all under the doc dir to the build dir + # NOTE this assumes the sort order is alpha by file name + # NOTE we exclude the Language dir + copy-dir-tree(:fromdir('doc'), :todir('build'), :exclude('Language')); +} + +=begin comment +sub copy-other-dirs-and-files(*@doc-subdirs) { + # The arguments are subdir names under 'doc' and + # the complete subdir trees are to be copied as + # subdir trees under 'build'. + + for @doc-subdirs -> $sdir { + say "DEBUG: working subdir tree '$sdir'" if $debug; + + # recursively copy tree to another tree + my $fromdir = "doc/$sdir"; # complete path + my $todir = "build/$sdir"; + copy-dir-tree(:$fromdir, :$todir, :$exclude); + } +} +=end comment + +sub copy-dir-tree(:$fromdir, :$todir, :$exclude) { + # recursively copy tree to another tree + return if !$fromdir.IO.d; + + say "DEBUG: \$fromdir: '$fromdir', \$exclude: '$exclude'" if $debug; + # skip some dirs, e.g., Language + return if $fromdir ~~ /$exclude/; + + # create the target dir if need be + mkdir $todir if !$todir.IO.d; + # need a file to save the dir for git + my $tf = "$todir/000-README"; + if !$tf.IO.f { + spurt $tf, 'This file is required to save the directory under git.'; + } + + # copy all files first, then recurse into directories we find + say "DEBUG: copy trees from '$fromdir' to '$todir'" if $debug; + my @sdirs; + for (dir $fromdir) -> $f { + my $fb = $f.IO.basename; + if $f.IO.d { + say " DEBUG: pushing dir '$fb' for later handling'" if $debug; + push @sdirs, $fb; + next; + } + # must be a file + next if $f !~~ /'.pod6'$/; + say " DEBUG: copying file '$f' to dir '$todir'" if $debug; + # for current rakudo we must copy to a file, not its dir + copy $f, "$todir/$fb"; + } + for @sdirs -> $d { + say " DEBUG: handling pushed dir '$d'..." if $debug; + say " copying from dir '$fromdir/$d' to '$todir/$d..." if $debug; + copy-dir-tree :fromdir("$fromdir/$d"), :todir("$todir/$d"), :$exclude; + } +} + +sub write-Language-files() { # the $todir must exist or be created mkdir $todir if !$todir.IO.d; - my %data; + my %data; # note data do NOT have page order, that is determined from the control file get-pod6-src-data(:$fromdir, :%data); # read the control file and generate the pod6 files accordingly die "FATAL: Control file '$cloc' not found." if !$cloc.IO.f; # need some indices - my @a = 'A' .. 'Z'; # for the group pages - my @n = '001' .. '999'; + my @a = 'A' .. 'Z'; # for the group pages + my @n = '001' .. '999'; # page order for all files for $cloc.IO.lines -> $line is copy { $line = strip-comment $line; @@ -76,7 +150,6 @@ sub do-update() { # straight numerical order for all pages, including group separator pages my $page-order = shift @n; - #my $title = "{$group-char}."; my $title = ~$0.words.join(' '); # the group page is written to the todir write-group-page $group-char, $title, $page-order; @@ -90,28 +163,34 @@ sub do-update() { next if !$idx.defined; my $n = $FILE.chars; - my $fname = substr $line, $idx+$n; - $fname .= trim; - $fname ~= '.pod6'; + my $key-fname = substr $line, $idx+$n; + $key-fname .= trim; + my $fname = $key-fname ~ '.pod6'; # source filename my $from = "$fromdir/$fname"; + say "DEBUG: from '$from'" if $debug; + + # straight numerical order for all pages, including group separator pages + my $page-order = shift @n; + # target filename - my $to = "$todir/$fname"; + # NOTE the target filename must have the page-order as prefix to its filename + my $to = "$todir/{$page-order}-{$fname}"; say "DEBUG: to '$to'" if $debug; - my $fh = open $to, :w; for $from.IO.lines -> $line is copy { if $line ~~ /^ \h* '=begin' \h* pod / { - my $page-order = shift @n; - # check any existing :page-order entry if $line ~~ / ':page-order<' (.*) '>' / { EXIT(":page-order entry not allowed in source files ('$from')"); } else { - $fh.say: "$line :page-order<{$page-order}>"; + #$fh.say: "$line :page-order<{$page-order}>"; + $fh.say: "# THIS FILE IS AUTO-GENERATED--ALL EDITS WILL BE LOST"; + $fh.say: $line; + #$fh.say: "=comment THIS FILE IS AUTO-GENERATED--ALL EDITS WILL BE LOST"; } next; } @@ -126,7 +205,8 @@ sub do-update() { } sub do-control() { - say "DEBUG: in sub {&?ROUTINE.name}" if $debug; + # write or refresh the Language control file + # TODO make more general for multiple dirs my %data; get-pod6-src-data(:$fromdir, :%data); @@ -143,7 +223,7 @@ sub do-control() { } sub refresh-control-file(%data) { - say "DEBUG: in sub {&?ROUTINE.name}" if $debug; + # TODO finish this sub # read the current file, update the title } @@ -193,7 +273,6 @@ sub write-orig-control-file(%data) { # desired order. # # DO NOT REMOVE OR MODIFY THE FILE NAME AT THE END OF EACH LINE - HERE for %p.keys.sort ->$o { @@ -216,21 +295,20 @@ sub write-orig-control-file(%data) { } sub strip-pod6-filename($fname is copy) { - say "DEBUG: in sub {&?ROUTINE.name}" if $debug; $fname .= basename; $fname ~~ s/'.pod6'$//; return $fname; } sub get-pod6-src-data(:fromdir($fdir), :%data) { - say "DEBUG: in sub {&?ROUTINE.name}" if $debug eq 'r'; # caller: do-control # data are used to generate a new control file + # AND the auto-generated target files die "FATAL: no such directory '$fdir'" if !$fdir.IO.d; # hash key: stripped-filename - # subkey: page-order # subkey: title + # subkey: subtitle [NOT used by the control file] my $n = 0; for (dir $fdir) -> $f { @@ -243,10 +321,10 @@ sub get-pod6-src-data(:fromdir($fdir), :%data) { # we use the name for indexing and reference my $fname = strip-pod6-filename $f; - # collect the title, ensure first line is =begin pod, + # collect the title, ensure first data line is =begin pod, # ensure last line is =end pod - my ($title, $order, $subtitle) = ('', '', ''); + my ($title, $subtitle) = ('', ''); my $begin = 0; my $first = 1; my $last = 0; @@ -268,12 +346,11 @@ sub get-pod6-src-data(:fromdir($fdir), :%data) { next if $line !~~ /\S/; if $line ~~ /^ \h* '=begin' \h+ pod/ { - # this should be the first data line + # this should be the first line die "FATAL: =begin pod is NOT the first data line" if !$first; $first = 0; $begin = 1; - $order = extract-page-order :begin-pod-line($line); } elsif $line ~~ /^ \h* $TITLE / { $title = extract-abbrev-block-line $line, :type($TITLE); @@ -306,15 +383,14 @@ sub get-pod6-src-data(:fromdir($fdir), :%data) { # data collected, put in hash # hash key: stripped-filename - # subkey: page-order # subkey: title + # subkey: subtitle %data{$fname} = $title; - %data{$fname}<page-order> = $order; + %data{$fname}<subtitle> = $subtitle; } } sub strip-comment($line is copy) { - say "DEBUG: in sub {&?ROUTINE.name}" if $debug; my $idx = rindex $line, '#'; if $idx.defined { $line = substr $line, 0, $idx; @@ -332,7 +408,6 @@ sub extract-page-order(:begin-pod-line($line)) { } sub extract-abbrev-block-line($line is copy, :$type) { - say "DEBUG: in sub {&?ROUTINE.name}" if $debug; $line = strip-comment $line; die "FATAL: Empty '$type' abbreviated block first line" if $line !~~ /\S/; @@ -384,7 +459,6 @@ sub help { } sub is-done(*@args) { - say "DEBUG: in sub {&?ROUTINE.name}" if $debug eq 'r'; # return False unless all @args elements are true for @args -> $v { return False if !$v; @@ -393,13 +467,13 @@ sub is-done(*@args) { } sub write-group-page($group-char, $title, $page-order) { - my $fname = $todir ~ "/00-Group{$group-char}.pod6"; + my $fname = $todir ~ "/{$page-order}-Group{$group-char}.pod6"; my $fh = open $fname, :w; - # x2588 big solid block <== using this char to make the separator - # x2501 solid thick horizontal line + # note no SUBTITLE $fh.print: qq:to/HERE/; - =begin pod :page-order<{$page-order}> + # THIS FILE IS AUTO-GENERATED--ALL EDITS WILL BE LOST + =begin pod =TITLE $title =SUBTITLE ~ =end pod