docs/cmdline: change to .md for cmdline docs

 - switch all invidual files documenting command line options into .md,
   as the documentation is now markdown-looking.

 - made the parser treat 4-space indents as quotes

 - switch to building the curl.1 manpage using the "mainpage.idx" file,
   which lists the files to include to generate it, instead of using the
   previous page-footer/headers. Also, those files are now also .md
   ones, using the same format. I gave them underscore prefixes to make
   them sort separately:
   _NAME.md, _SYNOPSIS.md, _DESCRIPTION.md, _URL.md, _GLOBBING.md,
   _VARIABLES.md, _OUTPUT.md, _PROTOCOLS.md, _PROGRESS.md, _VERSION.md,
   _OPTIONS.md, _FILES.md, _ENVIRONMENT.md, _PROXYPREFIX.md,
   _EXITCODES.md, _BUGS.md, _AUTHORS.md, _WWW.md, _SEEALSO.md

 - updated test cases accordingly

Closes #12751

.github/scripts/cleancmd.pl

@@ -0,0 +1,52 @@
+#!/usr/bin/perl
+# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# SPDX-License-Identifier: curl
+#
+# Input: a cmdline docs markdown, it gets modfied *in place*
+#
+# The main purpose is to strip off the leading meta-data part, but also to
+# clean up whatever else the spell checker might have a problem with that we
+# still deem is fine.
+
+my $header = 1;
+while(1) {
+    # set this if the markdown has no meta-data header to skip
+    if($ARGV[0] eq "--no-header") {
+        shift @ARGV;
+        $header = 0;
+    }
+    else {
+        last;
+    }
+}
+
+my $f = $ARGV[0];
+
+open(F, "<$f") or die;
+
+my $ignore = $header;
+my $sepcount = 0;
+my @out;
+while(<F>) {
+    if(/^---/ && $header) {
+        if(++$sepcount == 2) {
+            $ignore = 0;
+        }
+        next;
+    }
+    next if($ignore);
+
+    # strip out all long command line options
+    $_ =~ s/--[a-z0-9-]+//g;
+
+    # strip out https URLs, we don't want them spellchecked
+    $_ =~ s!https://[a-z0-9\#_/.-]+!!gi;
+
+    push @out, $_;
+}
+close(F);
+
+open(O, ">$f") or die;
+print O @out;
+close(O);

.github/scripts/spellcheck.words

@@ -50,6 +50,7 @@ backend
 backends
 backoff
 backticks
+balancers
 Baratov
 basename
 bashrc
@@ -445,6 +446,7 @@ Makefile
 makefiles
 malloc
 mallocs
+manpage
 maprintf
 Marek
 Mavrogiannopoulos

.github/workflows/linkcheck.yml

@@ -31,6 +31,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        name: checkout
+
+      - name: trim the cmdline docs markdown files
+        run: find docs/cmdline-opts -name "*.md" ! -name "_*" ! -name MANPAGE.md | xargs -n1 ./.github/scripts/cleancmd.pl
+
       - uses: gaurav-nelson/github-action-markdown-link-check@v1
         with:
           use-quiet-mode: 'yes'

.github/workflows/spellcheck.yml

@@ -57,6 +57,12 @@ jobs:
           perl -pi -e 's/\-\-[\a-z0-9-]*//ig' docs/curl.md
           perl -pi -e 's!https://[a-z0-9%/.-]*!!ig' docs/curl.md
 
+      - name: trim the cmdline docs markdown files
+        run: find docs/cmdline-opts -name "*.md" ! -name "_*" ! -name MANPAGE.md | xargs -n1 ./.github/scripts/cleancmd.pl
+
+      - name: trim the cmdline docs markdown _*.md files
+        run: find docs/cmdline-opts -name "_*.md" | xargs -n1 ./.github/scripts/cleancmd.pl --no-header
+
       - name: setup the custom wordlist
         run: grep -v '^#' .github/scripts/spellcheck.words >  wordlist.txt
 

docs/cmdline-opts/MANPAGE.md

@@ -9,24 +9,38 @@
 This is the curl man page generator. It generates a single nroff man page
 output from the set of sources files in this directory.
 
-There is one source file for each supported command line option. The output
-gets `page-header` prepended and `page-footer` appended. The format is
-described below.
+The `mainpage.idx` file lists all files that are rendered in that order to
+produce the output. The magic `%options` keyword inserts all command line
+options documented.
+
+The `%options` documentation is created with one source file for each
+supported command line option.
+
+The documentation file format is described below. It is meant to look similar
+to markdown which is why it uses `.md` file extensions.
 
 ## Option files
 
 Each command line option is described in a file named `<long name>.d`, where
 option name is written without any prefixing dashes. Like the file name for
-the -v, --verbose option is named `verbose.d`.
+the `-v, --verbose` option is named `verbose.d`.
+
+Each file has a set of meta-data in the top of the file, followed by a body of
+text.
 
-Each file has a set of meta-data and a body of text.
+The documentation files that do not document options have no meta-data part.
+
+A line that starts with `<!--` is a comment. It should also end with `-->`.
 
 ### Meta-data
 
+    --- (start of meta-data)
     Added: (version number in which this was added)
     Arg: (the argument the option takes)
     c: (copyright line)
-    Example: (example command line, without "curl" and can use `$URL`)
+    Example:
+      - (an example command line, without "curl" and can use `$URL`)
+      - (another example)
     Experimental: yes (if so)
     Help: (short text for the --help output for this option)
     Long: (long form name, without dashes)
@@ -36,7 +50,9 @@ Each file has a set of meta-data and a body of text.
     Protocols: (space separated list for which protocols this option works)
     Requires: (space separated list of features this requires, no dashes)
     Scope: global (if the option is global)
-    See-also: (space separated list of related options, no dashes)
+    See-also:
+      - (a related option, no dashes)
+      - (another related option, no dashes)
     Short: (single letter, without dash)
     SPDX-License-Identifier: curl
     Tags: (space separated list)
@@ -54,7 +70,7 @@ Text written within `*asterisks*` is shown using italics. Text within two
 Text that is prefixed with a space is treated like an "example" and gets
 output in monospace.
 
-Within the body, describe a lite of items like this:
+Within the body, describe a list of items like this:
 
     ## item 1
     description
@@ -67,12 +83,20 @@ explicitly with an empty "header":
 
     ##
 
-## Header and footer
+### Headers
+
+The `#` header can be used by non-option files and it produces produces a
+`.SH` output.
+
+If the `#` header is used for a command line option file, that header is
+simply ignored in the generated output. It can still serve a purpose in the
+source file as it helps the user identify what option the file is for.
 
-`page-header` is the file that is output before the generated options output
-for the master man page.
+### Variables
 
-`page-footer` is appended after all the individual options.
+There are three different "variables" that can be used when creating the
+output. They need to be written within backticks in the source file (to escape
+getting spellchecked by CI jobs): `%DATE`, `%VERSION` and `%GLOBALS`.
 
 ## Generate
 

docs/cmdline-opts/Makefile.am

@@ -28,7 +28,7 @@ MANPAGE = $(top_builddir)/docs/curl.1
 
 include Makefile.inc
 
-EXTRA_DIST = $(DPAGES) MANPAGE.md gen.pl $(OTHERPAGES) CMakeLists.txt
+EXTRA_DIST = $(DPAGES) MANPAGE.md gen.pl $(SUPPORT) CMakeLists.txt mainpage.idx
 
 GEN = $(GN_$(V))
 GN_0 = @echo "  GENERATE" $@;
@@ -37,5 +37,5 @@ GN_ = $(GN_0)
 
 all: $(MANPAGE)
 
-$(MANPAGE): $(DPAGES) $(OTHERPAGES) Makefile.inc gen.pl
+$(MANPAGE): $(DPAGES) $(SUPPORT) mainpage.idx Makefile.inc gen.pl
 	$(GEN)(rm -f $(MANPAGE) && cd $(srcdir) && @PERL@ ./gen.pl mainpage $(DPAGES) > $(builddir)/manpage.tmp && mv $(builddir)/manpage.tmp $(MANPAGE))