add whitespace, non-ASCII, json checkers to CI, fix issues

.github/workflows/checksrc.yml

@@ -53,6 +53,14 @@ jobs:
           # shellcheck disable=SC2046
           shellcheck $(grep -l -E '^#!(/usr/bin/env bash|/bin/sh|/bin/bash)' $(git ls-files))
 
+      - name: 'spacecheck'
+        run: scripts/spacecheck.pl
+
+      - name: 'json check'
+        run: |
+          jq --indent 4 < tests.json > tests-expected.json
+          diff -u tests.json tests-expected.json
+
       - name: 'codespell'
         run: |
           source ~/venv/bin/activate

README.md

@@ -173,7 +173,7 @@ good. For all the documented goodness, use a more modern libcurl.
 | trurl Feature   |  Minimum libcurl Version |
 |-----------------|--------------------------|
 | imap-options    |   7.30.0                 |
-| normalize-ipv   |   7.77.0                 | 
+| normalize-ipv   |   7.77.0                 |
 | white-space     |   7.78.0                 |
 | url-strerror    |   7.80.0                 |
 | zone-id         |   7.81.0                 |
@@ -183,4 +183,3 @@ good. For all the documented goodness, use a more modern libcurl.
 
 For more details on how trurl will behave if these features are missing see [URL Quirks](https://github.com/curl/trurl/blob/master/URL-QUIRKS.md).
 To see the features your version of trurl supports as well as the version of libcurl it is built with, run the following command:  `trurl --version`
-

completions/_trurl.zsh.in

@@ -26,7 +26,7 @@
 
 # This file is generated from trurls generate_completions.sh
 
-# standalone flags - things that have now follow on 
+# standalone flags - things that have now follow on
 standalone_flags=(@TRURL_STANDALONE_FLAGS@)
 
 # component options - flags that expected to come after them
@@ -52,11 +52,11 @@ if (( "${random_options[(Ie)${words[$CURRENT-1]}]}" )); then
     return 0
 fi
 
-# calling compadd directly allows us the let the flags be 
-# repeatable so we can recall --set, --get etc. 
+# calling compadd directly allows us the let the flags be
+# repeatable so we can recall --set, --get etc.
 repeatable=( "${component_options[@]}" "${random_options[@]}" )
 args=( "${repeatable[@]}" )
-# only apply single completions which haven't been used. 
+# only apply single completions which haven't been used.
 for sf in "${standalone_flags[@]}"; do
     if ! (( "${words[(Ie)$sf]}" ));  then
         args+=("$sf")

scripts/generate_completions.sh

@@ -32,7 +32,6 @@ fi
 TRURL_MD_FILE=$1
 
 
-
 ALL_FLAGS="$(sed  -n \
         -e 's/"//g' \
         -e '/\# URL COMPONENTS/q;p' \
@@ -53,7 +52,7 @@ TRURL_COMPONENT_LIST="$(sed -n \
     | awk '{printf "\"%s\" ", $2}')"
 
 for flag in $ALL_FLAGS; do
-  # these are now TRURL_STANDALONE 
+  # these are now TRURL_STANDALONE
   if echo "$flag" | grep -q "="; then
     TRURL_COMPONENT_OPTIONS+="$(echo "$flag" \
     | awk '{split($0, a, ","); for(i in a) {printf "%s ", a[i]}}' \

scripts/spacecheck.pl

@@ -0,0 +1,194 @@
+#!/usr/bin/env perl
+#***************************************************************************
+#                                  _   _ ____  _
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
+#
+# Copyright (C) Viktor Szakats
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+# SPDX-License-Identifier: curl
+#
+###########################################################################
+
+use strict;
+use warnings;
+
+my @tabs = (
+    "Makefile",
+    "\\.sln\$",
+    "^testfiles/test.+\.txt",
+);
+
+my @mixed_eol = (
+    "^testfiles/test.+\.txt",
+);
+
+my @need_crlf = (
+    "\\.sln\$",
+);
+
+my @space_at_eol = (
+    "^testfiles/test.+\.txt",
+);
+
+my @eol_at_eof = (
+    "^testfiles/test.+\.txt",
+);
+
+my @non_ascii_allowed = (
+    '\xC3\xA4',  # UTF-8 for https://codepoints.net/U+00E4 LATIN SMALL LETTER A WITH DIAERESIS
+    '\xC3\xA5',  # UTF-8 for https://codepoints.net/U+00E5 LATIN SMALL LETTER A WITH RING ABOVE
+    '\xC3\xB6',  # UTF-8 for https://codepoints.net/U+00F6 LATIN SMALL LETTER O WITH DIAERESIS
+);
+
+my $non_ascii_allowed = join(', ', @non_ascii_allowed);
+
+my @non_ascii = (
+    "THANKS",
+);
+
+sub fn_match {
+    my ($filename, @masklist) = @_;
+
+    foreach my $mask (@masklist) {
+        if($filename =~ $mask) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+sub eol_detect {
+    my ($content) = @_;
+
+    my $cr = () = $content =~ /\r/g;
+    my $lf = () = $content =~ /\n/g;
+
+    if($cr > 0 && $lf == 0) {
+        return "cr";
+    }
+    elsif($cr == 0 && $lf > 0) {
+        return "lf";
+    }
+    elsif($cr == 0 && $lf == 0) {
+        return "bin";
+    }
+    elsif($cr == $lf) {
+        return "crlf";
+    }
+
+    return "";
+}
+
+my $issues = 0;
+
+open my $git_ls_files, '-|', 'git ls-files' or die "Failed running git ls-files: $!";
+while(my $filename = <$git_ls_files>) {
+    chomp $filename;
+
+    open my $fh, '<', $filename or die "Cannot open '$filename': $!";
+    my $content = do { local $/; <$fh> };
+    close $fh;
+
+    my @err = ();
+
+    if(!fn_match($filename, @tabs) &&
+        $content =~ /\t/) {
+        push @err, "content: has tab";
+    }
+
+    my $eol = eol_detect($content);
+
+    if($eol eq "" &&
+        !fn_match($filename, @mixed_eol)) {
+        push @err, "content: has mixed EOL types";
+    }
+
+    if($eol ne "crlf" &&
+        fn_match($filename, @need_crlf)) {
+        push @err, "content: must use CRLF EOL for this file type";
+    }
+
+    if($eol ne "lf" && $content ne "" &&
+        !fn_match($filename, @need_crlf) &&
+        !fn_match($filename, @mixed_eol)) {
+        push @err, "content: must use LF EOL for this file type";
+    }
+
+    if(!fn_match($filename, @space_at_eol) &&
+       $content =~ /[ \t]\n/) {
+        my $line;
+        for my $l (split(/\n/, $content)) {
+            $line++;
+            if($l =~ /[ \t]$/) {
+                push @err, "line $line: trailing whitespace";
+            }
+        }
+    }
+
+    if($content ne "" &&
+        $content !~ /\n\z/ &&
+        !fn_match($filename, @eol_at_eof)) {
+        push @err, "content: has no EOL at EOF";
+    }
+
+    if($content =~ /\n\n\z/ ||
+        $content =~ /\r\n\r\n\z/) {
+        push @err, "content: has multiple EOL at EOF";
+    }
+
+    if($content =~ /\n\n\n\n/ ||
+        $content =~ /\r\n\r\n\r\n\r\n/) {
+        push @err, "content: has 3 or more consecutive empty lines";
+    }
+
+    if($content =~ /([\x00-\x08\x0b\x0c\x0e-\x1f\x7f])/) {
+        push @err, "content: has binary contents";
+    }
+
+    if($filename !~ /tests\/data/) {
+        # the tests have no allowed UTF bytes
+        $content =~ s/[$non_ascii_allowed]//g;
+    }
+
+    if(!fn_match($filename, @non_ascii) &&
+       ($content =~ /([\x80-\xff]+)/)) {
+        my $non = $1;
+        my $hex;
+        for my $e (split(//, $non)) {
+            $hex .= sprintf("%s%02x", $hex ? " ": "", ord($e));
+        }
+        my $line;
+        for my $l (split(/\n/, $content)) {
+            $line++;
+            if($l =~ /([\x80-\xff]+)/) {
+                push @err, "line $line: has non-ASCII: '$non' ($hex)";
+            }
+        }
+    }
+
+    if(@err) {
+        $issues++;
+        foreach my $err (@err) {
+            print "$filename: $err\n";
+        }
+    }
+}
+close $git_ls_files;
+
+if($issues) {
+    exit 1;
+}