Merge branch 'futurefeatures'

This merges support for feature flags.

Closes fish-shell#4940

CHANGELOG.md

@@ -5,6 +5,8 @@ This section is for changes merged to the `major` branch that are not also merge
 - The `IFS` variable is deprecated and will be removed in fish 4.0 (#4156).
 - The `function --on-process-exit` event will be removed in future (#4700). Use the `fish_exit` event instead.
 - `$_` is deprecated and will removed in the future (#813). Use `status current-command` in a subshell instead.
+- `^` as a redirection deprecated and will be removed in the future. (#4394). Use `2>` to redirect stderr. This is controlled by the `stderr-nocaret` feature flag.
+- `?` as a glob is deprecated and will be removed in the future. (#4520). This is controlled by the `qmark-noglob` feature flag.
 
 ## Notable non-backward compatible changes
 - `.` command no longer exists -- use `source` (#4294).
@@ -15,10 +17,9 @@ This section is for changes merged to the `major` branch that are not also merge
 - Successive commas in brace expansions are handled in less surprising manner (`{,,,}` expands to four empty strings rather than an empty string, a comma and an empty string again). (#3002, #4632).
 - `%` is no longer used for process and job expansion. `$fish_pid` and `$last_pid` have taken the place of `%self` and `%last` respectively. (#4230, #1202)
 - The new `math` builtin (see below) does not support logical expressions; `test` should be used instead (#4777).
-- The `?` wildcard has been removed (#4520).
-- The `^` caret redirection for stderr has been removed (#4394). To redirect stderr, `2>/some/path` may be used, or `2>|` as a pipe.
 
 ## Notable fixes and improvements
+- A new feature flags mechanism is added for staging deprecations and breaking changes. (#4940) 
 - `wait` builtin is added for waiting on processes (#4498).
 - `read` has a new `--delimiter` option as a better alternative to the `IFS` variable (#4256).
 - `read` writes directly to stdout if called without arguments (#4407)

CMakeLists.txt

@@ -58,6 +58,7 @@ SET(FISH_SRCS
     src/postfork.cpp src/proc.cpp src/reader.cpp src/sanity.cpp src/screen.cpp
     src/signal.cpp src/tnode.cpp src/tokenizer.cpp src/utf8.cpp src/util.cpp
     src/wcstringutil.cpp src/wgetopt.cpp src/wildcard.cpp src/wutil.cpp
+    src/future_feature_flags.cpp
 )
 
 # Header files are just globbed.

Makefile.in

@@ -118,7 +118,8 @@ FISH_OBJS := obj/autoload.o obj/builtin.o obj/builtin_bg.o obj/builtin_bind.o ob
 	obj/parse_productions.o obj/parse_tree.o obj/parse_util.o obj/parser.o \
 	obj/parser_keywords.o obj/path.o obj/postfork.o obj/proc.o obj/reader.o \
 	obj/sanity.o obj/screen.o obj/signal.o obj/tinyexpr.o obj/tokenizer.o obj/tnode.o obj/utf8.o \
-	obj/util.o obj/wcstringutil.o obj/wgetopt.o obj/wildcard.o obj/wutil.o
+	obj/util.o obj/wcstringutil.o obj/wgetopt.o obj/wildcard.o obj/wutil.o \
+	obj/future_feature_flags.o
 
 FISH_INDENT_OBJS := obj/fish_indent.o obj/print_help.o $(FISH_OBJS)
 

doc_src/index.hdr.in

@@ -92,6 +92,7 @@ Some characters can not be written directly on the command line. For these chara
 - '<code>\\$</code>' escapes the dollar character
 - '<code>\\\\</code>' escapes the backslash character
 - '<code>\\*</code>' escapes the star character
+- '<code>\\?</code>' escapes the question mark character
 - '<code>\\~</code>' escapes the tilde character
 - '<code>\\#</code>' escapes the hash character
 - '<code>\\(</code>' escapes the left parenthesis character
@@ -141,11 +142,11 @@ An example of a file redirection is `echo hello > output.txt`, which directs the
 
 - To read standard input from a file, write `<SOURCE_FILE`
 - To write standard output to a file, write `>DESTINATION`
-- To write standard error to a file, write `^DESTINATION`
+- To write standard error to a file, write `2>DESTINATION`
 - To append standard output to a file, write `>>DESTINATION_FILE`
-- To append standard error to a file, write `^^DESTINATION_FILE`
+- To append standard error to a file, write `2>>DESTINATION_FILE`
 
-- To not overwrite ("clobber") an existing file, write '>?DESTINATION' or '^?DESTINATION'
+- To not overwrite ("clobber") an existing file, write '>?DESTINATION' or '2>?DESTINATION'
 
 `DESTINATION` can be one of the following:
 
@@ -157,15 +158,15 @@ An example of a file redirection is `echo hello > output.txt`, which directs the
 
 Example:
 
-To redirect both standard output and standard error to the file 'all_output.txt', you can write `echo Hello > all_output.txt ^&1`.
+To redirect both standard output and standard error to the file 'all_output.txt', you can write `echo Hello > all_output.txt 2>&1`.
 
 Any file descriptor can be redirected in an arbitrary way by prefixing the redirection with the file descriptor.
 
 - To redirect input of FD N, write `N<DESTINATION`
 - To redirect output of FD N, write `N>DESTINATION`
 - To append the output of FD N to a file, write `N>>DESTINATION_FILE`
 
-Example: `echo Hello 2>output.stderr` and `echo Hello ^output.stderr` are equivalent, and write the standard error (file descriptor 2) of the target program to `output.stderr`.
+Example: `echo Hello 2>output.stderr` and `echo Hello 2>output.stderr` are equivalent, and write the standard error (file descriptor 2) of the target program to `output.stderr`.
 
 \subsection piping Piping
 
@@ -329,7 +330,7 @@ These are the general purpose tab completions that `fish` provides:
 
 - Completion of usernames for tilde expansion.
 
-- Completion of filenames, even on strings with wildcards such as '`*`' and '`**`'.
+- Completion of filenames, even on strings with wildcards such as '`*`', '`**`' and '`?`'.
 
 `fish` provides a large number of program specific completions. Most of these completions are simple options like the `-l` option for `ls`, but some are more advanced. The latter include:
 
@@ -417,7 +418,9 @@ When an argument for a program is given on the commandline, it undergoes the pro
 
 \subsection expand-wildcard Wildcards
 
-If a star (`*`) is present in the parameter, `fish` attempts to match the given parameter to any files in such a way that:
+If a star (`*`) or a question mark (`?`) is present in the parameter, `fish` attempts to match the given parameter to any files in such a way that:
+
+- `?` can match any single character except '/'.
 
 - `*` can match any string of characters not containing '/'. This includes matching an empty string.
 
@@ -443,6 +446,8 @@ Examples:
 
 - `a*` matches any files beginning with an 'a' in the current directory.
 
+- `???` matches any file in the current directory whose name is exactly three characters long.
+
 - `**` matches any files and directories in the current directory and all of its subdirectories.
 
 Note that for most commands, if any wildcard fails to expand, the command is not executed, <a href='#variables-status'>`$status`</a> is set to nonzero, and a warning is printed. This behavior is consistent with setting `shopt -s failglob` in bash. There are exactly 3 exceptions, namely <a href="commands.html#set">`set`</a>, <a href="commands.html#count">`count`</a> and <a href="commands.html#for">`for`</a>. Their globs are permitted to expand to zero arguments, as with `shopt -s nullglob` in bash.
@@ -1242,6 +1247,29 @@ function on_exit --on-process $fish_pid
 end
 \endfish
 
+\section featureflags Future feature flags
+
+Feature flags are how fish stages changes that might break scripts. Breaking changes are introduced as opt-in, in a few releases they become opt-out, and eventually the old behavior is removed.
+
+You can see the current list of features via `status features`:
+
+\fish
+> status features
+stderr-nocaret  on     3.0      ^ no longer redirects stderr
+qmark-noglob    off    3.0      ? no longer globs
+\endfish
+
+There are two breaking changes in fish 3.0: caret `^` no longer redirects stderr, and question mark `?` is no longer a glob. These changes are off by default. They can be enabled on a per session basis:
+
+\fish
+> fish --features qmark-noglob,stderr-nocaret
+\endfish
+
+or opted into globally for a user:
+
+\fish
+> set -U fish_features stderr-nocaret qmark-noglob
+\endfish
 
 \section other Other features
 

share/completions/git.fish

@@ -127,7 +127,11 @@ function __fish_git_files
         # Be careful about the ordering here!
         #
         # HACK: To allow this to work both with and without '?' globs
-        set -l dq '??'
+        set -l dq '\\?\\?'
+        if status test-feature qmark-noglob
+            # ? is not a glob
+            set dq '??'
+        end
         switch "$stat"
             case DD AU UD UA DU AA UU
                 # Unmerged

src/builtin_status.cpp

@@ -10,6 +10,7 @@
 #include "builtin_status.h"
 #include "common.h"
 #include "fallback.h"  // IWYU pragma: keep
+#include "future_feature_flags.h"
 #include "io.h"
 #include "parser.h"
 #include "proc.h"
@@ -31,6 +32,8 @@ enum status_cmd_t {
     STATUS_LINE_NUMBER,
     STATUS_SET_JOB_CONTROL,
     STATUS_STACK_TRACE,
+    STATUS_FEATURES,
+    STATUS_TEST_FEATURE,
     STATUS_UNDEF
 };
 
@@ -40,6 +43,7 @@ const enum_map<status_cmd_t> status_enum_map[] = {
     {STATUS_FILENAME, L"current-filename"},
     {STATUS_FUNCTION, L"current-function"},
     {STATUS_LINE_NUMBER, L"current-line-number"},
+    {STATUS_FEATURES, L"features"},
     {STATUS_FILENAME, L"filename"},
     {STATUS_FUNCTION, L"function"},
     {STATUS_IS_BLOCK, L"is-block"},
@@ -54,6 +58,7 @@ const enum_map<status_cmd_t> status_enum_map[] = {
     {STATUS_LINE_NUMBER, L"line-number"},
     {STATUS_STACK_TRACE, L"print-stack-trace"},
     {STATUS_STACK_TRACE, L"stack-trace"},
+    {STATUS_TEST_FEATURE, L"test-feature"},
     {STATUS_UNDEF, NULL}};
 #define status_enum_map_len (sizeof status_enum_map / sizeof *status_enum_map)
 
@@ -66,6 +71,9 @@ const enum_map<status_cmd_t> status_enum_map[] = {
         break;                                                                              \
     }
 
+/// Values that may be returned from the test-feature option to status.
+enum { TEST_FEATURE_ON, TEST_FEATURE_OFF, TEST_FEATURE_NOT_RECOGNIZED };
+
 int job_control_str_to_mode(const wchar_t *mode, wchar_t *cmd, io_streams_t &streams) {
     if (wcscmp(mode, L"full") == 0) {
         return JOB_CONTROL_ALL;
@@ -82,6 +90,7 @@ struct status_cmd_opts_t {
     bool print_help = false;
     int level = 1;
     int new_job_control_mode = -1;
+    const wchar_t *feature_name;
     status_cmd_t status_cmd = STATUS_UNDEF;
 };
 
@@ -126,6 +135,15 @@ static bool set_status_cmd(wchar_t *const cmd, status_cmd_opts_t &opts, status_c
     return true;
 }
 
+/// Print the features and their values.
+static void print_features(io_streams_t &streams) {
+    for (const auto &md : features_t::metadata) {
+        int set = feature_test(md.flag);
+        streams.out.append_format(L"%ls\t%s\t%ls\t%ls\n", md.name, set ? "on" : "off", md.groups,
+                                  md.description);
+    }
+}
+
 static int parse_cmd_opts(status_cmd_opts_t &opts, int *optind,  //!OCLINT(high ncss method)
                           int argc, wchar_t **argv, parser_t &parser, io_streams_t &streams) {
     wchar_t *cmd = argv[0];
@@ -307,6 +325,24 @@ int builtin_status(parser_t &parser, io_streams_t &streams, wchar_t **argv) {
             job_control_mode = opts.new_job_control_mode;
             break;
         }
+        case STATUS_FEATURES: {
+            print_features(streams);
+            break;
+        }
+        case STATUS_TEST_FEATURE: {
+            if (args.size() != 1) {
+                const wchar_t *subcmd_str = enum_to_str(opts.status_cmd, status_enum_map);
+                streams.err.append_format(BUILTIN_ERR_ARG_COUNT2, cmd, subcmd_str, 1, args.size());
+                return STATUS_INVALID_ARGS;
+            }
+            const auto *metadata = features_t::metadata_for(args.front().c_str());
+            if (!metadata) {
+                retval = TEST_FEATURE_NOT_RECOGNIZED;
+            } else {
+                retval = feature_test(metadata->flag) ? TEST_FEATURE_ON : TEST_FEATURE_OFF;
+            }
+            break;
+        }
         case STATUS_FILENAME: {
             CHECK_FOR_UNEXPECTED_STATUS_ARGS(opts.status_cmd)
             const wchar_t *fn = parser.current_filename();

src/common.cpp

@@ -38,6 +38,7 @@
 #include "env.h"
 #include "expand.h"
 #include "fallback.h"  // IWYU pragma: keep
+#include "future_feature_flags.h"
 #include "proc.h"
 #include "wildcard.h"
 #include "wutil.h"  // IWYU pragma: keep
@@ -928,9 +929,11 @@ static bool unescape_string_var(const wchar_t *in, wcstring *out) {
 static void escape_string_script(const wchar_t *orig_in, size_t in_len, wcstring &out,
                                  escape_flags_t flags) {
     const wchar_t *in = orig_in;
-    bool escape_all = static_cast<bool>(flags & ESCAPE_ALL);
-    bool no_quoted = static_cast<bool>(flags & ESCAPE_NO_QUOTED);
-    bool no_tilde = static_cast<bool>(flags & ESCAPE_NO_TILDE);
+    const bool escape_all = static_cast<bool>(flags & ESCAPE_ALL);
+    const bool no_quoted = static_cast<bool>(flags & ESCAPE_NO_QUOTED);
+    const bool no_tilde = static_cast<bool>(flags & ESCAPE_NO_TILDE);
+    const bool no_caret = feature_test(features_t::stderr_nocaret);
+    const bool no_qmark = feature_test(features_t::qmark_noglob);
 
     int need_escape = 0;
     int need_complex_escape = 0;
@@ -995,6 +998,11 @@ static void escape_string_script(const wchar_t *orig_in, size_t in_len, wcstring
                     out += *in;
                     break;
                 }
+                case ANY_CHAR: {
+                    // See #1614
+                    out += L'?';
+                    break;
+                }
                 case ANY_STRING: {
                     out += L'*';
                     break;
@@ -1003,10 +1011,12 @@ static void escape_string_script(const wchar_t *orig_in, size_t in_len, wcstring
                     out += L"**";
                     break;
                 }
+
                 case L'&':
                 case L'$':
                 case L' ':
                 case L'#':
+                case L'^':
                 case L'<':
                 case L'>':
                 case L'(':
@@ -1015,12 +1025,14 @@ static void escape_string_script(const wchar_t *orig_in, size_t in_len, wcstring
                 case L']':
                 case L'{':
                 case L'}':
+                case L'?':
                 case L'*':
                 case L'|':
                 case L';':
                 case L'"':
                 case L'~': {
-                    if (!no_tilde || c != L'~') {
+                    bool char_is_normal = (c == L'~' && no_tilde) || (c == L'^' && no_caret) || (c == L'?' && no_qmark);
+                    if (!char_is_normal) {
                         need_escape = 1;
                         if (escape_all) out += L'\\';
                     }
@@ -1348,6 +1360,12 @@ static bool unescape_string_internal(const wchar_t *const input, const size_t in
                     }
                     break;
                 }
+                case L'?': {
+                    if (unescape_special && !feature_test(features_t::qmark_noglob)) {
+                        to_append_or_none = ANY_CHAR;
+                    }
+                    break;
+                }
                 case L'$': {
                     if (unescape_special) {
                         to_append_or_none = VARIABLE_EXPAND;

src/expand.cpp

@@ -828,6 +828,10 @@ static void remove_internal_separator(wcstring *str, bool conv) {
     if (conv) {
         for (size_t idx = 0; idx < str->size(); idx++) {
             switch (str->at(idx)) {
+                case ANY_CHAR: {
+                    str->at(idx) = L'?';
+                    break;
+                }
                 case ANY_STRING:
                 case ANY_STRING_RECURSIVE: {
                     str->at(idx) = L'*';

src/fish.cpp

@@ -44,6 +44,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 #include "fallback.h"  // IWYU pragma: keep
 #include "fish_version.h"
 #include "function.h"
+#include "future_feature_flags.h"
 #include "history.h"
 #include "io.h"
 #include "parser.h"
@@ -61,6 +62,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 // container to hold the options specified within the command line
 class fish_cmd_opts_t {
    public:
+    // Future feature flags values string
+    wcstring features;
     // Commands to be executed in place of interactive shell.
     std::vector<std::string> batch_cmds;
     // Commands to execute after the shell's config has been read.
@@ -238,9 +241,10 @@ int run_command_list(std::vector<std::string> *cmds, const io_chain_t &io) {
 
 /// Parse the argument list, return the index of the first non-flag arguments.
 static int fish_parse_opt(int argc, char **argv, fish_cmd_opts_t *opts) {
-    static const char *short_opts = "+hilnvc:C:p:d:D:";
+    static const char *short_opts = "+hilnvc:C:p:d:f:D:";
     static const struct option long_opts[] = {{"command", required_argument, NULL, 'c'},
                                               {"init-command", required_argument, NULL, 'C'},
+                                              {"features", required_argument, NULL, 'f'},
                                               {"debug-level", required_argument, NULL, 'd'},
                                               {"debug-stack-frames", required_argument, NULL, 'D'},
                                               {"interactive", no_argument, NULL, 'i'},
@@ -277,6 +281,10 @@ static int fish_parse_opt(int argc, char **argv, fish_cmd_opts_t *opts) {
                 }
                 break;
             }
+            case 'f': {
+                opts->features = str2wcstring(optarg);
+                break;
+            }
             case 'h': {
                 opts->batch_cmds.push_back("__fish_print_help fish");
                 break;
@@ -375,6 +383,15 @@ int main(int argc, char **argv) {
 
     const struct config_paths_t paths = determine_config_directory_paths(argv[0]);
     env_init(&paths);
+    // Set features early in case other initialization depends on them.
+    // Start with the ones set in the environment, then those set on the command line (so the
+    // command line takes precedence).
+    if (auto features_var = env_get(L"fish_features")) {
+        for (const wcstring &s : features_var->as_list()) {
+            mutable_fish_features().set_from_string(s);
+        }
+    }
+    mutable_fish_features().set_from_string(opts.features);
     proc_init();
     builtin_init();
     misc_init();