Merge branch 'async-text-like-preview'

Implement asynchronous previewing for textual and pass-through (think
sixel) viewers.

Thanks to StillSteal, laur89, Joshua Jensch (a.k.a., patroclos) and
p-kolacz.

Fix hang on using previewer that takes over control over the
terminal (like 7z does to read password).

Thanks to PRESFIL.

Match by extension first in Default-256, which fixes some mismatches.

Fixed jobs with merged streams not being removed.

Fixed caching of graphics preview (broken on introducing proper caching).

ChangeLog

@@ -38,6 +38,10 @@
 	Extended cache of viewer's output to contain multiple entries (initial
 	version of the cache was added to skip redrawing graphics).
 
+	Implemented asynchronous previewing for textual and pass-through (think
+	sixel) viewers.  Thanks to StillSteal, laur89, Joshua
+	Jensch (a.k.a., patroclos) and p-kolacz.
+
 	Fixed pointing 'trashdir' to a symbolic link to a directory causing
 	issues.  Thanks to ChongChong He.
 
@@ -60,6 +64,9 @@
 	Fixed pane tabs being created after the last one instead of after current
 	tab.  Thanks to anonymous and n.e. at Vifm Q2A site.
 
+	Fixed hang on using previewer that takes over control over the
+	terminal (like 7z does to read password).  Thanks to PRESFIL.
+
 0.11-beta to 0.11 (2020-09-24)
 
 	Recommend against setting 'shellcmdflag' to "-ic" value.

THANKS

@@ -120,6 +120,7 @@ ksandr1v
 laggardkernel
 landhar
 Larry Hynes (larryhynes)
+laur89
 lcj
 Lomov Vladimir (lomov_vl)
 loongw
@@ -166,6 +167,7 @@ Ondrej Novy (onovy)
 oo-
 opennota
 ovk
+p-kolacz
 Pavel (neoascetic)
 Pavel Platto (hinidu)
 Pavneet Arora
@@ -216,6 +218,7 @@ Stephano (cao)
 Stephen Horst (sjhorst)
 Stephen L. Holtz (stephenholtz)
 Stéphane (istib)
+StillSteal
 sudo-nice
 Svadkos
 Svenn Are Bjerkem (svenn)

data/colors/Default-256.vifm

@@ -2,7 +2,7 @@
 
 " This is Vifm's default color scheme for terminals that support 256 colors.
 "
-" This file last updated: 4 December, 2020
+" This file last updated: 19 December, 2020
 
 highlight clear
 
@@ -49,10 +49,6 @@ highlight User2 ctermbg=240 ctermfg=231 cterm=bold
 highlight User3 ctermbg=240 ctermfg=white
 highlight User4 ctermbg=white ctermfg=240
 
-" software documentation
-highlight {COPYRIGHT,COPYING*,BUGS,ChangeLog*,FAQ,INSTALL*,LICENCE,LICENSE,NEWS,
-          \README,README.*,AUTHORS,TODO,THANKS}
-        \ cterm=none ctermfg=187 ctermbg=default
 " build system files
 highlight {Makefile,Makefile.am,Makefile.in,Makefile.win,*.mak,*.mk,*.m4,*.ac,
           \configure,CMakeLists.txt,*.cmake,*.pro,*.pri,*.sln}
@@ -90,3 +86,7 @@ highlight {*.patch,*.diff,*.py,*.cpp,*.hpp,*.mk,*.c,*.h,*.cpp,*.hpp,*.cc,*.hs,
           \*.pl,*.pm,*.t,*.cs,*.asp,*.dart,*.js,*.rb,*.scala,*.ts,*.coffee,
           \*.ml,*.mli,*.rs,*.sql,*.qml,vifmrc,vimrc,.vimrc,*.flex,*.ypp}
         \ cterm=none ctermfg=193 ctermbg=default
+" software documentation
+highlight {COPYRIGHT,COPYING*,BUGS,ChangeLog*,FAQ,INSTALL*,LICENCE,LICENSE,NEWS,
+          \README,README.*,AUTHORS,TODO,THANKS}
+        \ cterm=none ctermfg=187 ctermbg=default

data/vim/doc/plugin/vifm-plugin.txt

@@ -1,4 +1,4 @@
-*vifm-plugin.txt*    For Vifm version 0.11  Last change: 2020 Sep 24
+*vifm-plugin.txt*    For Vifm version 0.11  Last change: 2020 Dec 18
 
  Email for bugs and suggestions: <xaizek@posteo.net>
 
@@ -79,8 +79,12 @@ Effectively always enabled in neovim.
                                                *g:vifm_embed_split*
 A boolean variable.  When evaluates to true and Vifm is embedded in a
 terminal, it will be run inside a new split.  This allows commands to support
-|<mods>| and a |<count>| for controlling the orientation and size of the
-split.  False by default.
+Vim's |<mod>| and a |<count>| for controlling the orientation and size of the
+split, making it possible to run a command inside of Vim such as:
+  - `:vertical Vifm`
+  - `:leftabove vertical 60Vifm`
+See `:help :command-modifiers` for Vim's command modifiers documentation.
+False by default.
 
                                                *g:vifm_embed_cwd*
                                                {not on MS-Windows}

src/Makefile.win

@@ -16,7 +16,7 @@ override CFLAGS += -pthread
 ifeq ($(RELEASE),1)
     override CFLAGS += -O2
 else
-    override CFLAGS += -g -O0
+    override CFLAGS += -gdwarf-3 -O0
 endif
 
 MANGEN := groff

src/background.c

@@ -29,7 +29,7 @@
 #ifndef _WIN32
 #include <sys/wait.h> /* waitpid() */
 #endif
-#include <signal.h> /* kill() */
+#include <signal.h> /* SIG* kill() */
 #include <unistd.h> /* execve() fork() setpgid() setsid() */
 
 #include <assert.h> /* assert() */
@@ -190,7 +190,7 @@ bg_check(void)
 		int can_remove = (!p->running && p->use_count == 0);
 		pthread_spin_unlock(&p->status_lock);
 
-		active_jobs += (running != 0);
+		active_jobs += (running != 0 && p->in_menu);
 
 		if(!running && p->on_job_bar)
 		{
@@ -298,6 +298,10 @@ job_free(bg_job_t *job)
 	{
 		CloseHandle(job->hprocess);
 	}
+	if(job->hjob != NO_JOB_ID)
+	{
+		CloseHandle(job->hjob);
+	}
 #endif
 	if(job->output != NULL)
 	{
@@ -802,6 +806,8 @@ bg_run_external_job(const char cmd[], BgJobFlags flags)
 		place_on_job_bar(job);
 	}
 
+	job->in_menu = (flags & BJF_MENU_VISIBLE);
+
 	return job;
 }
 
@@ -812,7 +818,7 @@ launch_external(const char cmd[], int capture_output, int new_session,
 {
 	/* TODO: simplify this function (launch_external()) somehow, maybe split in
 	 *       two. */
-	int visible = (flags & BJF_JOB_BAR_VISIBLE);
+	int jb_visible = (flags & BJF_JOB_BAR_VISIBLE);
 	int merge_streams = (capture_output && (flags & BJF_MERGE_STREAMS));
 
 #ifndef _WIN32
@@ -928,7 +934,7 @@ launch_external(const char cmd[], int capture_output, int new_session,
 	}
 
 	bg_job_t *job = add_background_job(pid, cmd, (uintptr_t)error_pipe[0], 0,
-			BJT_COMMAND, visible);
+			BJT_COMMAND, jb_visible);
 
 	if(capture_output)
 	{
@@ -988,12 +994,15 @@ launch_external(const char cmd[], int capture_output, int new_session,
 	sh_cmd = win_make_sh_cmd(cmd, by);
 
 	wide_cmd = to_wide(sh_cmd);
-	int started = CreateProcessW(NULL, wide_cmd, NULL, NULL, 1, 0, NULL, NULL,
-			&startup, &pinfo);
+	int started = CreateProcessW(NULL, wide_cmd, NULL, NULL, 1, CREATE_SUSPENDED,
+			NULL, NULL, &startup, &pinfo);
 	free(wide_cmd);
 	CloseHandle(hnul);
 	CloseHandle(startup.hStdOutput);
-	CloseHandle(startup.hStdError);
+	if(startup.hStdError != startup.hStdOutput)
+	{
+		CloseHandle(startup.hStdError);
+	}
 
 	if(!started)
 	{
@@ -1002,18 +1011,28 @@ launch_external(const char cmd[], int capture_output, int new_session,
 		return NULL;
 	}
 
+	/* Put the process into its own job object and start its main thread. */
+	HANDLE hjob = CreateJobObject(NULL, NULL);
+	AssignProcessToJobObject(hjob, pinfo.hProcess);
+	ResumeThread(pinfo.hThread);
 	CloseHandle(pinfo.hThread);
 
 	bg_job_t *job = add_background_job(pinfo.dwProcessId, sh_cmd,
-			(uintptr_t)herr, (uintptr_t)pinfo.hProcess, BJT_COMMAND, visible);
+			(uintptr_t)herr, (uintptr_t)pinfo.hProcess, BJT_COMMAND, jb_visible);
 	free(sh_cmd);
 
 	if(job == NULL)
 	{
 		CloseHandle(herr);
 		CloseHandle(hout);
+		CloseHandle(pinfo.hProcess);
+		CloseHandle(hjob);
+		return NULL;
 	}
-	else if(capture_output)
+
+	job->hjob = hjob;
+
+	if(capture_output)
 	{
 		int fd = _open_osfhandle((intptr_t)hout, _O_RDONLY);
 		if(fd == -1)
@@ -1127,7 +1146,7 @@ add_background_job(pid_t pid, const char cmd[], uintptr_t err, uintptr_t data,
 	pthread_spin_init(&new->errors_lock, PTHREAD_PROCESS_PRIVATE);
 	pthread_spin_init(&new->status_lock, PTHREAD_PROCESS_PRIVATE);
 	new->running = 1;
-	new->use_count = (type == BJT_COMMAND ? 1 : 0);
+	new->use_count = 0;
 	new->exit_code = -1;
 
 	new->output = NULL;
@@ -1137,10 +1156,12 @@ add_background_job(pid_t pid, const char cmd[], uintptr_t err, uintptr_t data,
 #else
 	new->err_stream = (HANDLE)err;
 	new->hprocess = (HANDLE)data;
+	new->hjob = INVALID_HANDLE_VALUE;
 #endif
 
 	if(new->err_stream != NO_JOB_ID)
 	{
+		++new->use_count;
 		pthread_mutex_lock(&new_err_jobs_lock);
 		new->err_next = new_err_jobs;
 		new_err_jobs = new;
@@ -1160,6 +1181,8 @@ add_background_job(pid_t pid, const char cmd[], uintptr_t err, uintptr_t data,
 	new->bg_op.descr = NULL;
 	new->bg_op.cancelled = 0;
 
+	new->in_menu = 1;
+
 	bg_jobs = new;
 	return new;
 }
@@ -1265,6 +1288,30 @@ bg_job_cancelled(bg_job_t *job)
 	return job->cancelled;
 }
 
+void
+bg_job_terminate(bg_job_t *job)
+{
+	if(job->type != BJT_COMMAND || !bg_job_is_running(job))
+	{
+		return;
+	}
+
+#ifndef _WIN32
+	if(kill(job->pid, SIGKILL) != 0)
+	{
+		LOG_SERROR_MSG(errno, "Failed to send SIGKILL to %" PRINTF_ULL,
+				(unsigned long long)job->pid);
+	}
+#else
+	if(!TerminateJobObject(job->hjob, 0))
+	{
+		LOG_ERROR_MSG("Failed to terminate job of process %" PRINTF_ULL,
+				(unsigned long long)job->pid);
+		LOG_WERROR(GetLastError());
+	}
+#endif
+}
+
 int
 bg_job_is_running(bg_job_t *job)
 {

src/background.h

@@ -50,7 +50,8 @@ typedef enum
 {
 	BJF_NONE            = 0,      /* No flags set. */
 	BJF_JOB_BAR_VISIBLE = 1 << 0, /* Makes the job appear on the job bar. */
-	BJF_MERGE_STREAMS   = 1 << 1, /* Merge error stream into output stream. */
+	BJF_MENU_VISIBLE    = 1 << 1, /* Makes the job appear in :jobs menu. */
+	BJF_MERGE_STREAMS   = 1 << 2, /* Merge error stream into output stream. */
 }
 BgJobFlags;
 
@@ -102,13 +103,16 @@ typedef struct bg_job_t
 #else
 	HANDLE err_stream; /* stderr stream of the job or invalid handle. */
 	HANDLE hprocess;   /* Handle to the process of the job or invalid handle. */
+	HANDLE hjob;       /* Handle to the process job object. */
 #endif
 
 	struct bg_job_t *next;     /* Link to the next element in bg_jobs list. */
 
 	/* Used by error thread for BJT_COMMAND jobs. */
-	int drained;               /* Whether error stream of no interest anymore. */
 	struct bg_job_t *err_next; /* Link to the next element in error read list. */
+	int drained;               /* Whether error stream of no interest anymore. */
+
+	int in_menu; /* Whether this task is visible in :jobs menu. */
 }
 bg_job_t;
 
@@ -167,6 +171,9 @@ int bg_job_cancel(bg_job_t *job);
  * zero is returned. */
 int bg_job_cancelled(bg_job_t *job);
 
+/* Terminates the job in a forceful way leaving it no chance to respond. */
+void bg_job_terminate(bg_job_t *job);
+
 /* Checks whether the job is still running.  Returns non-zero if so, otherwise
  * zero is returned. */
 int bg_job_is_running(bg_job_t *job);

src/event_loop.c

@@ -31,6 +31,7 @@
 
 #include "cfg/config.h"
 #include "compat/curses.h"
+#include "compat/fs_limits.h"
 #include "engine/completion.h"
 #include "engine/keys.h"
 #include "engine/mode.h"
@@ -44,6 +45,7 @@
 #include "ui/ui.h"
 #include "utils/log.h"
 #include "utils/macros.h"
+#include "utils/path.h"
 #include "utils/test_helpers.h"
 #include "utils/utf8.h"
 #include "utils/utils.h"
@@ -53,10 +55,12 @@
 #include "ipc.h"
 #include "registers.h"
 #include "status.h"
+#include "vcache.h"
 #include "vifm.h"
 
 static int ensure_term_is_ready(void);
 static int get_char_async_loop(WINDOW *win, wint_t *c, int timeout);
+static int is_previewed(const char path[]);
 static void process_scheduled_updates(void);
 TSTATIC int process_scheduled_updates_of_view(view_t *view);
 static void update_hardware_cursor(void);
@@ -377,6 +381,11 @@ get_char_async_loop(WINDOW *win, wint_t *c, int timeout)
 				ipc_check(curr_stats.ipc);
 			}
 
+			if(vcache_check(&is_previewed))
+			{
+				stats_redraw_later();
+			}
+
 			wtimeout(win, delay_slice);
 			timeout -= delay_slice;
 
@@ -421,6 +430,21 @@ get_char_async_loop(WINDOW *win, wint_t *c, int timeout)
 	return ERR;
 }
 
+/* Checks if preview of specified path is visible.  Returns non-zero if so and
+ * zero otherwise. */
+static int
+is_previewed(const char path[])
+{
+	if(curr_stats.preview.on)
+	{
+		char previewed[PATH_MAX + 1];
+		get_current_full_path(curr_view, sizeof(previewed), previewed);
+		return paths_are_equal(path, previewed);
+	}
+
+	return (fview_previews(curr_view, path) || fview_previews(other_view, path));
+}
+
 /* Updates TUI or its elements if something is scheduled. */
 static void
 process_scheduled_updates(void)

src/lua/vlua.c

@@ -335,7 +335,7 @@ vifm_startjob(lua_State *lua)
 	check_field(lua, 1, "cmd", LUA_TSTRING);
 	const char *cmd = lua_tostring(lua, -1);
 
-	BgJobFlags flags = BJF_NONE;
+	BgJobFlags flags = BJF_MENU_VISIBLE;
 	if(check_opt_field(lua, 1, "visible", LUA_TBOOLEAN))
 	{
 		flags |= (lua_toboolean(lua, -1) ? BJF_JOB_BAR_VISIBLE : BJF_NONE);

src/modes/view.c

@@ -1122,12 +1122,14 @@ get_view_data(modview_info_t *vi, const char file_to_view[])
 	if(vi->curr_viewer == vi->ext_viewer)
 	{
 		/* No macros in this viewer. */
-		lines = vcache_lookup(file_to_view, vi->ext_viewer, kind, INT_MAX, &error);
+		lines = vcache_lookup(file_to_view, vi->ext_viewer, kind, INT_MAX, VC_SYNC,
+				&error);
 	}
 	else
 	{
 		char *expanded = (viewer == NULL ? NULL : qv_expand_viewer(viewer));
-		lines = vcache_lookup(file_to_view, expanded, kind, INT_MAX, &error);
+		lines = vcache_lookup(file_to_view, expanded, kind, INT_MAX, VC_SYNC,
+				&error);
 		free(expanded);
 	}