tests: bump Docker image (Alpine 3.12), fix tests (#2518)

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 
 common: &common
   docker:
-    - image: neomake/vims-for-tests:50@sha256:7a9de350cebf98b1d67edc0fc74cc3e925a45b5a956ffbbf26a93d0262ada45d
+    - image: neomake/vims-for-tests:51@sha256:b513549922f56d291d41cb3f5660be27ba1cd1bb7d696d13925df7a624a347a4
   working_directory: ~/repo
   steps:
     - checkout

Dockerfile.tests

@@ -1,5 +1,5 @@
 # From https://github.com/tweekmonster/vim-testbed.
-FROM testbed/vim:18@sha256:2b1872c68b6e9dfbfd8f804ad9727516ee971150142a80df354c244dcafa59c4
+FROM testbed/vim:19@sha256:5a55710f70f20e6a45aa07bcf5aaac58c5545c90f7fdc1eaaf55790cf5318d78
 
 # Currently tested versions:
 #  - v7.3.429 (Ubuntu Precise, 12.04LTS)
@@ -18,7 +18,7 @@ RUN install_vim -tag v8.1.0622 -name vim81 -build \
                 -tag neovim:v0.3.8 -py3 -build \
   && rm -rf /vim-build/**/runtime/tutor
 
-ENV NEOMAKE_DOCKERFILE_UPDATE=2020-01-27
+ENV NEOMAKE_DOCKERFILE_UPDATE=2020-12-17
 
 # Git master in a separate layer, since the above is meant to be stable.
 RUN install_vim -tag master -build \

Makefile

@@ -183,7 +183,7 @@ vimhelplint: | $(if $(VIMHELPLINT_DIR),,build/vimhelplint)
 
 # Run tests in dockerized Vims.
 DOCKER_REPO:=neomake/vims-for-tests
-DOCKER_TAG:=50
+DOCKER_TAG:=51
 NEOMAKE_DOCKER_IMAGE?=
 DOCKER_IMAGE:=$(if $(NEOMAKE_DOCKER_IMAGE),$(NEOMAKE_DOCKER_IMAGE),$(DOCKER_REPO):$(DOCKER_TAG))
 DOCKER_STREAMS:=-ti

tests/compat.vader

@@ -30,7 +30,8 @@ Execute (neomake#compat#systemlist):
       AssertNeomakeMessage "systemlist error: Vim(return):E475: Invalid value for argument cmd: 'echo 1' is not executable.", 0
     endif
   else
-    AssertEqual r, [&shell.': echo 1: command not found']
+    AssertEqual len(r), 1
+    Assert r[0] =~# printf('\v%s(: line 1)?: echo 1: command not found$', &shell), r[0]
     AssertEqual v:shell_error, 127
   endif
 
@@ -46,7 +47,9 @@ Execute (neomake#compat#systemlist):
 Execute (neomake#compat#systemlist with empty args):
   AssertEqual neomake#compat#systemlist(''), []
   AssertEqual neomake#compat#systemlist([]), []
-  AssertEqual neomake#compat#systemlist('0'), [&shell.': 0: command not found']
+  let output = neomake#compat#systemlist('0')
+  AssertEqual len(output), 1
+  Assert output[0] =~# printf('\v%s(: line 1)?: 0: command not found$', &shell), output[0]
 
 Execute (neomake#compat#json_decode):
   AssertEqual neomake#compat#json_decode(''), g:neomake#compat#json_none
@@ -55,7 +58,11 @@ Execute (neomake#compat#json_decode):
   if has('nvim')
     let expected_exception = 'Vim(return):E474: Unidentified byte: success'
   elseif exists('*json_decode')
-    let expected_exception = 'Vim(return):E474: Invalid argument'
+    if has('patch-8.2.0800')
+      let expected_exception = 'Vim(return):E491: json decode error at ''success'''
+    else
+      let expected_exception = 'Vim(return):E474: Invalid argument'
+    endif
   else
     let expected_exception = 'Neomake: Failed to parse JSON input: invalid input'
   endif

tests/customqf.vader

@@ -475,7 +475,7 @@ Execute (neomake#quickfix#FormatQuickfix logs exception from nmcfg)):
     lopen
     AssertEqual getline(1), '????   text1 nmcfg:{undefined}'
     AssertNeomakeMessage 'Error when evaluating nmcfg ({undefined}): Vim(let):E121: Undefined variable: undefined.', 0
-    AssertNeomakeMessage '\v\(in function neomake#quickfix#FormatQuickfix, line \d+\)', 3
+    AssertNeomakeMessage '\v\(in (command line\.\..{-}\.\.)?function neomake#quickfix#FormatQuickfix, line \d+\)$', 3
     lclose
     bwipe
   finally

tests/hooks-queue.vader

@@ -119,18 +119,19 @@ Execute (Does not nest hooks / User autocommands):
   if NeomakeAsyncTestsSetup()
     new
 
+    let s:uses_action_queue = has('nvim') || !has('patch-8.2.0452')
+
     augroup neomake_tests
       autocmd User NeomakeJobFinished nested call s:OnNeomakeJobFinished()
     augroup END
 
     function! s:OnNeomakeJobFinished() abort
       let jobinfo = g:neomake_hook_context.jobinfo
-      if jobinfo.maker.name == 'maker1'
+      if jobinfo.maker.name == 'maker1' && s:uses_action_queue
         NeomakeTestsWaitForMessage '\v^Queuing User autocmd NeomakeCountsChanged for nested invocation ', 3
         AssertNeomakeMessage 'Queuing action handle_hook for Timer, BufEnter, WinEnter, InsertLeave, CursorHold, CursorHoldI.', 3
         AssertNeomakeMessage '\V\^Retrying Timer event in 10ms', 3
       endif
-      " call neomake#log#debug('OnNeomakeJobFinished finished.')
     endfunction
 
     let maker1 = NeomakeTestsCommandMaker('maker1', 'echo error1; exit 1')
@@ -143,9 +144,13 @@ Execute (Does not nest hooks / User autocommands):
     AssertEqual map(getloclist(0), 'v:val.text'), ['error1', 'error2']
 
     doautocmd WinEnter
-    AssertNeomakeMessage 'action queue: calling handle_hook.'
+    if s:uses_action_queue
+      AssertNeomakeMessage 'action queue: calling handle_hook.'
+    endif
     AssertNeomakeMessage '\VCalling User autocmd NeomakeCountsChanged with context: ', 3
-    AssertNeomakeMessage 'action queue: calling CleanJobinfo.', 3
+    if s:uses_action_queue
+      AssertNeomakeMessage 'action queue: calling CleanJobinfo.', 3
+    endif
     AssertNeomakeMessage 'Cleaning jobinfo.', 3
     AssertNeomakeMessage '\VCalling User autocmd NeomakeJobFinished with context: ', 3
     AssertNeomakeMessage '\VCalling User autocmd NeomakeFinished with context: ', 3

tests/log.vader

@@ -37,5 +37,7 @@ Execute (neomake#log#debug throws with missing make_options in tests):
   AssertEqual g:neomake_test_errors, []
   call neomake#log#debug('msg1.', {'make_id': -42})
   AssertEqual len(g:neomake_test_errors), 1
-  Assert g:neomake_test_errors[0] =~# '\v^GetMakeOptions failed: Vim\(let\):E716: Key not present in Dictionary: -42 \(in function .*\)$'
+  Assert g:neomake_test_errors[0] =~#
+  \ '\v^GetMakeOptions failed: Vim\(let\):E716: Key not present in Dictionary: "?-42"?'
+  \ .' \(in .*neomake#GetMakeOptions, line 3\)$'
   let g:neomake_test_errors = []

tests/makers.vader

@@ -612,7 +612,7 @@ Execute (Makers: get_list_entries with exception):
 
   CallNeomake 1, [maker]
   AssertNeomakeMessage 'Error during get_list_entries for unnamed_maker: TEST_ERROR.'
-  AssertNeomakeMessage '\m^(in function .*)$', 3
+  AssertNeomakeMessage '\m^(in .*handle_get_list_entries.*, line 1)$', 3
 
   AssertEqual len(g:neomake_test_countschanged), 0
   AssertEqual len(g:neomake_test_jobfinished), 1
@@ -646,7 +646,7 @@ Execute (Makers: process_json with invalid JSON):
   if has('nvim')
     AssertNeomakeMessage "Failed to decode JSON: Vim(return):E474: Unidentified byte: success (output: 'success').", 0
   elseif exists('*json_decode')
-    AssertNeomakeMessage "Failed to decode JSON: Vim(return):E474: Invalid argument (output: 'success').", 0
+    AssertNeomakeMessage '\vFailed to decode JSON: Vim\(return\):(E474|E491): .* \(output: ''success''\).', 0
   else
     AssertNeomakeMessage "Failed to decode JSON: Failed to parse JSON input: invalid input (output: 'success').", 0
   endif
@@ -690,7 +690,7 @@ Execute (Makers: get_list_entries with sandbox exception):
 
   let jobinfo = neomake#Make({'enabled_makers': [maker]})[0]
   AssertNeomakeMessage '\mError during pcall: Vim(bprevious):E48: Not allowed in sandbox:'
-  AssertNeomakeMessage '\m^(in function .*)$', 3
+  AssertNeomakeMessage '\m^(in .*_handle_get_list_entries.*)$', 3
   AssertEqual len(g:neomake_test_countschanged), 0
   AssertEqual len(g:neomake_test_jobfinished), 0
   AssertEqual len(g:neomake_test_finished), 0
@@ -734,7 +734,7 @@ Execute (pcall aborts after 3 attempts per job):
   " Calls both makers initially.
   AssertNeomakeMessage 'maker1: getting entries via get_list_entries.'
   AssertNeomakeMessage '\mError during pcall: Vim(bprevious):E48: Not allowed in sandbox:', 3, jobs[0]
-  AssertNeomakeMessage '\m^(in function .*)$', 3
+  AssertNeomakeMessage '\m^(in .*_handle_get_list_entries.*)$', 3
   if has('timers')
     AssertNeomakeMessage '\V\^Retrying Timer event in 10ms', 3, jobs[0]
   else
@@ -771,7 +771,7 @@ Execute (pcall aborts after 3 attempts per job):
   endif
 
   NeomakeTestsWaitForMessage 'Giving up handling Timer callbacks after 3 attempts. Please report this. See :messages for more information.', 0, jobs[0]
-  AssertNeomakeMessage '\m^(in function .*neomake#action_queue#add,.*)$', 3
+  AssertNeomakeMessage '\m^(in .*neomake#action_queue#add,.*)$', 3
 
   " Now maker2 gets processed (after giving up on maker1).
   if has('timers')
@@ -786,7 +786,7 @@ Execute (pcall aborts after 3 attempts per job):
   AssertEqual map(getloclist(0), 'v:val.text'), ['error']
 
   let vim_msgs = NeomakeTestsGetVimMessages()
-  Assert vim_msgs[-1] =~# '\vNeomake error in: function .*neomake#action_queue#add, line \d+'
+  Assert vim_msgs[-1] =~# '\vNeomake error in: .*neomake#action_queue#add, line \d+'
   AssertEqual len(vim_msgs), 1
   bwipe
 

tests/postprocess.vader

@@ -405,7 +405,7 @@ Execute (Hook context gets cleaned on error in postprocess):
   NeomakeTestsWaitForFinishedJobs
   AssertNeomakeMessage 'Error during output processing for error-maker: error from postprocess.', 0
   let vim_msgs = NeomakeTestsGetVimMessages()
-  Assert vim_msgs[-1] =~# 'Neomake error in: function .*', 'message found'
+  Assert vim_msgs[-1] =~# 'Neomake error in: .*_AddExprCallback.*, line 1'
   AssertEqual len(vim_msgs), 1
   Assert !exists('g:neomake_postprocess_context'), 'Hook context was cleaned.'
 

tests/processing.vader

@@ -363,6 +363,7 @@ Execute (Sleep in postprocess gets handled correctly):
   " Reproduces flakiness with https://github.com/neomake/neomake/issues/899.
   call neomake#statusline#ResetCounts()
   if NeomakeAsyncTestsSetup()
+    let uses_delayed_exit = !has('nvim-0.2.0') && !has('patch-8.2.0452')
     let s:postprocess_count = 0
     function! s:postprocess(entry) dict
       let s:postprocess_count += 1
@@ -380,15 +381,16 @@ Execute (Sleep in postprocess gets handled correctly):
     let jobinfo = neomake#Make(1, [maker])[0]
     NeomakeTestsWaitForFinishedJobs
     AssertNeomakeMessage 'Processing 1 lines of output.', 3, jobinfo
-    if !has('nvim-0.2.0')
+    if uses_delayed_exit
       AssertNeomakeMessage 'exit (delayed): unnamed_maker: 0.', 3, jobinfo
       AssertNeomakeMessage '\VCalling User autocmd NeomakeCountsChanged with context:', 3
     endif
 
-    if has('nvim-0.4.0')
+    if has('nvim-0.4.0') || (!has('nvim') && has('patch-8.2.0452') && !has('patch-8.2.0466'))
       " Neovim refactored event processing, so that the job's 2nd sleep finishes
       " before the one in the first postprocessing.
       " https://github.com/neovim/neovim/commit/d4938743e6aef04c83d02907048768d0d79aaa30
+      " Vim showed the same behavior for a small patch range.
       let expected_final_countchanges = 2
       AssertNeomakeMessage "output on stdout: ['out-22', 'out-333', ''].", 3, jobinfo
       AssertNeomakeMessage 'Processing 2 lines of output.', 3, jobinfo
@@ -403,8 +405,10 @@ Execute (Sleep in postprocess gets handled correctly):
       AssertNeomakeMessage '\VCalling User autocmd NeomakeCountsChanged with context:', 3
     endif
 
-    if !has('nvim-0.2.0')
+    if uses_delayed_exit
       AssertNeomakeMessage 'Trigger delayed exit.', 3, jobinfo
+    else
+      AssertNeomakeMessage 'exit: unnamed_maker: 0.', 3, jobinfo
     endif
 
     AssertEqual map(getloclist(0), 'v:val.text'), ['out-1', 'out-22', 'out-333']