Skip to content

Commit cd929f7

Browse files
committed
patch 8.1.0633: crash when out of memory while opening a terminal window
Problem: Crash when out of memory while opening a terminal window. Solution: Handle out-of-memory more gracefully.
1 parent 7a2d989 commit cd929f7

File tree

5 files changed

+70
-20
lines changed

5 files changed

+70
-20
lines changed

Diff for: src/libvterm/src/state.c

+8
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ static VTermState *vterm_state_new(VTerm *vt)
5353
{
5454
VTermState *state = vterm_allocator_malloc(vt, sizeof(VTermState));
5555

56+
if (state == NULL)
57+
return NULL;
5658
state->vt = vt;
5759

5860
state->rows = vt->rows;
@@ -1693,13 +1695,19 @@ static const VTermParserCallbacks parser_callbacks = {
16931695
on_resize /* resize */
16941696
};
16951697

1698+
/*
1699+
* Return the existing state or create a new one.
1700+
* Returns NULL when out of memory.
1701+
*/
16961702
VTermState *vterm_obtain_state(VTerm *vt)
16971703
{
16981704
VTermState *state;
16991705
if(vt->state)
17001706
return vt->state;
17011707

17021708
state = vterm_state_new(vt);
1709+
if (state == NULL)
1710+
return NULL;
17031711
vt->state = state;
17041712

17051713
state->combine_chars_size = 16;

Diff for: src/libvterm/src/termscreen.c

+16-12
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include "vterm_internal.h"
22

3+
/* vim: set sw=2 : */
34
#include <stdio.h>
45
#include <string.h>
56

@@ -95,8 +96,7 @@ static ScreenCell *realloc_buffer(VTermScreen *screen, ScreenCell *buffer, int n
9596
}
9697
}
9798

98-
if(buffer)
99-
vterm_allocator_free(screen->vt, buffer);
99+
vterm_allocator_free(screen->vt, buffer);
100100

101101
return new_buffer;
102102
}
@@ -518,8 +518,7 @@ static int resize(int new_rows, int new_cols, VTermPos *delta, void *user)
518518
screen->rows = new_rows;
519519
screen->cols = new_cols;
520520

521-
if(screen->sb_buffer)
522-
vterm_allocator_free(screen->vt, screen->sb_buffer);
521+
vterm_allocator_free(screen->vt, screen->sb_buffer);
523522

524523
screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
525524

@@ -619,16 +618,21 @@ static VTermStateCallbacks state_cbs = {
619618
&setlineinfo /* setlineinfo */
620619
};
621620

621+
/*
622+
* Allocate a new screen and return it.
623+
* Return NULL when out of memory.
624+
*/
622625
static VTermScreen *screen_new(VTerm *vt)
623626
{
624627
VTermState *state = vterm_obtain_state(vt);
625628
VTermScreen *screen;
626629
int rows, cols;
627630

628-
if(!state)
631+
if (state == NULL)
629632
return NULL;
630-
631633
screen = vterm_allocator_malloc(vt, sizeof(VTermScreen));
634+
if (screen == NULL)
635+
return NULL;
632636

633637
vterm_get_size(vt, &rows, &cols);
634638

@@ -646,10 +650,13 @@ static VTermScreen *screen_new(VTerm *vt)
646650
screen->cbdata = NULL;
647651

648652
screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols);
649-
650653
screen->buffer = screen->buffers[0];
651-
652654
screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols);
655+
if (screen->buffer == NULL || screen->sb_buffer == NULL)
656+
{
657+
vterm_screen_free(screen);
658+
return NULL;
659+
}
653660

654661
vterm_state_set_callbacks(screen->state, &state_cbs, screen);
655662

@@ -659,11 +666,8 @@ static VTermScreen *screen_new(VTerm *vt)
659666
INTERNAL void vterm_screen_free(VTermScreen *screen)
660667
{
661668
vterm_allocator_free(screen->vt, screen->buffers[0]);
662-
if(screen->buffers[1])
663-
vterm_allocator_free(screen->vt, screen->buffers[1]);
664-
669+
vterm_allocator_free(screen->vt, screen->buffers[1]);
665670
vterm_allocator_free(screen->vt, screen->sb_buffer);
666-
667671
vterm_allocator_free(screen->vt, screen);
668672
}
669673

Diff for: src/libvterm/src/vterm.c

+19-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#define DEFINE_INLINES
22

3+
/* vim: set sw=2 : */
34
#include "vterm_internal.h"
45

56
#include <stdio.h>
@@ -41,6 +42,8 @@ VTerm *vterm_new_with_allocator(int rows, int cols, VTermAllocatorFunctions *fun
4142
/* Need to bootstrap using the allocator function directly */
4243
VTerm *vt = (*funcs->malloc)(sizeof(VTerm), allocdata);
4344

45+
if (vt == NULL)
46+
return NULL;
4447
vt->allocator = funcs;
4548
vt->allocdata = allocdata;
4649

@@ -55,10 +58,21 @@ VTerm *vterm_new_with_allocator(int rows, int cols, VTermAllocatorFunctions *fun
5558
vt->parser.strbuffer_len = 500; /* should be able to hold an OSC string */
5659
vt->parser.strbuffer_cur = 0;
5760
vt->parser.strbuffer = vterm_allocator_malloc(vt, vt->parser.strbuffer_len);
61+
if (vt->parser.strbuffer == NULL)
62+
{
63+
vterm_allocator_free(vt, vt);
64+
return NULL;
65+
}
5866

5967
vt->outbuffer_len = 200;
6068
vt->outbuffer_cur = 0;
6169
vt->outbuffer = vterm_allocator_malloc(vt, vt->outbuffer_len);
70+
if (vt->outbuffer == NULL)
71+
{
72+
vterm_allocator_free(vt, vt->parser.strbuffer);
73+
vterm_allocator_free(vt, vt);
74+
return NULL;
75+
}
6276

6377
return vt;
6478
}
@@ -82,9 +96,13 @@ INTERNAL void *vterm_allocator_malloc(VTerm *vt, size_t size)
8296
return (*vt->allocator->malloc)(size, vt->allocdata);
8397
}
8498

99+
/*
100+
* Free "ptr" unless it is NULL.
101+
*/
85102
INTERNAL void vterm_allocator_free(VTerm *vt, void *ptr)
86103
{
87-
(*vt->allocator->free)(ptr, vt->allocdata);
104+
if (ptr)
105+
(*vt->allocator->free)(ptr, vt->allocdata);
88106
}
89107

90108
void vterm_get_size(const VTerm *vt, int *rowsp, int *colsp)

Diff for: src/terminal.c

+25-7
Original file line numberDiff line numberDiff line change
@@ -3430,6 +3430,7 @@ set_vterm_palette(VTerm *vterm, long_u *rgb)
34303430
{
34313431
int index = 0;
34323432
VTermState *state = vterm_obtain_state(vterm);
3433+
34333434
for (; index < 16; index++)
34343435
{
34353436
VTermColor color;
@@ -3703,8 +3704,9 @@ static VTermAllocatorFunctions vterm_allocator = {
37033704

37043705
/*
37053706
* Create a new vterm and initialize it.
3707+
* Return FAIL when out of memory.
37063708
*/
3707-
static void
3709+
static int
37083710
create_vterm(term_T *term, int rows, int cols)
37093711
{
37103712
VTerm *vterm;
@@ -3714,15 +3716,26 @@ create_vterm(term_T *term, int rows, int cols)
37143716

37153717
vterm = vterm_new_with_allocator(rows, cols, &vterm_allocator, NULL);
37163718
term->tl_vterm = vterm;
3719+
if (vterm == NULL)
3720+
return FAIL;
3721+
3722+
// Allocate screen and state here, so we can bail out if that fails.
3723+
state = vterm_obtain_state(vterm);
37173724
screen = vterm_obtain_screen(vterm);
3725+
if (state == NULL || screen == NULL)
3726+
{
3727+
vterm_free(vterm);
3728+
return FAIL;
3729+
}
3730+
37183731
vterm_screen_set_callbacks(screen, &screen_callbacks, term);
37193732
/* TODO: depends on 'encoding'. */
37203733
vterm_set_utf8(vterm, 1);
37213734

37223735
init_default_colors(term);
37233736

37243737
vterm_state_set_default_colors(
3725-
vterm_obtain_state(vterm),
3738+
state,
37263739
&term->tl_default_color.fg,
37273740
&term->tl_default_color.bg);
37283741

@@ -3746,9 +3759,10 @@ create_vterm(term_T *term, int rows, int cols)
37463759
#else
37473760
value.boolean = 0;
37483761
#endif
3749-
state = vterm_obtain_state(vterm);
37503762
vterm_state_set_termprop(state, VTERM_PROP_CURSORBLINK, &value);
37513763
vterm_state_set_unrecognised_fallbacks(state, &parser_fallbacks, term);
3764+
3765+
return OK;
37523766
}
37533767

37543768
/*
@@ -5629,7 +5643,8 @@ term_and_job_init(
56295643
vim_free(cwd_wchar);
56305644
vim_free(env_wchar);
56315645

5632-
create_vterm(term, term->tl_rows, term->tl_cols);
5646+
if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL)
5647+
goto failed;
56335648

56345649
#if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS)
56355650
if (opt->jo_set2 & JO2_ANSI_COLORS)
@@ -5710,7 +5725,8 @@ create_pty_only(term_T *term, jobopt_T *options)
57105725
char in_name[80], out_name[80];
57115726
channel_T *channel = NULL;
57125727

5713-
create_vterm(term, term->tl_rows, term->tl_cols);
5728+
if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL)
5729+
return FAIL;
57145730

57155731
vim_snprintf(in_name, sizeof(in_name), "\\\\.\\pipe\\vim-%d-in-%d",
57165732
GetCurrentProcessId(),
@@ -5822,7 +5838,8 @@ term_and_job_init(
58225838
jobopt_T *opt,
58235839
jobopt_T *orig_opt UNUSED)
58245840
{
5825-
create_vterm(term, term->tl_rows, term->tl_cols);
5841+
if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL)
5842+
return FAIL;
58265843

58275844
#if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS)
58285845
if (opt->jo_set2 & JO2_ANSI_COLORS)
@@ -5844,7 +5861,8 @@ term_and_job_init(
58445861
static int
58455862
create_pty_only(term_T *term, jobopt_T *opt)
58465863
{
5847-
create_vterm(term, term->tl_rows, term->tl_cols);
5864+
if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL)
5865+
return FAIL;
58485866

58495867
term->tl_job = job_alloc();
58505868
if (term->tl_job == NULL)

Diff for: src/version.c

+2
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,8 @@ static char *(features[]) =
799799

800800
static int included_patches[] =
801801
{ /* Add new patch number below this line */
802+
/**/
803+
633,
802804
/**/
803805
632,
804806
/**/

0 commit comments

Comments
 (0)