Skip to content

Use powershell as a backup to handle.exe (backport #6614) (backport #6626) #6627

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Dec 11, 2022
Merged

Use powershell as a backup to handle.exe (backport #6614) (backport #6626) #6627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bcd39ea
Use powershell as a backup to handle.exe
lukebakken Dec 9, 2022
6ccb827
Execute powershell directly
lukebakken Dec 10, 2022
b4e6e4a
Use win32_cmd/2 to run handle.exe because it is great.
lukebakken Dec 10, 2022
c570338
Handle handle.exe output correctly
lukebakken Dec 10, 2022
56c73b6
Resolve a conflict
michaelklishin Dec 11, 2022
da112dc
Resolve a conflict in rabbit_mgmt_external_stats.erl
michaelklishin Dec 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 96 additions & 16 deletions deps/rabbit_common/src/rabbit_misc.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
-export([pget/2, pget/3, pupdate/3, pget_or_die/2, pmerge/3, pset/3, plmerge/2]).
-export([format_message_queue/2]).
-export([append_rpc_all_nodes/4, append_rpc_all_nodes/5]).
-export([os_cmd/1]).
-export([os_cmd/1, pwsh_cmd/1, win32_cmd/2]).
-export([is_os_process_alive/1]).
-export([gb_sets_difference/2]).
-export([version/0, otp_release/0, platform_and_version/0, otp_system_version/0,
Expand Down Expand Up @@ -1155,6 +1155,14 @@ os_cmd(Command) ->
end
end.

pwsh_cmd(Command) ->
case os:type() of
{win32, _} ->
do_pwsh_cmd(Command);
_ ->
{error, invalid_os_type}
end.

is_os_process_alive(Pid) ->
with_os([{unix, fun () ->
run_ps(Pid) =:= 0
Expand All @@ -1163,26 +1171,17 @@ is_os_process_alive(Pid) ->
PidS = rabbit_data_coercion:to_list(Pid),
case os:find_executable("tasklist.exe") of
false ->
Cmd =
format(
"powershell.exe -NoLogo -NoProfile -NonInteractive -Command "
"\"(Get-Process -Id ~s).ProcessName\"",
[PidS]),
Res =
os_cmd(Cmd ++ " 2>&1") -- [$\r, $\n],
Cmd = format("(Get-Process -Id ~ts).ProcessName", [PidS]),
{ok, [Res]} = pwsh_cmd(Cmd),
case Res of
"erl" -> true;
"werl" -> true;
_ -> false
end;
_ ->
Cmd =
"tasklist /nh /fi "
"\"pid eq " ++ PidS ++ "\"",
Res = os_cmd(Cmd ++ " 2>&1"),
match =:= re:run(Res,
"erl\\.exe",
[{capture, none}])
TasklistExe ->
Args = ["/nh", "/fi", "pid eq " ++ PidS],
{ok, [Res]} = win32_cmd(TasklistExe, Args),
match =:= re:run(Res, "erl\\.exe", [{capture, none}])
end
end}]).

Expand Down Expand Up @@ -1456,3 +1455,84 @@ whereis_name(Name) ->

%% End copypasta from gen_server2.erl
%% -------------------------------------------------------------------------

%% This will execute a Powershell command without an intervening cmd.exe
%% process. Output lines can't exceed 512 bytes.
%%
%% Inspired by os:cmd/1 in lib/kernel/src/os.erl
do_pwsh_cmd(Command) ->
Pwsh = find_powershell(),
Args = ["-NoLogo",
"-NonInteractive",
"-NoProfile",
"-InputFormat", "Text",
"-OutputFormat", "Text",
"-Command", Command],
win32_cmd(Pwsh, Args).

win32_cmd(Exe, Args) ->
SystemRootDir = os:getenv("SystemRoot", "/"),
% Note: 'hide' must be used or this will not work!
A0 = [exit_status, stderr_to_stdout, in, hide,
{cd, SystemRootDir}, {line, 512}, {arg0, Exe}, {args, Args}],
Port = erlang:open_port({spawn_executable, Exe}, A0),
MonRef = erlang:monitor(port, Port),
Result = win32_cmd_receive(Port, MonRef, []),
true = erlang:demonitor(MonRef, [flush]),
Result.

win32_cmd_receive(Port, MonRef, Acc0) ->
receive
{Port, {exit_status, 0}} ->
win32_cmd_receive_finish(Port, MonRef),
{ok, lists:reverse(Acc0)};
{Port, {exit_status, Status}} ->
win32_cmd_receive_finish(Port, MonRef),
{error, {exit_status, Status}};
{Port, {data, {eol, Data0}}} ->
Data1 = string:trim(Data0),
Acc1 = case Data1 of
[] -> Acc0; % Note: skip empty lines in output
Data2 -> [Data2 | Acc0]
end,
win32_cmd_receive(Port, MonRef, Acc1);
{'DOWN', MonRef, _, _, _} ->
flush_exit(Port),
{error, nodata}
after 5000 ->
{error, timeout}
end.

win32_cmd_receive_finish(Port, MonRef) ->
catch erlang:port_close(Port),
flush_until_down(Port, MonRef).

flush_until_down(Port, MonRef) ->
receive
{Port, {data, _Bytes}} ->
flush_until_down(Port, MonRef);
{'DOWN', MonRef, _, _, _} ->
flush_exit(Port)
after 500 ->
flush_exit(Port)
end.

flush_exit(Port) ->
receive
{'EXIT', Port, _} -> ok
after 0 ->
ok
end.

find_powershell() ->
case os:find_executable("pwsh.exe") of
false ->
case os:find_executable("powershell.exe") of
false ->
"powershell.exe";
PowershellExe ->
PowershellExe
end;
PwshExe ->
PwshExe
end.
116 changes: 45 additions & 71 deletions deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@
fd_total,
fhc_stats,
node_owners,
last_ts,
interval,
error_logged_time
error_logged_time,
fd_warning_logged
}).

%%--------------------------------------------------------------------
Expand Down Expand Up @@ -102,91 +102,58 @@ get_used_fd({unix, _}, State0) ->
end;

%% handle.exe can be obtained from
%% https://technet.microsoft.com/en-us/sysinternals/bb896655.aspx

%% Output looks like:

%% Handle v3.42
%% Copyright (C) 1997-2008 Mark Russinovich
%% Sysinternals - www.sysinternals.com
%%
%% Handle type summary:
%% ALPC Port : 2
%% Desktop : 1
%% Directory : 1
%% Event : 108
%% File : 25
%% IoCompletion : 3
%% Key : 7
%% KeyedEvent : 1
%% Mutant : 1
%% Process : 3
%% Process : 38
%% Thread : 41
%% Timer : 3
%% TpWorkerFactory : 2
%% WindowStation : 2
%% Total handles: 238

%% Nthandle v4.22 - Handle viewer
%% Copyright (C) 1997-2019 Mark Russinovich
%% Sysinternals - www.sysinternals.com
%%
%% Handle type summary:
%% <Unknown type> : 1
%% <Unknown type> : 166
%% ALPC Port : 11
%% Desktop : 1
%% Directory : 2
%% Event : 226
%% File : 122
%% IoCompletion : 8
%% IRTimer : 6
%% Key : 42
%% Mutant : 7
%% Process : 3
%% Section : 2
%% Semaphore : 43
%% Thread : 36
%% TpWorkerFactory : 3
%% WaitCompletionPacket: 25
%% WindowStation : 2
%% Total handles: 706

%% https://learn.microsoft.com/en-us/sysinternals/downloads/handle
%% Note that the "File" number appears to include network sockets too; I assume
%% that's the number we care about. Note also that if you omit "-s" you will
%% see a list of file handles *without* network sockets. If you then add "-a"
%% you will see a list of handles of various types, including network sockets
%% shown as file handles to \Device\Afd.

get_used_fd({win32, _}, State0) ->
Handle = rabbit_misc:os_cmd(
"handle.exe /accepteula -s -p " ++ os:getpid() ++ " 2> nul"),
case Handle of
[] ->
State1 = log_fd_error("Could not find handle.exe, please install from sysinternals", [], State0),
{State1, 0};
_ ->
case find_files_line(string:tokens(Handle, "\r\n")) of
unknown ->
State1 = log_fd_error("handle.exe output did not contain "
"a line beginning with ' File ', unable "
"to determine used file descriptor "
"count: ~p", [Handle], State0),
{State1, 0};
UsedFd ->
{State0, UsedFd}
Pid = os:getpid(),
case os:find_executable("handle.exe") of
false ->
State1 = log_fd_warning_once("Could not find handle.exe, using powershell to determine handle count", [], State0),
UsedFd = get_used_fd_via_powershell(Pid),
{State1, UsedFd};
HandleExe ->
Args = ["/accepteula", "-s", "-p", Pid],
{ok, HandleExeOutput} = rabbit_misc:win32_cmd(HandleExe, Args),
case HandleExeOutput of
[] ->
State1 = log_fd_warning_once("Could not execute handle.exe, using powershell to determine handle count", [], State0),
UsedFd = get_used_fd_via_powershell(Pid),
{State1, UsedFd};
_ ->
case find_files_line(HandleExeOutput) of
unknown ->
State1 = log_fd_warning_once("handle.exe output did not contain "
"a line beginning with 'File', using "
"powershell to determine used file descriptor "
"count: ~tp", [HandleExeOutput], State0),
UsedFd = get_used_fd_via_powershell(Pid),
{State1, UsedFd};
UsedFd ->
{State0, UsedFd}
end
end
end.

find_files_line([]) ->
unknown;
find_files_line([" File " ++ Rest | _T]) ->
% Note:
% rabbit_misc:win32_cmd trims the output, so there will be no
% leading/trailing whitespace
find_files_line(["File " ++ Rest | _T]) ->
[Files] = string:tokens(Rest, ": "),
list_to_integer(Files);
find_files_line([_H | T]) ->
find_files_line(T).

get_used_fd_via_powershell(Pid) ->
Cmd = "Get-Process -Id " ++ Pid ++ " | Select-Object -ExpandProperty HandleCount",
{ok, [Result]} = rabbit_misc:pwsh_cmd(Cmd),
list_to_integer(Result).

-define(SAFE_CALL(Fun, NoProcFailResult),
try
Fun
Expand All @@ -199,6 +166,13 @@ get_disk_free_limit() -> ?SAFE_CALL(rabbit_disk_monitor:get_disk_free_limit(),
get_disk_free() -> ?SAFE_CALL(rabbit_disk_monitor:get_disk_free(),
disk_free_monitoring_disabled).

log_fd_warning_once(Fmt, Args, #state{fd_warning_logged = undefined}=State) ->
% no warning has been logged, so log it and make a note of when
ok = rabbit_log:warning(Fmt, Args),
State#state{fd_warning_logged = true};
log_fd_warning_once(_Fmt, _Args, #state{fd_warning_logged = true}=State) ->
State.

log_fd_error(Fmt, Args, #state{error_logged_time = undefined}=State) ->
% rabbitmq/rabbitmq-management#90
% no errors have been logged, so log it and make a note of when
Expand Down