From 9eb993eb6547a92629c423e5ba52c51c329ae128 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Fri, 9 Dec 2022 12:44:04 -0800 Subject: [PATCH 1/4] Use powershell as a backup to handle.exe Fixes #6613 --- .../src/rabbit_mgmt_external_stats.erl | 123 ++++++++---------- 1 file changed, 53 insertions(+), 70 deletions(-) diff --git a/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl b/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl index 4ef3ac3ad929..0e19290c14db 100644 --- a/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl +++ b/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl @@ -37,9 +37,9 @@ fd_total, fhc_stats, node_owners, - last_ts, interval, - error_logged_time + error_logged_time, + fd_warning_logged }). %%-------------------------------------------------------------------- @@ -102,80 +102,38 @@ get_used_fd({unix, _}, State0) -> end; %% handle.exe can be obtained from -%% https://technet.microsoft.com/en-us/sysinternals/bb896655.aspx - -%% Output looks like: - -%% Handle v3.42 -%% Copyright (C) 1997-2008 Mark Russinovich -%% Sysinternals - www.sysinternals.com -%% -%% Handle type summary: -%% ALPC Port : 2 -%% Desktop : 1 -%% Directory : 1 -%% Event : 108 -%% File : 25 -%% IoCompletion : 3 -%% Key : 7 -%% KeyedEvent : 1 -%% Mutant : 1 -%% Process : 3 -%% Process : 38 -%% Thread : 41 -%% Timer : 3 -%% TpWorkerFactory : 2 -%% WindowStation : 2 -%% Total handles: 238 - -%% Nthandle v4.22 - Handle viewer -%% Copyright (C) 1997-2019 Mark Russinovich -%% Sysinternals - www.sysinternals.com -%% -%% Handle type summary: -%% : 1 -%% : 166 -%% ALPC Port : 11 -%% Desktop : 1 -%% Directory : 2 -%% Event : 226 -%% File : 122 -%% IoCompletion : 8 -%% IRTimer : 6 -%% Key : 42 -%% Mutant : 7 -%% Process : 3 -%% Section : 2 -%% Semaphore : 43 -%% Thread : 36 -%% TpWorkerFactory : 3 -%% WaitCompletionPacket: 25 -%% WindowStation : 2 -%% Total handles: 706 - +%% https://learn.microsoft.com/en-us/sysinternals/downloads/handle %% Note that the "File" number appears to include network sockets too; I assume %% that's the number we care about. Note also that if you omit "-s" you will %% see a list of file handles *without* network sockets. If you then add "-a" %% you will see a list of handles of various types, including network sockets %% shown as file handles to \Device\Afd. - get_used_fd({win32, _}, State0) -> - Handle = rabbit_misc:os_cmd( - "handle.exe /accepteula -s -p " ++ os:getpid() ++ " 2> nul"), - case Handle of - [] -> - State1 = log_fd_error("Could not find handle.exe, please install from sysinternals", [], State0), - {State1, 0}; - _ -> - case find_files_line(string:tokens(Handle, "\r\n")) of - unknown -> - State1 = log_fd_error("handle.exe output did not contain " - "a line beginning with ' File ', unable " - "to determine used file descriptor " - "count: ~tp", [Handle], State0), - {State1, 0}; - UsedFd -> - {State0, UsedFd} + Pid = os:getpid(), + case os:find_executable("handle.exe") of + false -> + State1 = log_fd_warning_once("Could not find handle.exe, using powershell to determine handle count", [], State0), + UsedFd = get_used_fd_via_powershell(Pid), + {State1, UsedFd}; + HandleExe -> + Handle = rabbit_misc:os_cmd("\"" ++ HandleExe ++ "\" /accepteula -s -p " ++ Pid ++ " 2> nul"), + case Handle of + [] -> + State1 = log_fd_warning_once("Could not execute handle.exe, using powershell to determine handle count", [], State0), + UsedFd = get_used_fd_via_powershell(Pid), + {State1, UsedFd}; + _ -> + case find_files_line(string:tokens(Handle, "\r\n")) of + unknown -> + State1 = log_fd_warning_once("handle.exe output did not contain " + "a line beginning with 'File', using " + "powershell to determine used file descriptor " + "count: ~tp", [Handle], State0), + UsedFd = get_used_fd_via_powershell(Pid), + {State1, UsedFd}; + UsedFd -> + {State0, UsedFd} + end end end. @@ -187,6 +145,11 @@ find_files_line([" File " ++ Rest | _T]) -> find_files_line([_H | T]) -> find_files_line(T). +get_used_fd_via_powershell(Pid) -> + PwshExe = find_powershell(), + Handle = rabbit_misc:os_cmd("\"" ++ PwshExe ++ "\" -NoLogo -NoProfile -NonInteractive -Command (Get-Process -Id " ++ Pid ++ ").Handles"), + list_to_integer(string:trim(Handle)). + -define(SAFE_CALL(Fun, NoProcFailResult), try Fun @@ -199,6 +162,13 @@ get_disk_free_limit() -> ?SAFE_CALL(rabbit_disk_monitor:get_disk_free_limit(), get_disk_free() -> ?SAFE_CALL(rabbit_disk_monitor:get_disk_free(), disk_free_monitoring_disabled). +log_fd_warning_once(Fmt, Args, #state{fd_warning_logged = undefined}=State) -> + % no warning has been logged, so log it and make a note of when + ok = rabbit_log:warning(Fmt, Args), + State#state{fd_warning_logged = true}; +log_fd_warning_once(_Fmt, _Args, #state{fd_warning_logged = true}=State) -> + State. + log_fd_error(Fmt, Args, #state{error_logged_time = undefined}=State) -> % rabbitmq/rabbitmq-management#90 % no errors have been logged, so log it and make a note of when @@ -460,3 +430,16 @@ get_fhc_stats() -> get_ra_io_metrics() -> lists:sort(ets:tab2list(ra_io_metrics)). + +find_powershell() -> + case os:find_executable("pwsh.exe") of + false -> + case os:find_executable("powershell.exe") of + false -> + "powershell.exe"; + PowershellExe -> + PowershellExe + end; + PwshExe -> + PwshExe + end. From f7b65abc1592dde817b1fc263a6ed908f4c34d55 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Fri, 9 Dec 2022 19:13:40 -0800 Subject: [PATCH 2/4] Execute powershell directly --- deps/rabbit_common/src/rabbit_misc.erl | 110 +++++++++++++++--- .../src/rabbit_mgmt_external_stats.erl | 19 +-- 2 files changed, 97 insertions(+), 32 deletions(-) diff --git a/deps/rabbit_common/src/rabbit_misc.erl b/deps/rabbit_common/src/rabbit_misc.erl index bd08af29ab9b..bf43ac59af0e 100644 --- a/deps/rabbit_common/src/rabbit_misc.erl +++ b/deps/rabbit_common/src/rabbit_misc.erl @@ -62,7 +62,7 @@ -export([pget/2, pget/3, pupdate/3, pget_or_die/2, pmerge/3, pset/3, plmerge/2]). -export([format_message_queue/2]). -export([append_rpc_all_nodes/4, append_rpc_all_nodes/5]). --export([os_cmd/1]). +-export([os_cmd/1, pwsh_cmd/1]). -export([is_os_process_alive/1]). -export([version/0, otp_release/0, platform_and_version/0, otp_system_version/0, rabbitmq_and_erlang_versions/0, which_applications/0]). @@ -1156,6 +1156,14 @@ os_cmd(Command) -> end end. +pwsh_cmd(Command) -> + case os:type() of + {win32, _} -> + do_pwsh_cmd(Command); + _ -> + {error, invalid_os_type} + end. + is_os_process_alive(Pid) -> with_os([{unix, fun () -> run_ps(Pid) =:= 0 @@ -1164,26 +1172,17 @@ is_os_process_alive(Pid) -> PidS = rabbit_data_coercion:to_list(Pid), case os:find_executable("tasklist.exe") of false -> - Cmd = - format( - "powershell.exe -NoLogo -NoProfile -NonInteractive -Command " - "\"(Get-Process -Id ~ts).ProcessName\"", - [PidS]), - Res = - os_cmd(Cmd ++ " 2>&1") -- [$\r, $\n], + Cmd = format("(Get-Process -Id ~ts).ProcessName", [PidS]), + {ok, Res} = pwsh_cmd(Cmd), case Res of "erl" -> true; "werl" -> true; _ -> false end; - _ -> - Cmd = - "tasklist /nh /fi " - "\"pid eq " ++ PidS ++ "\"", - Res = os_cmd(Cmd ++ " 2>&1"), - match =:= re:run(Res, - "erl\\.exe", - [{capture, none}]) + TasklistExe -> + Args = ["/nh", "/fi", "pid eq " ++ PidS], + {ok, Res} = do_win32_cmd(TasklistExe, Args), + match =:= re:run(Res, "erl\\.exe", [{capture, none}]) end end}]). @@ -1472,3 +1471,82 @@ whereis_name(Name) -> %% End copypasta from gen_server2.erl %% ------------------------------------------------------------------------- + +%% This will execute a Powershell command that is expected to return a +%% single-line result. Output lines can't exceed 512 bytes. If multiple +%% lines are returned by the command, these functions will return the +%% last line. +%% +%% Inspired by os:cmd/1 in lib/kernel/src/os.erl +do_pwsh_cmd(Command) -> + Pwsh = find_powershell(), + A1 = ["-NoLogo", + "-NonInteractive", + "-NoProfile", + "-InputFormat", "Text", + "-OutputFormat", "Text", + "-Command", Command], + do_win32_cmd(Pwsh, A1). + +do_win32_cmd(Exe, Args) -> + SystemRootDir = os:getenv("SystemRoot", "/"), + % Note: 'hide' must be used or this will not work! + A0 = [exit_status, stderr_to_stdout, in, hide, + {cd, SystemRootDir}, {line, 512}, {arg0, Exe}, {args, Args}], + Port = erlang:open_port({spawn_executable, Exe}, A0), + MonRef = erlang:monitor(port, Port), + Result = win32_cmd_receive(Port, MonRef, <<>>), + true = erlang:demonitor(MonRef, [flush]), + Result. + +win32_cmd_receive(Port, MonRef, Data0) -> + receive + {Port, {exit_status, 0}} -> + win32_cmd_receive_finish(Port, MonRef), + {ok, Data0}; + {Port, {exit_status, Status}} -> + win32_cmd_receive_finish(Port, MonRef), + {error, {exit_status, Status}}; + {Port, {data, {eol, Data1}}} -> + Data2 = string:trim(Data1), + win32_cmd_receive(Port, MonRef, Data2); + {'DOWN', MonRef, _, _, _} -> + flush_exit(Port), + {error, nodata} + after 5000 -> + {error, timeout} + end. + +win32_cmd_receive_finish(Port, MonRef) -> + catch erlang:port_close(Port), + flush_until_down(Port, MonRef). + +flush_until_down(Port, MonRef) -> + receive + {Port, {data, _Bytes}} -> + flush_until_down(Port, MonRef); + {'DOWN', MonRef, _, _, _} -> + flush_exit(Port) + after 500 -> + flush_exit(Port) + end. + +flush_exit(Port) -> + receive + {'EXIT', Port, _} -> ok + after 0 -> + ok + end. + +find_powershell() -> + case os:find_executable("pwsh.exe") of + false -> + case os:find_executable("powershell.exe") of + false -> + "powershell.exe"; + PowershellExe -> + PowershellExe + end; + PwshExe -> + PwshExe + end. diff --git a/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl b/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl index 0e19290c14db..8a2b765f2b11 100644 --- a/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl +++ b/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl @@ -146,9 +146,9 @@ find_files_line([_H | T]) -> find_files_line(T). get_used_fd_via_powershell(Pid) -> - PwshExe = find_powershell(), - Handle = rabbit_misc:os_cmd("\"" ++ PwshExe ++ "\" -NoLogo -NoProfile -NonInteractive -Command (Get-Process -Id " ++ Pid ++ ").Handles"), - list_to_integer(string:trim(Handle)). + Cmd = "Get-Process -Id " ++ Pid ++ " | Select-Object -ExpandProperty HandleCount", + {ok, Result} = rabbit_misc:pwsh_cmd(Cmd), + list_to_integer(Result). -define(SAFE_CALL(Fun, NoProcFailResult), try @@ -430,16 +430,3 @@ get_fhc_stats() -> get_ra_io_metrics() -> lists:sort(ets:tab2list(ra_io_metrics)). - -find_powershell() -> - case os:find_executable("pwsh.exe") of - false -> - case os:find_executable("powershell.exe") of - false -> - "powershell.exe"; - PowershellExe -> - PowershellExe - end; - PwshExe -> - PwshExe - end. From 10fcc64b7439a536c24d41d43b1e518dfd81c3bd Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Sat, 10 Dec 2022 14:20:02 -0800 Subject: [PATCH 3/4] Use win32_cmd/2 to run handle.exe because it is great. --- deps/rabbit_common/src/rabbit_misc.erl | 46 ++++++++++--------- .../src/rabbit_mgmt_external_stats.erl | 11 +++-- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/deps/rabbit_common/src/rabbit_misc.erl b/deps/rabbit_common/src/rabbit_misc.erl index bf43ac59af0e..aadf485ddfe6 100644 --- a/deps/rabbit_common/src/rabbit_misc.erl +++ b/deps/rabbit_common/src/rabbit_misc.erl @@ -62,7 +62,7 @@ -export([pget/2, pget/3, pupdate/3, pget_or_die/2, pmerge/3, pset/3, plmerge/2]). -export([format_message_queue/2]). -export([append_rpc_all_nodes/4, append_rpc_all_nodes/5]). --export([os_cmd/1, pwsh_cmd/1]). +-export([os_cmd/1, pwsh_cmd/1, win32_cmd/2]). -export([is_os_process_alive/1]). -export([version/0, otp_release/0, platform_and_version/0, otp_system_version/0, rabbitmq_and_erlang_versions/0, which_applications/0]). @@ -1173,7 +1173,7 @@ is_os_process_alive(Pid) -> case os:find_executable("tasklist.exe") of false -> Cmd = format("(Get-Process -Id ~ts).ProcessName", [PidS]), - {ok, Res} = pwsh_cmd(Cmd), + {ok, [Res]} = pwsh_cmd(Cmd), case Res of "erl" -> true; "werl" -> true; @@ -1181,7 +1181,7 @@ is_os_process_alive(Pid) -> end; TasklistExe -> Args = ["/nh", "/fi", "pid eq " ++ PidS], - {ok, Res} = do_win32_cmd(TasklistExe, Args), + {ok, [Res]} = win32_cmd(TasklistExe, Args), match =:= re:run(Res, "erl\\.exe", [{capture, none}]) end end}]). @@ -1472,44 +1472,46 @@ whereis_name(Name) -> %% End copypasta from gen_server2.erl %% ------------------------------------------------------------------------- -%% This will execute a Powershell command that is expected to return a -%% single-line result. Output lines can't exceed 512 bytes. If multiple -%% lines are returned by the command, these functions will return the -%% last line. +%% This will execute a Powershell command without an intervening cmd.exe +%% process. Output lines can't exceed 512 bytes. %% %% Inspired by os:cmd/1 in lib/kernel/src/os.erl do_pwsh_cmd(Command) -> Pwsh = find_powershell(), - A1 = ["-NoLogo", - "-NonInteractive", - "-NoProfile", - "-InputFormat", "Text", - "-OutputFormat", "Text", - "-Command", Command], - do_win32_cmd(Pwsh, A1). - -do_win32_cmd(Exe, Args) -> + Args = ["-NoLogo", + "-NonInteractive", + "-NoProfile", + "-InputFormat", "Text", + "-OutputFormat", "Text", + "-Command", Command], + win32_cmd(Pwsh, Args). + +win32_cmd(Exe, Args) -> SystemRootDir = os:getenv("SystemRoot", "/"), % Note: 'hide' must be used or this will not work! A0 = [exit_status, stderr_to_stdout, in, hide, {cd, SystemRootDir}, {line, 512}, {arg0, Exe}, {args, Args}], Port = erlang:open_port({spawn_executable, Exe}, A0), MonRef = erlang:monitor(port, Port), - Result = win32_cmd_receive(Port, MonRef, <<>>), + Result = win32_cmd_receive(Port, MonRef, []), true = erlang:demonitor(MonRef, [flush]), Result. -win32_cmd_receive(Port, MonRef, Data0) -> +win32_cmd_receive(Port, MonRef, Acc0) -> receive {Port, {exit_status, 0}} -> win32_cmd_receive_finish(Port, MonRef), - {ok, Data0}; + {ok, lists:reverse(Acc0)}; {Port, {exit_status, Status}} -> win32_cmd_receive_finish(Port, MonRef), {error, {exit_status, Status}}; - {Port, {data, {eol, Data1}}} -> - Data2 = string:trim(Data1), - win32_cmd_receive(Port, MonRef, Data2); + {Port, {data, {eol, Data0}}} -> + Data1 = string:trim(Data0), + Acc1 = case Data1 of + [] -> Acc0; % Note: skip empty lines in output + Data2 -> [Data2 | Acc0] + end, + win32_cmd_receive(Port, MonRef, Acc1); {'DOWN', MonRef, _, _, _} -> flush_exit(Port), {error, nodata} diff --git a/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl b/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl index 8a2b765f2b11..fe25326272be 100644 --- a/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl +++ b/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl @@ -116,19 +116,20 @@ get_used_fd({win32, _}, State0) -> UsedFd = get_used_fd_via_powershell(Pid), {State1, UsedFd}; HandleExe -> - Handle = rabbit_misc:os_cmd("\"" ++ HandleExe ++ "\" /accepteula -s -p " ++ Pid ++ " 2> nul"), - case Handle of + Args = ["/accepteula", "-s", "-p", Pid], + HandleExeOutput = rabbit_misc:win32_cmd(HandleExe, Args), + case HandleExeOutput of [] -> State1 = log_fd_warning_once("Could not execute handle.exe, using powershell to determine handle count", [], State0), UsedFd = get_used_fd_via_powershell(Pid), {State1, UsedFd}; _ -> - case find_files_line(string:tokens(Handle, "\r\n")) of + case find_files_line(HandleExeOutput) of unknown -> State1 = log_fd_warning_once("handle.exe output did not contain " "a line beginning with 'File', using " "powershell to determine used file descriptor " - "count: ~tp", [Handle], State0), + "count: ~tp", [HandleExeOutput], State0), UsedFd = get_used_fd_via_powershell(Pid), {State1, UsedFd}; UsedFd -> @@ -147,7 +148,7 @@ find_files_line([_H | T]) -> get_used_fd_via_powershell(Pid) -> Cmd = "Get-Process -Id " ++ Pid ++ " | Select-Object -ExpandProperty HandleCount", - {ok, Result} = rabbit_misc:pwsh_cmd(Cmd), + {ok, [Result]} = rabbit_misc:pwsh_cmd(Cmd), list_to_integer(Result). -define(SAFE_CALL(Fun, NoProcFailResult), From b067aa612ea148706779fd31c5dcde4ffdcc8349 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Sat, 10 Dec 2022 14:34:23 -0800 Subject: [PATCH 4/4] Handle handle.exe output correctly --- .../src/rabbit_mgmt_external_stats.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl b/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl index fe25326272be..00a7d2053c9b 100644 --- a/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl +++ b/deps/rabbitmq_management_agent/src/rabbit_mgmt_external_stats.erl @@ -117,7 +117,7 @@ get_used_fd({win32, _}, State0) -> {State1, UsedFd}; HandleExe -> Args = ["/accepteula", "-s", "-p", Pid], - HandleExeOutput = rabbit_misc:win32_cmd(HandleExe, Args), + {ok, HandleExeOutput} = rabbit_misc:win32_cmd(HandleExe, Args), case HandleExeOutput of [] -> State1 = log_fd_warning_once("Could not execute handle.exe, using powershell to determine handle count", [], State0), @@ -140,7 +140,10 @@ get_used_fd({win32, _}, State0) -> find_files_line([]) -> unknown; -find_files_line([" File " ++ Rest | _T]) -> +% Note: +% rabbit_misc:win32_cmd trims the output, so there will be no +% leading/trailing whitespace +find_files_line(["File " ++ Rest | _T]) -> [Files] = string:tokens(Rest, ": "), list_to_integer(Files); find_files_line([_H | T]) ->