CI Failure (NodeCrash) in DataTransformsTest.test_tracked_offsets_cleaned_up

[Lazin]

https://buildkite.com/redpanda/redpanda/builds/44535#018d6097-cb18-4eb6-9d0f-24e28ec59d17/6-8053

test_id:    rptest.tests.data_transforms_test.DataTransformsTest.test_tracked_offsets_cleaned_up
--
  | status:     FAIL
  | run time:   1 minute 9.885 seconds
  |  
  |  
  | <NodeCrash docker-rp-9: ==6115==ERROR: AddressSanitizer: heap-use-after-free on address 0x611000438900 at pc 0x7f06a354ff58 bp 0x7ffc4a0995e0 sp 0x7ffc4a0995d8
  | >
  | Traceback (most recent call last):
  | File "/usr/local/lib/python3.10/dist-packages/ducktape/utils/util.py", line 44, in wait_until
  | if condition():
  | File "/root/tests/rptest/tests/data_transforms_test.py", line 110, in transform_is_gone
  | transforms = self._rpk.list_wasm()
  | File "/root/tests/rptest/clients/rpk.py", line 1649, in list_wasm
  | out = self._run_wasm([
  | File "/root/tests/rptest/clients/rpk.py", line 1630, in _run_wasm
  | return self._execute(cmd)
  | File "/root/tests/rptest/clients/rpk.py", line 1082, in _execute
  | raise RpkException(
  | rptest.clients.rpk.RpkException: RpkException<command /var/lib/buildkite-agent/builds/buildkite-amd64-xfs-builders-i-06454664b47d68853-1/redpanda/redpanda/vbuild/redpanda_installs/ci/bin/rpk transform -X admin.hosts=docker-rp-9:9644,docker-rp-24:9644,docker-rp-4:9644 list --format json --detailed -v returned 1, output: ; stderr: 18:26:41.956  DEBUG  Sending request  {"method": "GET", "url": "http://docker-rp-24:9644/v1/transform/", "bearer": false, "basic": false}
  | 18:26:43.474  WARN  Request error, trying another node: request GET http://docker-rp-24:9644/v1/transform/ failed: Internal Server Error, body: "{\"message\": \"std::runtime_error (failed to generate transform report: Timeout occurred while processing request)\", \"code\": 500}"
  |  
  | 18:26:43.475  DEBUG  Sending request  {"method": "GET", "url": "http://docker-rp-4:9644/v1/transform/", "bearer": false, "basic": false}
  | 18:26:45.003  WARN  Request error, trying another node: request GET http://docker-rp-4:9644/v1/transform/ failed: Internal Server Error, body: "{\"message\": \"std::runtime_error (failed to generate transform report: Timeout occurred while processing request)\", \"code\": 500}"
  |  
  | 18:26:45.003  DEBUG  Sending request  {"method": "GET", "url": "http://docker-rp-9:9644/v1/transform/", "bearer": false, "basic": false}
  | unable to list transforms: Get "http://docker-rp-9:9644/v1/transform/": dial tcp 172.16.32.8:9644: connect: connection refused
  | ; returncode: 1>
  |  
  | The above exception was the direct cause of the following exception:
  |  
  | Traceback (most recent call last):
  | File "/root/tests/rptest/services/cluster.py", line 99, in wrapped
  | r = f(self, *args, **kwargs)
  | File "/root/tests/rptest/tests/data_transforms_test.py", line 229, in test_tracked_offsets_cleaned_up
  | self._delete_wasm(name="identity-xform")
  | File "/root/tests/rptest/tests/data_transforms_test.py", line 116, in _delete_wasm
  | wait_until(
  | File "/usr/local/lib/python3.10/dist-packages/ducktape/utils/util.py", line 57, in wait_until
  | raise TimeoutError(err_msg() if callable(err_msg) else err_msg) from last_exception
  | ducktape.errors.TimeoutError: wasm transform identity-xform was not fully shutdown
  |  
  | During handling of the above exception, another exception occurred:
  |  
  | Traceback (most recent call last):
  | File "/usr/local/lib/python3.10/dist-packages/ducktape/tests/runner_client.py", line 184, in _do_run
  | data = self.run_test()
  | File "/usr/local/lib/python3.10/dist-packages/ducktape/tests/runner_client.py", line 269, in run_test
  | return self.test_context.function(self.test)
  | File "/root/tests/rptest/services/cluster.py", line 120, in wrapped
  | redpanda.raise_on_crash(log_allow_list=log_allow_list)
  | File "/root/tests/rptest/services/redpanda.py", line 2805, in raise_on_crash
  | raise NodeCrash(crashes)
  | rptest.services.utils.NodeCrash: <NodeCrash docker-rp-9: ==6115==ERROR: AddressSanitizer: heap-use-after-free on address 0x611000438900 at pc 0x7f06a354ff58 bp 0x7ffc4a0995e0 sp 0x7ffc4a0995d8
  | >

test_id:    rptest.tests.module.ClassTest.test_methods.if_present=True
status:     FAIL
run time:   4 minutes 21.145 seconds
 
    TimeoutError("Something is wrong")
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/ducktape/tests/runner_client.py", line 135, in run
    data = self.run_test()
  File "/usr/local/lib/python3.10/dist-packages/ducktape/tests/runner_client.py", line 227, in run_test
    return self.test_context.function(self.test)
  File "/usr/local/lib/python3.10/dist-packages/ducktape/mark/_mark.py", line 476, in wrapper
    return functools.partial(f, *args, **kwargs)(*w_args, **w_kwargs)
  File "/root/tests/rptest/utils/mode_checks.py", line 63, in f
    return func(*args, **kwargs)
  File "/root/tests/rptest/services/cluster.py", line 35, in wrapped
    r = f(self, *args, **kwargs)
  File "/root/tests/rptest/tests/module.py", line 105, in test_methods
    raise TimeoutError("Something is wrong")
ducktape.errors.TimeoutError

[rockwotj]

absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> >, 256, false> >::parent() const at ??:?
absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> >, 256, false> >::is_root() const at ??:?
absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> >, 256, false> >::get_root_generation() const at ??:?
absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> >, 256, false> >::generation() const at ??:?
void absl::lts_20230802::container_internal::btree_iterator_generation_info_enabled::assert_valid_generation<absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> >, 256, false> > >(absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> >, 256, false> > const*) const at ??:?
absl::lts_20230802::container_internal::btree_iterator<absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> >, 256, false> >, std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value>&, std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value>*>::increment() at ??:?
absl::lts_20230802::container_internal::btree_iterator<absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> >, 256, false> >, std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value>&, std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value>*>::operator++() at ??:?
addr2line: DWARF error: invalid or unhandled FORM value: 0x23
cluster::distributed_kv_stm<model::transform_offsets_key, model::transform_offsets_value, fixed_string<21ul>{char [21]{(char)116, (char)114, (char)97, (char)110, (char)115, (char)102, (char)111, (char)114, (char)109, (char)95, (char)111, (char)102, (char)102, (char)115, (char)101, (char)116, (char)115, (char)95, (char)115, (char)116, (char)109}}, 1048576ul>::list() [clone .resume] at deps.cc:?
std::__1::coroutine_handle<seastar::internal::coroutine_traits_base<boost::outcome_v2::basic_result<absl::lts_20230802::btree_map<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> > >, cluster::errc, boost::outcome_v2::policy::error_code_throw_as_system_error<absl::lts_20230802::btree_map<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> > >, cluster::errc, void> > >::promise_type>::resume[abi:v160004]() const at client.cc:?
seastar::internal::coroutine_traits_base<boost::outcome_v2::basic_result<absl::lts_20230802::btree_map<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> > >, cluster::errc, boost::outcome_v2::policy::error_code_throw_as_system_error<absl::lts_20230802::btree_map<model::transform_offsets_key, model::transform_offsets_value, std::__1::less<model::transform_offsets_key>, std::__1::allocator<std::__1::pair<model::transform_offsets_key const, model::transform_offsets_value> > >, cluster::errc, void> > >::promise_type::run_and_dispose() at ??:?
addr2line: DWARF error: invalid or unhandled FORM value: 0x23
seastar::reactor::run_tasks(seastar::reactor::task_queue&) at ??:?
seastar::reactor::run_some_tasks() at ??:?
seastar::reactor::do_run() at ??:?
seastar::reactor::run() at ??:?
seastar::app_template::run_deprecated(int, char**, std::__1::function<void ()>&&) at ??:?
seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&) at ??:?
application::run(int, char**) at ??:?
addr2line: DWARF error: invalid or unhandled FORM value: 0x23
main at ??:?
/home/rockwood/Workspace/tmp/share/lib/libc.so.6: ELF 64-bit LSB shared object, x86-64, version 1 (GNU/Linux), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=6e7b96dfb83f0bdcb6a410469b82f86415e5ada3, for GNU/Linux 3.2.0, not stripped