Dereferencing kernel struct data not working with v0.17.0 despite having entry in BTF

[SSapkal]

Trying to test BTF support in v0.17.0, I noticed I was unable to get the nr_running of the 'struct rq'. I am able to get the same with v0.16.0.

What reproduces the bug?

With bpftrace v0.17.0

# ./bpftrace -e 'tracepoint:sched:sched_wakeup_new { printf("Runq length: %d\n", curtask->se.cfs_rq->rq->nr_running); }'
stdin:1:65-99: ERROR: Struct/union of type 'struct rq' does not contain a field named 'nr_running'
tracepoint:sched:sched_wakeup_new { printf("Runq length: %d\n", curtask->se.cfs_rq->rq->nr_running); }

With bpftrace v0.16.0

# ./bpftrace -e 'tracepoint:sched:sched_wakeup_new { printf("Runq length: %d\n", curtask->se.cfs_rq->nr_running); }'
Attaching 1 probe...
Runq length: 1
Runq length: 1
Runq length: 1

Field is exposed by BTF (Same for both v0.16.0 and v.17.0)

# ./bpftrace -lv 'struct rq'
struct rq {
        raw_spinlock_t __lock;
        **unsigned int nr_running;**
        unsigned int nr_numa_running;
        unsigned int nr_preferred_running;
        unsigned int numa_migrate_on;
        long unsigned int last_blocked_load_update_tick;
        unsigned int has_blocked_load;
        long: 64;
        long: 64;
        long: 64;
        call_single_data_t nohz_csd;
        unsigned int nohz_tick_stopped;
        atomic_t nohz_flags;
        unsigned int ttwu_pending;
        u64 nr_switches;
        long: 64;
        struct uclamp_rq uclamp[2];
        unsigned int uclamp_flags;
        long: 64;
        long: 64;
        long: 64;
        struct cfs_rq cfs;
        struct rt_rq rt;
        struct dl_rq dl;
        struct list_head leaf_cfs_rq_list;
        struct list_head *tmp_alone_branch;
        unsigned int nr_uninterruptible;
        struct task_struct *curr;
        struct task_struct *idle;
        struct task_struct *stop;
        long unsigned int next_balance;
        struct mm_struct *prev_mm;
        unsigned int clock_update_flags;
        u64 clock;
        long: 64;
        long: 64;
        long: 64;
        long: 64;
        long: 64;
        u64 clock_task;
        u64 clock_pelt;
        long unsigned int lost_idle_time;
        atomic_t nr_iowait;
        u64 last_seen_need_resched_ns;
        int ticks_without_resched;
        int membarrier_state;
        struct root_domain *rd;
        struct sched_domain *sd;
        long unsigned int cpu_capacity;
        long unsigned int cpu_capacity_orig;
        struct callback_head *balance_callback;
        unsigned char nohz_idle_balance;
        unsigned char idle_balance;
        long unsigned int misfit_task_load;
        int active_balance;
        int push_cpu;
        struct cpu_stop_work active_balance_work;
        int cpu;
        int online;
        struct list_head cfs_tasks;
        long: 64;
        struct sched_avg avg_rt;
        struct sched_avg avg_dl;
        u64 idle_stamp;
        u64 avg_idle;
        long unsigned int wake_stamp;
        u64 wake_avg_idle;
        u64 max_idle_balance_cost;
        struct rcuwait hotplug_wait;
        u64 prev_steal_time;
        long unsigned int calc_load_update;
        long int calc_load_active;
        long: 64;
        long: 64;
        long: 64;
        call_single_data_t hrtick_csd;
        struct hrtimer hrtick_timer;
        ktime_t hrtick_time;
        struct sched_info rq_sched_info;
        long long unsigned int rq_cpu_time;
        unsigned int yld_count;
        unsigned int sched_count;
        unsigned int sched_goidle;
        unsigned int ttwu_count;
        unsigned int ttwu_local;
        struct cpuidle_state *idle_state;
        unsigned int nr_pinned;
        unsigned int push_busy;
        struct cpu_stop_work push_work;
        struct rq *core;
        struct task_struct *core_pick;
        unsigned int core_enabled;
        unsigned int core_sched_seq;
        struct rb_root core_tree;
        unsigned int core_task_seq;
        unsigned int core_pick_seq;
        long unsigned int core_cookie;
        unsigned char core_forceidle;
        unsigned int core_forceidle_seq;
};

#cat /boot/config-$(uname -r) | grep -E 'BPF|BTF|PROBE'

CONFIG_GENERIC_IRQ_PROBE=y
CONFIG_BPF=y
CONFIG_HAVE_EBPF_JIT=y
CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
# BPF subsystem
CONFIG_BPF_SYSCALL=y
CONFIG_BPF_JIT=y
CONFIG_BPF_JIT_ALWAYS_ON=y
CONFIG_BPF_JIT_DEFAULT_ON=y
CONFIG_BPF_UNPRIV_DEFAULT_OFF=y
# CONFIG_BPF_PRELOAD is not set
CONFIG_BPF_LSM=y
# end of BPF subsystem
CONFIG_CGROUP_BPF=y
CONFIG_ARCH_SUPPORTS_UPROBES=y
CONFIG_ARCH_MEMORY_PROBE=y
CONFIG_KPROBES=y
CONFIG_OPTPROBES=y
CONFIG_KPROBES_ON_FTRACE=y
CONFIG_UPROBES=y
CONFIG_KRETPROBES=y
CONFIG_HAVE_KPROBES=y
CONFIG_HAVE_KRETPROBES=y
CONFIG_HAVE_OPTPROBES=y
CONFIG_HAVE_KPROBES_ON_FTRACE=y
CONFIG_MODPROBE_PATH="/sbin/modprobe"
CONFIG_IPV6_SEG6_BPF=y
CONFIG_NETFILTER_XT_MATCH_BPF=m
CONFIG_BPFILTER=y
CONFIG_BPFILTER_UMH=m
CONFIG_NET_CLS_BPF=m
CONFIG_NET_ACT_BPF=m
CONFIG_BPF_STREAM_PARSER=y
CONFIG_LWTUNNEL_BPF=y
# CONFIG_TEST_ASYNC_DRIVER_PROBE is not set
CONFIG_GENERIC_CPU_AUTOPROBE=y
CONFIG_MTD_JEDECPROBE=m
CONFIG_MTD_GEN_PROBE=m
# CONFIG_MTD_NAND_DISKONCHIP_PROBE_ADVANCED is not set
CONFIG_MTD_NAND_DISKONCHIP_PROBE_ADDRESS=0
CONFIG_MTD_QINFO_PROBE=m
CONFIG_VIDEO_SONY_BTF_MPX=m
CONFIG_DRM_I915_FORCE_PROBE=""
# CONFIG_SND_SOC_SOF_DEBUG_PROBES is not set
CONFIG_SND_SOC_SOF_PROBE_WORK_QUEUE=y
CONFIG_DEBUG_INFO_BTF=y
CONFIG_PAHOLE_HAS_SPLIT_BTF=y
CONFIG_DEBUG_INFO_BTF_MODULES=y
CONFIG_KPROBE_EVENTS=y
# CONFIG_KPROBE_EVENTS_ON_NOTRACE is not set
CONFIG_UPROBE_EVENTS=y
CONFIG_BPF_EVENTS=y
CONFIG_PROBE_EVENTS=y
CONFIG_BPF_KPROBE_OVERRIDE=y
# CONFIG_KPROBE_EVENT_GEN_TEST is not set
# CONFIG_SAMPLE_KPROBES is not set
# CONFIG_KPROBES_SANITY_TEST is not set
CONFIG_TEST_BPF=m

bpftrace --info (v0.16.0)

System
  OS: Linux 5.15.0-57-generic #63-Ubuntu SMP Thu Nov 24 13:43:17 UTC 2022
  Arch: x86_64

Build
  version: v0.16.0
  LLVM: 12.0.0
  ORC: v2
  foreach_sym: yes
  unsafe uprobe: no
  bfd: yes
  bcc_usdt_addsem: yes
  bcc bpf_attach_uprobe refcount: yes
  bcc library path resolution: yes
  libbpf btf dump: yes
  libbpf btf dump type decl: yes
  libbpf bpf_prog_load: no
  libbpf bpf_map_create: no
  libdw (DWARF support): no

Kernel helpers
  probe_read: yes
  probe_read_str: yes
  probe_read_user: yes
  probe_read_user_str: yes
  probe_read_kernel: yes
  probe_read_kernel_str: yes
  get_current_cgroup_id: yes
  send_signal: yes
  override_return: yes
  get_boot_ns: yes
  dpath: yes
  skboutput: no

Kernel features
  Instruction limit: 1000000
  Loop support: yes
  btf: yes
  map batch: yes
  uprobe refcount (depends on Build:bcc bpf_attach_uprobe refcount): yes

Map types
  hash: yes
  percpu hash: yes
  array: yes
  percpu array: yes
  stack_trace: yes
  perf_event_array: yes

Probe types
  kprobe: yes
  tracepoint: yes
  perf_event: yes
  kfunc: yes
  iter:task: yes
  iter:task_file: yes
  kprobe_multi: no
  raw_tp_special: yes

bpftrace --info (v0.17.0)

System
  OS: Linux 5.15.0-57-generic #63-Ubuntu SMP Thu Nov 24 13:43:17 UTC 2022
  Arch: x86_64

Build
  version: v0.17.0
  LLVM: 12.0.0
  unsafe uprobe: no
  bfd: yes
  libdw (DWARF support): no

Kernel helpers
  probe_read: yes
  probe_read_str: yes
  probe_read_user: yes
  probe_read_user_str: yes
  probe_read_kernel: yes
  probe_read_kernel_str: yes
  get_current_cgroup_id: yes
  send_signal: yes
  override_return: yes
  get_boot_ns: yes
  dpath: yes
  skboutput: no

Kernel features
  Instruction limit: 1000000
  Loop support: yes
  btf: yes
  map batch: yes
  uprobe refcount (depends on Build:bcc bpf_attach_uprobe refcount): yes

Map types
  hash: yes
  percpu hash: yes
  array: yes
  percpu array: yes
  stack_trace: yes
  perf_event_array: yes

Probe types
  kprobe: yes
  tracepoint: yes
  perf_event: yes
  kfunc: yes
  iter:task: yes
  iter:task_file: yes
  kprobe_multi: no
  raw_tp_special: yes

[viktormalik]

Thanks for reporting this!

The problem is in the way we changed parsing of struct defs in #2315 (we now prefer parsing BTF over Clang-parsing BTF-generated header). Unfortunately, we're not handling structs with bitfields (such as struct task_struct) correctly, which leads to this kind of errors. I opened #2501 to track that.

In the meantime, you can use a workaround with the BPFTRACE_MAX_TYPE_RES_ITERATIONS env var:

# BPFTRACE_MAX_TYPE_RES_ITERATIONS=2 ./bpftrace -e 'tracepoint:sched:sched_wakeup_new { printf("Runq length: %d\n", curtask->se.cfs_rq->rq->nr_running); }'

I'll try to implement a proper fix as soon as possible.