Support parsing bitfields from BTF/DWARF

.github/workflows/codeql.yml

@@ -33,7 +33,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get remove 'llvm-*' 'clang-*' 'libclang-*'
-          sudo apt-get install --yes bison cmake flex g++ git libelf-dev libgtest-dev libgmock-dev zlib1g-dev libfl-dev libcereal-dev libdw-dev libpcap-dev systemtap-sdt-dev binutils-dev llvm-11 llvm-11-dev llvm-11-runtime libllvm11 clang-11 libclang-11-dev libclang-common-11-dev libclang1-11 libbpfcc-dev systemtap-sdt-dev python3 python3-distutils xxd libssl-dev pkg-config make
+          sudo apt-get install --yes bison cmake flex g++ git libelf-dev libgtest-dev libgmock-dev zlib1g-dev libfl-dev libcereal-dev libdw-dev libpcap-dev systemtap-sdt-dev binutils-dev llvm-11 llvm-11-dev llvm-11-runtime libllvm11 clang-11 libclang-11-dev libclang-common-11-dev libclang1-11 libbpfcc-dev systemtap-sdt-dev python3 python3-distutils xxd libssl-dev pkg-config make dwarves
 
       - name: Configure (cpp)
         if: ${{ matrix.language == 'cpp' }}

.gitignore

@@ -2,7 +2,7 @@
 build/
 build-*/
 tests/runtime/*.pyc
-tests/data/dwarf_data.h
+tests/data/*.h
 .vagrant
 tests/**/*.bc
 *~

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to
 #### Added
 - Add `iter:task_vma` iterators detection
   - [#2524](https://github.com/iovisor/bpftrace/pull/2524)
+- Support parsing bitfields from BTF/DWARF
+  - [#2505](https://github.com/iovisor/bpftrace/pull/2505)
 #### Changed
 #### Deprecated
 #### Removed

docker/Dockerfile.alpine

@@ -3,9 +3,11 @@ FROM alpine:${ALPINE_VERSION}
 
 ARG LLVM_VERSION="10"
 ARG CEREAL_VERSION=1.3.0
+ARG PAHOLE_VERSION=1.24
 
 RUN apk add --update \
   asciidoctor \
+  argp-standalone \
   bash \
   bison \
   bcc-dev \
@@ -26,6 +28,7 @@ RUN apk add --update \
   linux-headers \
   llvm${LLVM_VERSION}-dev \
   llvm${LLVM_VERSION}-static \
+  musl-obstack-dev \
   python3 \
   wget \
   xxd \
@@ -42,6 +45,17 @@ RUN wget https://github.com/USCiLab/cereal/archive/refs/tags/v${CEREAL_VERSION}.
     tar xf v${CEREAL_VERSION}.tar.gz && \
     cp -r cereal-${CEREAL_VERSION}/include/cereal /usr/include
 
+# bpftrace needs pahole to build BTF data for tests.
+# Since pahole is not packaged in Alpine 3.12, we build it from source.
+# An alternative would be to bump the Alpine version, but I wasn't able to make
+# work any version that would have pahole.
+RUN git clone --recurse-submodules https://github.com/acmel/dwarves/ && \
+    cd dwarves && \
+    git checkout v${PAHOLE_VERSION} && \
+    mkdir build && cd build && \
+    cmake -D__LIB=lib -DBUILD_SHARED_LIBS=OFF .. && \
+    make install
+
 COPY build.sh /build.sh
 RUN chmod 755 /build.sh
 ENTRYPOINT ["/bin/sh", "/build.sh"]

docker/Dockerfile.focal

@@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y \
       asciidoctor \
       bison \
       binutils-dev \
+      dwarves \
       flex \
       g++ \
       git \

docker/Dockerfile.ubuntu-glibc

@@ -7,6 +7,7 @@ ARG LLVM_VERSION
 
 ARG CMAKE_VER="3.16"
 ARG CMAKE_BUILD="2"
+ARG CEREAL_VERSION=1.3.0
 
 ENV LLVM_VERSION=$LLVM_VERSION
 ENV CMAKE_VER=${CMAKE_VER}
@@ -19,23 +20,28 @@ RUN apt-get update \
         asciidoctor \
         bison \
         binutils-dev \
+        dwarves \
         flex \
         git \
         libelf-dev \
         zlib1g-dev \
         libiberty-dev \
         libbfd-dev \
-        libcereal-dev \
         libedit-dev \
         libpcap-dev \
         systemtap-sdt-dev \
         python3 \
         python3-setuptools \
         quilt \
         libssl-dev \
+        wget \
+        xxd \
     && apt-get install --no-install-recommends -y \
         pkg-config
 
+RUN wget https://github.com/USCiLab/cereal/archive/refs/tags/v${CEREAL_VERSION}.tar.gz && \
+    tar xf v${CEREAL_VERSION}.tar.gz && \
+    cp -r cereal-${CEREAL_VERSION}/include/cereal /usr/include
 
 COPY build.sh /build.sh
 RUN chmod 755 /build.sh

src/ast/passes/codegen_llvm.cpp

@@ -1896,11 +1896,12 @@ void CodegenLLVM::visit(FieldAccess &acc)
   {
     // Structs may contain two kinds of fields that must be handled separately
     // (bitfields and _data_loc)
-    if (field.type.IsIntTy() && (field.is_bitfield || field.is_data_loc))
+    if (field.type.IsIntTy() &&
+        (field.bitfield.has_value() || field.is_data_loc))
     {
       Value *src = b_.CreateAdd(expr_, b_.getInt64(field.offset));
 
-      if (field.is_bitfield)
+      if (field.bitfield.has_value())
       {
         Value *raw;
         auto field_type = b_.GetType(field.type);
@@ -1919,7 +1920,7 @@ void CodegenLLVM::visit(FieldAccess &acc)
           b_.CREATE_MEMSET(dst, b_.getInt8(0), field.type.GetSize(), 1);
           b_.CreateProbeRead(ctx_,
                              dst,
-                             b_.getInt32(field.bitfield.read_bytes),
+                             b_.getInt32(field.bitfield->read_bytes),
                              src,
                              type.GetAS(),
                              acc.loc);
@@ -1928,13 +1929,13 @@ void CodegenLLVM::visit(FieldAccess &acc)
         }
         size_t rshiftbits;
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-        rshiftbits = field.bitfield.access_rshift;
+        rshiftbits = field.bitfield->access_rshift;
 #else
-        rshiftbits = (field.type.GetSize() - field.bitfield.read_bytes) * 8;
-        rshiftbits += field.bitfield.access_rshift;
+        rshiftbits = (field.type.GetSize() - field.bitfield->read_bytes) * 8;
+        rshiftbits += field.bitfield->access_rshift;
 #endif
         Value *shifted = b_.CreateLShr(raw, rshiftbits);
-        Value *masked = b_.CreateAnd(shifted, field.bitfield.mask);
+        Value *masked = b_.CreateAnd(shifted, field.bitfield->mask);
         expr_ = masked;
       }
       else

src/ast/passes/resource_analyser.cpp

@@ -87,13 +87,7 @@ void ResourceAnalyser::visit(Call &call)
           .name = "",
           .type = ty,
           .offset = 0,
-          .is_bitfield = false,
-          .bitfield =
-              Bitfield{
-                  .read_bytes = 0,
-                  .access_rshift = 0,
-                  .mask = 0,
-              },
+          .bitfield = std::nullopt,
       });
     }
 

src/ast/passes/semantic_analyser.cpp

@@ -987,13 +987,7 @@ void SemanticAnalyser::visit(Call &call)
               .name = "",
               .type = ty,
               .offset = 0,
-              .is_bitfield = false,
-              .bitfield =
-                  Bitfield{
-                      .read_bytes = 0,
-                      .access_rshift = 0,
-                      .mask = 0,
-                  },
+              .bitfield = std::nullopt,
           });
         }
         std::string msg = validate_format_string(fmt.str, args, call.func);

src/btf.cpp

@@ -773,6 +773,18 @@ void BTF::resolve_fields(SizedType &type)
   resolve_fields(type_id, record.get(), 0);
 }
 
+static std::optional<Bitfield> resolve_bitfield(
+    const struct btf_type *record_type,
+    __u32 member_idx)
+{
+  __u32 bitfield_width = btf_member_bitfield_size(record_type, member_idx);
+  if (bitfield_width <= 0)
+    return std::nullopt;
+
+  return Bitfield(btf_member_bit_offset(record_type, member_idx) % 8,
+                  bitfield_width);
+}
+
 void BTF::resolve_fields(const BTFId &type_id,
                          Struct *record,
                          __u32 start_offset)
@@ -795,13 +807,8 @@ void BTF::resolve_fields(const BTFId &type_id,
 
     std::string field_name = btf__name_by_offset(type_id.btf,
                                                  members[i].name_off);
-    if (members[i].offset % 8 != 0)
-    {
-      // bitfield - not supported yet
-      record->ClearFields();
-      return;
-    }
-    __u32 field_offset = start_offset + members[i].offset / 8;
+
+    __u32 field_offset = start_offset + btf_member_bit_offset(btf_type, i) / 8;
 
     if (btf_is_composite(field_type) &&
         (field_name.empty() || field_name == "(anon)"))
@@ -810,8 +817,11 @@ void BTF::resolve_fields(const BTFId &type_id,
       return;
     }
 
-    record->AddField(
-        field_name, get_stype(field_id), field_offset, false, {}, false);
+    record->AddField(field_name,
+                     get_stype(field_id),
+                     field_offset,
+                     resolve_bitfield(btf_type, i),
+                     false);
   }
 }
 

src/clang_parser.cpp

@@ -202,54 +202,14 @@ static CXCursor get_named_parent(CXCursor c)
   return parent;
 }
 
-// @returns true on success, false otherwise
-static bool getBitfield(CXCursor c, Bitfield &bitfield)
+static std::optional<Bitfield> getBitfield(CXCursor c)
 {
   if (!clang_Cursor_isBitField(c)) {
-    return false;
+    return std::nullopt;
   }
 
-  // Algorithm description:
-  // To handle bitfields, we need to give codegen 3 additional pieces
-  // of information: `read_bytes`, `access_rshift`, and `mask`.
-  //
-  // `read_bytes` tells codegen how many bytes to read starting at `Field::offset`.
-  // This information is necessary because we can't always issue, for example, a
-  // 1 byte read, as the bitfield could be the last 4 bits of the struct. Reading
-  // past the end of the struct could cause a page fault. Therefore, we compute the
-  // minimum number of bytes necessary to fully read the bitfield. This will always
-  // keep the read within the bounds of the struct.
-  //
-  // `access_rshift` tells codegen how much to shift the masked value so that the
-  // LSB of the bitfield is the LSB of the interpreted integer.
-  //
-  // `mask` tells codegen how to mask out the surrounding bitfields.
-
-  size_t bitfield_offset = clang_Cursor_getOffsetOfField(c) % 8;
-  size_t bitfield_bitwidth = clang_getFieldDeclBitWidth(c);
-  size_t bitfield_bitdidth_max = sizeof(uint64_t) * 8;
-
-  if (bitfield_bitwidth > bitfield_bitdidth_max)
-  {
-    LOG(WARNING) << "bitfiled bitwidth " << bitfield_bitwidth
-                 << "is not supporeted."
-                 << " Use bitwidth " << bitfield_bitdidth_max;
-    bitfield_bitwidth = bitfield_bitdidth_max;
-  }
-  if (bitfield_bitwidth == bitfield_bitdidth_max)
-    bitfield.mask = std::numeric_limits<uint64_t>::max();
-  else
-    bitfield.mask = (1ULL << bitfield_bitwidth) - 1;
-  // Round up to nearest byte
-  bitfield.read_bytes = (bitfield_offset + bitfield_bitwidth + 7) / 8;
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-  bitfield.access_rshift = bitfield_offset;
-#else
-  bitfield.access_rshift = (bitfield.read_bytes * 8 - bitfield_offset -
-                            bitfield_bitwidth);
-#endif
-
-  return true;
+  return Bitfield(clang_Cursor_getOffsetOfField(c) % 8,
+                  clang_getFieldDeclBitWidth(c));
 }
 
 // NOTE(mmarchini): as suggested in http://clang-developers.42468.n3.nabble.com/Extracting-macro-information-using-libclang-the-C-Interface-to-Clang-td4042648.html#message4042666
@@ -547,8 +507,7 @@ bool ClangParser::visit_children(CXCursor &cursor, BPFtrace &bpftrace)
           auto offset = clang_Type_getOffsetOf(ptype, ident.c_str()) / 8;
           auto type = clang_getCanonicalType(clang_getCursorType(c));
           auto sized_type = get_sized_type(type, structs);
-          Bitfield bitfield;
-          bool is_bitfield = getBitfield(c, bitfield);
+          auto bitfield = getBitfield(c);
           bool is_data_loc = false;
 
           // Process field annotations
@@ -583,7 +542,7 @@ bool ClangParser::visit_children(CXCursor &cursor, BPFtrace &bpftrace)
           // checked clang diagnostics. The diagnostics will tell us if we have
           // duplicated types.
           structs.Lookup(ptypestr).lock()->AddField(
-              ident, sized_type, offset, is_bitfield, bitfield, is_data_loc);
+              ident, sized_type, offset, bitfield, is_data_loc);
         }
 
         return CXChildVisit_Recurse;