Feature/convert: Add filename and support native method #1846

zdyj3170101136 · 2023-07-10T07:47:03Z

Why?

the old logic could not process native method.

i need the file name to connect source code and pprof.

What?

support native method and add file name.

How?

support native method: see https://github.com/async-profiler/async-profiler/blob/master/src/converter/jfr2pprof.java#L188

connect source code and pprof:
1, uncompress jar file
2, decompile jar file in ./BOOT-INF/lib
3, rewrite java file
example:

decompile jar file

mkdir src && cd src && jar xf /Users/jie.yang05/Downloads/risk_management_web.web.jar
wget https://github.com/intoolswetrust/jd-cli/releases/download/jd-cli-1.2.0/jd-cli-1.2.0-dist.tar.gz && tar xvf jd-cli-1.2.0-dist.tar.gz
 java -jar jd-cli.jar --displayLineNumbers ./BOOT-INF/lib/kafka-clients-2.5.0.jar --outputDir ./
cd org/apache/kafka/clients

the java file could not directly used for pprof, we should rewrite it:

/yangjie05-mac:clients jie.yang05$ cat  org/apache/kafka/clients/NodeApiVersions.java
/*    0 */ package org.apache.kafka.clients;
/*    0 */ 
/*    0 */ import java.util.ArrayList;
/*    0 */ import java.util.Collection;
/*    0 */ import java.util.Collections;
/*    0 */ import java.util.EnumMap;
/*    0 */ import java.util.LinkedList;
/*    0 */ import java.util.List;
/*    0 */ import java.util.Map;
/*    0 */ import java.util.TreeMap;
/*    0 */ import org.apache.kafka.common.errors.UnsupportedVersionException;
/*    0 */ import org.apache.kafka.common.message.ApiVersionsResponseData;
/*    0 */ import org.apache.kafka.common.protocol.ApiKeys;
/*    0 */ import org.apache.kafka.common.utils.Utils;
/*    0 */ 
/*    0 */ public class NodeApiVersions {
/*   40 */   private final Map<ApiKeys, ApiVersion> supportedVersions = new EnumMap<>(ApiKeys.class);
/*    0 */   
/*   43 */   private final List<ApiVersion> unknownApis = new ArrayList<>();
/*    0 */   
/*    0 */   public static NodeApiVersions create() {
/*   51 */     return create(Collections.emptyList());
/*    0 */   }
/*    0 */   
/*    0 */   public static NodeApiVersions create(Collection<ApiVersion> overrides) {
/*   62 */     List<ApiVersion> apiVersions = new LinkedList<>(overrides);
/*   63 */     for (ApiKeys apiKey : ApiKeys.values()) {
/*   64 */       boolean exists = false;
/*   65 */       for (ApiVersion apiVersion : apiVersions) {
/*   66 */         if (apiVersion.apiKey == apiKey.id) {
/*   67 */           exists = true;
/*    0 */           break;
/*    0 */         } 
/*    0 */       } 
/*   71 */       if (!exists)
/*   72 */         apiVersions.add(new ApiVersion(apiKey)); 
/*    0 */     } 
/*   75 */     return new NodeApiVersions(apiVersions);
/*    0 */   }
/*    0 */   
/*    0 */   public static NodeApiVersions create(short apiKey, short minVersion, short maxVersion) {
/*   88 */     return create(Collections.singleton(new ApiVersion(apiKey, minVersion, maxVersion)));
/*    0 */   }
/*    0 */   
/*    0 */   public NodeApiVersions(ApiVersionsResponseData.ApiVersionsResponseKeyCollection nodeApiVersions) {
/*   92 */     for (ApiVersionsResponseData.ApiVersionsResponseKey nodeApiVersion : nodeApiVersions) {
/*   93 */       if (ApiKeys.hasId(nodeApiVersion.apiKey())) {
/*   94 */         ApiKeys nodeApiKey = ApiKeys.forId(nodeApiVersion.apiKey());
/*   95 */         this.supportedVersions.put(nodeApiKey, new ApiVersion(nodeApiVersion));
/*    0 */         continue;
/*    0 */       } 
/*   98 */       this.unknownApis.add(new ApiVersion(nodeApiVersion));
/*    0 */     } 
/*    0 */   }
/*    0 */   
/*    0 */   public NodeApiVersions(Collection<ApiVersion> nodeApiVersions) {
/*  104 */     for (ApiVersion nodeApiVersion : nodeApiVersions) {
/*  105 */       if (ApiKeys.hasId(nodeApiVersion.apiKey)) {
/*  106 */         ApiKeys nodeApiKey = ApiKeys.forId(nodeApiVersion.apiKey);
/*  107 */         this.supportedVersions.put(nodeApiKey, nodeApiVersion);
/*    0 */         continue;
/*    0 */       } 
/*  110 */       this.unknownApis.add(nodeApiVersion);
/*    0 */     } 
/*    0 */   }
/*    0 */   
/*    0 */   public short latestUsableVersion(ApiKeys apiKey) {
/*  119 */     return latestUsableVersion(apiKey, apiKey.oldestVersion(), apiKey.latestVersion());
/*    0 */   }
/*    0 */   
/*    0 */   public short latestUsableVersion(ApiKeys apiKey, short oldestAllowedVersion, short latestAllowedVersion) {
/*  126 */     ApiVersion usableVersion = this.supportedVersions.get(apiKey);
/*  127 */     if (usableVersion == null)
/*  128 */       throw new UnsupportedVersionException("The broker does not support " + apiKey); 
/*  129 */     return latestUsableVersion(apiKey, usableVersion, oldestAllowedVersion, latestAllowedVersion);
/*    0 */   }
/*    0 */   
/*    0 */   private short latestUsableVersion(ApiKeys apiKey, ApiVersion supportedVersions, short minAllowedVersion, short maxAllowedVersion) {
/*  134 */     short minVersion = (short)Math.max(minAllowedVersion, supportedVersions.minVersion);
/*  135 */     short maxVersion = (short)Math.min(maxAllowedVersion, supportedVersions.maxVersion);
/*  136 */     if (minVersion > maxVersion)
/*  137 */       throw new UnsupportedVersionException("The broker does not support " + apiKey + " with version in range [" + minAllowedVersion + "," + maxAllowedVersion + "]. The supported range is [" + supportedVersions.minVersion + "," + supportedVersions.maxVersion + "]."); 
/*  140 */     return maxVersion;
/*    0 */   }
/*    0 */   
/*    0 */   public String toString() {
/*  150 */     return toString(false);
/*    0 */   }
/*    0 */   
/*    0 */   public String toString(boolean lineBreaks) {
/*  162 */     TreeMap<Short, String> apiKeysText = new TreeMap<>();
/*  163 */     for (ApiVersion supportedVersion : this.supportedVersions.values())
/*  164 */       apiKeysText.put(Short.valueOf(supportedVersion.apiKey), apiVersionToText(supportedVersion)); 
/*  165 */     for (ApiVersion apiVersion : this.unknownApis)
/*  166 */       apiKeysText.put(Short.valueOf(apiVersion.apiKey), apiVersionToText(apiVersion)); 
/*  170 */     for (ApiKeys apiKey : ApiKeys.values()) {
/*  171 */       if (!apiKeysText.containsKey(Short.valueOf(apiKey.id))) {
/*  172 */         StringBuilder stringBuilder = new StringBuilder();
/*  173 */         stringBuilder.append(apiKey.name).append("(")
/*  174 */           .append(apiKey.id).append("): ").append("UNSUPPORTED");
/*  175 */         apiKeysText.put(Short.valueOf(apiKey.id), stringBuilder.toString());
/*    0 */       } 
/*    0 */     } 
/*  178 */     String separator = lineBreaks ? ",\n\t" : ", ";
/*  179 */     StringBuilder bld = new StringBuilder();
/*  180 */     bld.append("(");
/*  181 */     if (lineBreaks)
/*  182 */       bld.append("\n\t"); 
/*  183 */     bld.append(Utils.join(apiKeysText.values(), separator));
/*  184 */     if (lineBreaks)
/*  185 */       bld.append("\n"); 
/*  186 */     bld.append(")");
/*  187 */     return bld.toString();
/*    0 */   }
/*    0 */   
/*    0 */   private String apiVersionToText(ApiVersion apiVersion) {
/*  191 */     StringBuilder bld = new StringBuilder();
/*  192 */     ApiKeys apiKey = null;
/*  193 */     if (ApiKeys.hasId(apiVersion.apiKey)) {
/*  194 */       apiKey = ApiKeys.forId(apiVersion.apiKey);
/*  195 */       bld.append(apiKey.name).append("(").append(apiKey.id).append("): ");
/*    0 */     } else {
/*  197 */       bld.append("UNKNOWN(").append(apiVersion.apiKey).append("): ");
/*    0 */     } 
/*  200 */     if (apiVersion.minVersion == apiVersion.maxVersion) {
/*  201 */       bld.append(apiVersion.minVersion);
/*    0 */     } else {
/*  203 */       bld.append(apiVersion.minVersion).append(" to ").append(apiVersion.maxVersion);
/*    0 */     } 
/*  206 */     if (apiKey != null) {
/*  207 */       ApiVersion supportedVersion = this.supportedVersions.get(apiKey);
/*  208 */       if (apiKey.latestVersion() < supportedVersion.minVersion) {
/*  209 */         bld.append(" [unusable: node too new]");
/*  210 */       } else if (supportedVersion.maxVersion < apiKey.oldestVersion()) {
/*  211 */         bld.append(" [unusable: node too old]");
/*    0 */       } else {
/*  213 */         short latestUsableVersion = Utils.min(apiKey.latestVersion(), supportedVersion.maxVersion);
/*  214 */         bld.append(" [usable: ").append(latestUsableVersion).append("]");
/*    0 */       } 
/*    0 */     } 
/*  217 */     return bld.toString();
/*    0 */   }
/*    0 */   
/*    0 */   public ApiVersion apiVersion(ApiKeys apiKey) {
/*  227 */     return this.supportedVersions.get(apiKey);
/*    0 */   }
/*    0 */ }

after rewrite it:

yangjie05-mac:testdata jie.yang05$ cat -n a.java.tmp 
     1  
     2  
     3  
     4  
     5  
     6  
     7  
     8  
     9  
    10  
    11  
    12  
    13  
    14  
    15  
    16  
    17  
    18  
    19  
    20  
    21  
    22  
    23  
    24   package org.apache.kafka.clients;
    25  
    26   import java.util.ArrayList;
    27   import java.util.Collection;
    28   import java.util.Collections;
    29   import java.util.EnumMap;
    30   import java.util.LinkedList;
    31   import java.util.List;
    32   import java.util.Map;
    33   import java.util.TreeMap;
    34   import org.apache.kafka.common.errors.UnsupportedVersionException;
    35   import org.apache.kafka.common.message.ApiVersionsResponseData;
    36   import org.apache.kafka.common.protocol.ApiKeys;
    37   import org.apache.kafka.common.utils.Utils;
    38  
    39   public class NodeApiVersions {
    40     private final Map<ApiKeys, ApiVersion> supportedVersions = new EnumMap<>(ApiKeys.class);
    41  
    42  
    43     private final List<ApiVersion> unknownApis = new ArrayList<>();
    44  
    45  
    46  
    47  
    48  
    49  
    50     public static NodeApiVersions create() {
    51       return create(Collections.emptyList());
    52     }
    53  
    54  
    55  
    56  
    57  
    58  
    59  
    60  
    61     public static NodeApiVersions create(Collection<ApiVersion> overrides) {
    62       List<ApiVersion> apiVersions = new LinkedList<>(overrides);
    63       for (ApiKeys apiKey : ApiKeys.values()) {
    64         boolean exists = false;
    65         for (ApiVersion apiVersion : apiVersions) {
    66           if (apiVersion.apiKey == apiKey.id) {
    67             exists = true;
    68             break;
    69           }
    70         }
    71         if (!exists)
    72           apiVersions.add(new ApiVersion(apiKey));
    73       }
    74  
    75       return new NodeApiVersions(apiVersions);
    76     }
    77  
    78  
    79  
    80  
    81  
    82  
    83  
    84  
    85  
    86  
    87     public static NodeApiVersions create(short apiKey, short minVersion, short maxVersion) {
    88       return create(Collections.singleton(new ApiVersion(apiKey, minVersion, maxVersion)));
    89     }
    90  
    91     public NodeApiVersions(ApiVersionsResponseData.ApiVersionsResponseKeyCollection nodeApiVersions) {
    92       for (ApiVersionsResponseData.ApiVersionsResponseKey nodeApiVersion : nodeApiVersions) {
    93         if (ApiKeys.hasId(nodeApiVersion.apiKey())) {
    94           ApiKeys nodeApiKey = ApiKeys.forId(nodeApiVersion.apiKey());
    95           this.supportedVersions.put(nodeApiKey, new ApiVersion(nodeApiVersion));
    96           continue;
    97         }
    98         this.unknownApis.add(new ApiVersion(nodeApiVersion));
    99       }
   100     }
   101  
   102  
   103     public NodeApiVersions(Collection<ApiVersion> nodeApiVersions) {
   104       for (ApiVersion nodeApiVersion : nodeApiVersions) {
   105         if (ApiKeys.hasId(nodeApiVersion.apiKey)) {
   106           ApiKeys nodeApiKey = ApiKeys.forId(nodeApiVersion.apiKey);
   107           this.supportedVersions.put(nodeApiKey, nodeApiVersion);
   108           continue;
   109         }
   110         this.unknownApis.add(nodeApiVersion);
   111       }
   112     }
   113  
   114  
   115  
   116  
   117  
   118     public short latestUsableVersion(ApiKeys apiKey) {
   119       return latestUsableVersion(apiKey, apiKey.oldestVersion(), apiKey.latestVersion());
   120     }
   121  
   122  
   123  
   124  
   125     public short latestUsableVersion(ApiKeys apiKey, short oldestAllowedVersion, short latestAllowedVersion) {
   126       ApiVersion usableVersion = this.supportedVersions.get(apiKey);
   127       if (usableVersion == null)
   128         throw new UnsupportedVersionException("The broker does not support " + apiKey);
   129       return latestUsableVersion(apiKey, usableVersion, oldestAllowedVersion, latestAllowedVersion);
   130     }
   131  
   132  
   133     private short latestUsableVersion(ApiKeys apiKey, ApiVersion supportedVersions, short minAllowedVersion, short maxAllowedVersion) {
   134       short minVersion = (short)Math.max(minAllowedVersion, supportedVersions.minVersion);
   135       short maxVersion = (short)Math.min(maxAllowedVersion, supportedVersions.maxVersion);
   136       if (minVersion > maxVersion)
   137         throw new UnsupportedVersionException("The broker does not support " + apiKey + " with version in range [" + minAllowedVersion + "," + maxAllowedVersion + "]. The supportedrange is [" + supportedVersions.minVersion + "," + supportedVersions.maxVersion + "].");
   138  
   139  
   140       return maxVersion;
   141     }
   142  
   143  
   144  
   145  
   146  
   147  
   148  
   149     public String toString() {
   150       return toString(false);
   151     }
   152  
   153  
   154  
   155  
   156  
   157  
   158  
   159  
   160  
   161     public String toString(boolean lineBreaks) {
   162       TreeMap<Short, String> apiKeysText = new TreeMap<>();
   163       for (ApiVersion supportedVersion : this.supportedVersions.values())
   164         apiKeysText.put(Short.valueOf(supportedVersion.apiKey), apiVersionToText(supportedVersion));
   165       for (ApiVersion apiVersion : this.unknownApis)
   166         apiKeysText.put(Short.valueOf(apiVersion.apiKey), apiVersionToText(apiVersion));
   167  
   168  
   169  
   170       for (ApiKeys apiKey : ApiKeys.values()) {
   171         if (!apiKeysText.containsKey(Short.valueOf(apiKey.id))) {
   172           StringBuilder stringBuilder = new StringBuilder();
   173           stringBuilder.append(apiKey.name).append("(")
   174             .append(apiKey.id).append("): ").append("UNSUPPORTED");
   175           apiKeysText.put(Short.valueOf(apiKey.id), stringBuilder.toString());
   176         }
   177       }
   178       String separator = lineBreaks ? ",\n\t" : ", ";
   179       StringBuilder bld = new StringBuilder();
   180       bld.append("(");
   181       if (lineBreaks)
   182         bld.append("\n\t");
   183       bld.append(Utils.join(apiKeysText.values(), separator));
   184       if (lineBreaks)
   185         bld.append("\n");
   186       bld.append(")");
   187       return bld.toString();
   188     }
   189  
   190     private String apiVersionToText(ApiVersion apiVersion) {
   191       StringBuilder bld = new StringBuilder();
   192       ApiKeys apiKey = null;
   193       if (ApiKeys.hasId(apiVersion.apiKey)) {
   194         apiKey = ApiKeys.forId(apiVersion.apiKey);
   195         bld.append(apiKey.name).append("(").append(apiKey.id).append("): ");
   196       } else {
   197         bld.append("UNKNOWN(").append(apiVersion.apiKey).append("): ");
   198       }
   199  
   200       if (apiVersion.minVersion == apiVersion.maxVersion) {
   201         bld.append(apiVersion.minVersion);
   202       } else {
   203         bld.append(apiVersion.minVersion).append(" to ").append(apiVersion.maxVersion);
   204       }
   205  
   206       if (apiKey != null) {
   207         ApiVersion supportedVersion = this.supportedVersions.get(apiKey);
   208         if (apiKey.latestVersion() < supportedVersion.minVersion) {
   209           bld.append(" [unusable: node too new]");
   210         } else if (supportedVersion.maxVersion < apiKey.oldestVersion()) {
   211           bld.append(" [unusable: node too old]");
   212         } else {
   213           short latestUsableVersion = Utils.min(apiKey.latestVersion(), supportedVersion.maxVersion);
   214           bld.append(" [usable: ").append(latestUsableVersion).append("]");
   215         }
   216       }
   217       return bld.toString();
   218     }
   219  
   220  
   221  
   222  
   223  
   224  
   225  
   226     public ApiVersion apiVersion(ApiKeys apiKey) {
   227       return this.supportedVersions.get(apiKey);
   228     }
   229   }

``

brancz

Very nice! Thank you!

brancz · 2023-07-10T08:27:43Z

Looks like there is a failing test though, can you fix that?

zdyj3170101136 · 2023-07-13T06:18:13Z

@brancz

brancz

Awesome! Thank you!

converter: add filename and support native method

9bd1424

zdyj3170101136 requested a review from a team as a code owner July 10, 2023 07:47

brancz approved these changes Jul 10, 2023

View reviewed changes

converter: fix

53a561a

zdyj3170101136 requested a review from brancz July 11, 2023 02:19

Merge branch 'main' into feature/convert

a95b16c

zdyj3170101136 added 4 commits July 13, 2023 18:14

converter: fix getFileName

742121c

converter: add args into func name

da467a2

converter: only add args for jvm method

72ed5f3

jfr: only get filename when JVM method

44118e5

brancz approved these changes Jul 27, 2023

View reviewed changes

brancz merged commit cc92bee into parca-dev:main Jul 27, 2023
22 checks passed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Feature/convert: Add filename and support native method #1846

Feature/convert: Add filename and support native method #1846

zdyj3170101136 commented Jul 10, 2023

brancz left a comment

brancz commented Jul 10, 2023

zdyj3170101136 commented Jul 13, 2023

brancz left a comment

Feature/convert: Add filename and support native method #1846

Feature/convert: Add filename and support native method #1846

Conversation

zdyj3170101136 commented Jul 10, 2023

Why?

What?

How?

brancz left a comment

Choose a reason for hiding this comment

brancz commented Jul 10, 2023

zdyj3170101136 commented Jul 13, 2023

brancz left a comment

Choose a reason for hiding this comment