-
-
Notifications
You must be signed in to change notification settings - Fork 1.6k
/
docenizer-amd64.py
executable file
·378 lines (337 loc) · 13.1 KB
/
docenizer-amd64.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import json
import os
import re
import sys
import tarfile
import urllib
from urllib import request
from urllib import parse
try:
from bs4 import BeautifulSoup
except ImportError:
raise ImportError("Please install BeautifulSoup (apt-get install python3-bs4 or pip install beautifulsoup4 should do it)")
parser = argparse.ArgumentParser(description='Docenizes HTML version of the official Intel Asm PDFs')
parser.add_argument('-i', '--inputfolder', type=str,
help='Folder where the input files reside as .html. Default is ./asm-docs/',
default='asm-docs')
parser.add_argument('-o', '--outputpath', type=str, help='Final path of the .ts file. Default is ./asm-docs-amd64.ts',
default='./asm-docs-amd64.ts')
parser.add_argument('-d', '--downloadfolder', type=str,
help='Folder where the archive will be downloaded and extracted', default='asm-docs')
# The maximum number of paragraphs from the description to copy.
MAX_DESC_PARAS = 5
STRIP_PREFIX = re.compile(r'^(([0-9a-fA-F]{2}|m64|NP|(REX|E?VEX\.)[.0-9A-Z]*|/[0-9a-z]+|[a-z]+)\b\s*)*')
INSTRUCTION_RE = re.compile(r'^([A-Z][A-Z0-9]+)\*?(\s+|$)')
# Some instructions are so broken we just take their names from the filename
UNPARSEABLE_INSTR_NAMES = ['PSRLW:PSRLD:PSRLQ', 'PSLLW:PSLLD:PSLLQ', 'MOVBE']
# Some files contain instructions which cannot be parsed and which compilers are unlikely to emit
IGNORED_FILE_NAMES = [
# SGX pseudo-instructions
"EADD",
"EACCEPT",
"EAUG",
"EACCEPTCOPY",
"EDECVIRTCHILD",
"EINCVIRTCHILD",
"EINIT",
"ELDB:ELDU:ELDBC:ELBUC",
"EMODPE",
"EMODPR",
"EMODT",
"ERDINFO",
"ESETCONTEXT",
"ETRACKC",
"EBLOCK",
"ECREATE",
"EDBGRD",
"EDBGWR",
"EENTER",
"EEXIT",
"EEXTEND",
"EGETKEY",
"ELDB",
"ELDU",
"ENCLS",
"ENCLU",
"EPA",
"EREMOVE",
"EREPORT",
"ERESUME",
"ETRACK",
"EWB",
# VMX instructions
"INVEPT",
"INVVPID",
"VMCALL",
"VMCLEAR",
"VMFUNC",
"VMLAUNCH",
"VMLAUNCH:VMRESUME",
"VMPTRLD",
"VMPTRST",
"VMREAD",
"VMRESUME",
"VMWRITE",
"VMXOFF",
"VMXON",
# Other instructions
"INVLPG",
"LAHF",
"RDMSR",
"SGDT",
# Unparsable instructions
# These instructions should be supported in the future
"MONITOR",
"MOVDQ2Q",
"MFENCE",
]
# Some instructions are defined in multiple files. We ignore a specific set of the
# duplicates here.
IGNORED_DUPLICATES = [
'MOV-1', # move to control reg
'MOV-2', # move to debug reg
'CMPSD', # compare doubleword (defined in CMPS:CMPSB:CMPSW:CMPSD:CMPSQ)
'MOVQ', # defined in MOVD:MOVQ
'MOVSD', # defined in MOVS:MOVSB:MOVSW:MOVSD:MOVSQ
'VPBROADCASTB:VPBROADCASTW:VPBROADCASTD:VPBROADCASTQ', # defined in VPBROADCAST
"VGATHERDPS:VGATHERDPD",
"VGATHERQPS:VGATHERQPD",
"VPGATHERDD:VPGATHERQD",
"VPGATHERDQ:VPGATHERQQ",
]
# Where to extract the asmdoc archive.
ASMDOC_DIR = "asm-docs"
ARCHIVE_URL = "https://www.felixcloutier.com/x86/x86.tbz2"
ARCHIVE_NAME = "x86.tbz2"
class Instruction(object):
def __init__(self, name, names, tooltip, body):
self.name = name
self.names = names
self.tooltip = tooltip.rstrip(': ,')
self.body = body
def __str__(self):
return f"{self.name} = {self.tooltip}\n{self.body}"
def get_url_for_instruction(instr):
return f"https://www.felixcloutier.com/x86/{urllib.parse.quote(instr.name)}.html"
def download_asm_doc_archive(downloadfolder):
if not os.path.exists(downloadfolder):
print(f"Creating {downloadfolder} as download folder")
os.makedirs(downloadfolder)
elif not os.path.isdir(downloadfolder):
print(f"Error: download folder {downloadfolder} is not a directory")
sys.exit(1)
archive_name = os.path.join(downloadfolder, ARCHIVE_NAME)
print("Downloading archive...")
urllib.request.urlretrieve(ARCHIVE_URL, archive_name)
def extract_asm_doc_archive(downloadfolder, inputfolder):
print("Extracting file...")
if os.path.isdir(os.path.join(inputfolder, "html")):
for root, dirs, files in os.walk(os.path.join(inputfolder, "html")):
for file in files:
if os.path.splitext(file)[1] == ".html":
os.remove(os.path.join(root, file))
tar = tarfile.open(os.path.join(downloadfolder, ARCHIVE_NAME))
tar.extractall(path=inputfolder)
def strip_non_instr(i):
# removes junk from encodings where the opcode is in the middle
# of prefix stuff. e.g.
# 66 0f 38 30 /r PMOVZXBW xmm1, xmm2/m64
return STRIP_PREFIX.sub('', i)
def instr_name(i):
match = INSTRUCTION_RE.match(strip_non_instr(i))
if match:
return match.group(1)
def get_description_paragraphs(document_soup):
description_header_node = document_soup.find(id="description")
i = 0
description_paragraph_node = description_header_node.next_sibling.next_sibling
description_paragraphs = []
while i < MAX_DESC_PARAS and len(description_paragraph_node.text) > 20:
if description_paragraph_node.name == "p":
description_paragraphs.append(description_paragraph_node)
i = i + 1
# Move two siblings forward. Next sibling is the line feed.
description_paragraph_node = description_paragraph_node.next_sibling.next_sibling
return description_paragraphs
def parse(filename, f):
doc = BeautifulSoup(f, 'html.parser')
if doc.table is None:
print(f"{filename}: Failed to find table")
return None
table = read_table(doc.table)
names = set()
def add_all(instrs):
for i in instrs:
instruction_name = instr_name(i)
if instruction_name:
names.add(instruction_name)
for inst in table:
if 'Opcode/Instruction' in inst:
add_all(inst['Opcode/Instruction'].split("\n"))
elif 'OpcodeInstruction' in inst:
add_all(inst['OpcodeInstruction'].split("\n"))
elif 'Opcode Instruction' in inst:
add_all(inst['Opcode Instruction'].split("\n"))
elif 'Opcode*/Instruction' in inst:
add_all(inst['Opcode*/Instruction'].split("\n"))
elif 'Opcode / Instruction' in inst:
add_all(inst['Opcode / Instruction'].split("\n"))
elif 'Instruction' in inst:
instruction_name = instr_name(inst['Instruction'])
if not instruction_name:
print(f"Unable to get instruction from: {inst['Instruction']}")
else:
names.add(instruction_name)
# else, skip the line
if not names:
if filename in UNPARSEABLE_INSTR_NAMES:
for inst in filename.split(":"):
names.add(inst)
else:
print(f"{filename}: Failed to read instruction table")
return None
description_paragraphs = get_description_paragraphs(doc)
for para in description_paragraphs:
for link in para.find_all('a'):
# this urljoin will only ensure relative urls are prefixed
# if a url is already absolute it does nothing
link['href'] = urllib.parse.urljoin('https://www.felixcloutier.com/x86/', link['href'])
link['target'] = '_blank'
link['rel'] = 'noreferrer noopener'
return Instruction(
filename,
names,
description_paragraphs[0].text.strip(),
''.join(map(lambda x: str(x), description_paragraphs)).strip())
def read_table(start_table):
# Tables on felixcloutier may be split in half, e.g. on https://www.felixcloutier.com/x86/sal:sar:shl:shr
# This traverses the immediate siblings of the input table
tables = []
current_node = start_table
while current_node:
if current_node.name == 'table':
tables.append(current_node)
elif current_node.name is not None: # whitespace between the tables, i.e. the \n, is a none tag
break
current_node = current_node.next_sibling
# Finding all 'th' is not enough, since some headers are 'td'.
# Instead, walk through all children of the first 'tr', filter out those
# that are only whitespace, keep `get_text()` on the others.
headers = list(
map(lambda th: th.get_text(),
filter(lambda th: str(th).strip(), tables[0].tr.children)))
result = []
if headers:
# common case
for table in tables:
for row in table.find_all('tr'):
obj = {}
for column, name in zip(row.find_all('td'), headers):
# Remove '\n's in names that contain it.
obj[name.replace('\n', '')] = column.get_text()
if obj:
result.append(obj)
else:
# Cases like BEXTR and BZHI
for table in tables:
rows = table.find_all('tr')
if len(rows) != 1:
return []
obj = {}
for td in rows[0].find_all('td'):
header = td.p.strong.get_text()
td.p.strong.decompose()
obj[header] = td.get_text()
result.append(obj)
return result
def parse_html(directory):
print("Parsing instructions...")
instructions = []
for root, dirs, files in os.walk(directory):
for file in files:
if file.endswith(".html") and file != 'index.html':
with open(os.path.join(root, file), encoding='utf-8') as f2:
name = os.path.splitext(file)[0]
if name in IGNORED_DUPLICATES or name in IGNORED_FILE_NAMES:
continue
try:
instruction = parse(name, f2)
if not instruction:
continue
patch_instruction(instruction)
instructions.append(instruction)
except Exception as e:
print(f"Error parsing {name}:\n{e}")
return instructions
def self_test(instructions, directory):
# For each generated instruction, check that there is a path to a file in
# the documentation.
directory = os.path.join(directory, "html")
ok = True
for inst in instructions:
if not os.path.isfile(os.path.join(directory, inst.name + ".html")):
print(f"Warning: {inst.name} has not file associated")
ok = False
return ok
def patch_instruction(instruction):
if instruction.name == "ADDSS":
print("\nPatching ADDSS")
print("REMINDER: Check if https://github.com/compiler-explorer/compiler-explorer/issues/2380 is still relevant\n")
old_body = instruction.body
old_tooltip = instruction.tooltip
instruction.body = old_body.replace("stores the double-precision", "stores the single-precision")
instruction.tooltip = old_tooltip.replace("stores the double-precision", "stores the single-precision")
def main():
args = parser.parse_args()
print(f"Called with: {args}")
# If we don't have the html folder already...
if not os.path.isdir(os.path.join(args.inputfolder, 'html')):
# We don't, try with the compressed file
if not os.path.isfile(os.path.join(args.downloadfolder, "x86.tbz2")):
# We can't find that either. Download it
try:
download_asm_doc_archive(args.downloadfolder)
extract_asm_doc_archive(args.downloadfolder, args.inputfolder)
except IOError as e:
print("Error when downloading archive:")
print(e)
sys.exit(1)
else:
# We have a file already downloaded
extract_asm_doc_archive(args.downloadfolder, args.inputfolder)
instructions = parse_html(args.inputfolder)
instructions.sort(key=lambda b: b.name)
self_test(instructions, args.inputfolder)
all_inst = set()
for inst in instructions:
if not all_inst.isdisjoint(inst.names):
print(f"Overlap in instruction names: {inst.names.intersection(all_inst)} for {inst.name}")
all_inst = all_inst.union(inst.names)
if not self_test(instructions, args.inputfolder):
print("Tests do not pass. Not writing output file. Aborting.")
sys.exit(3)
print(f"Writing {len(instructions)} instructions")
with open(args.outputpath, 'w') as f:
f.write("""
import {AssemblyInstructionInfo} from '../base';
export function getAsmOpcode(opcode: string | undefined): AssemblyInstructionInfo | undefined {
if (!opcode) return;
switch (opcode.toUpperCase()) {
""".lstrip())
for inst in instructions:
for name in sorted(inst.names):
f.write(f' case "{name}":\n')
f.write(' return {}'.format(json.dumps({
"tooltip": inst.tooltip,
"html": inst.body,
"url": get_url_for_instruction(inst)
}, indent=16, separators=(',', ': '), sort_keys=True))[:-1] + ' };\n\n')
f.write("""
}
}
""")
if __name__ == '__main__':
main()