-
-
Notifications
You must be signed in to change notification settings - Fork 2.4k
/
genwebsitejson.py
executable file
·146 lines (123 loc) · 5.29 KB
/
genwebsitejson.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
#!/usr/bin/env python3
# Copyright (c) 2014-present, The osquery authors
#
# This source code is licensed as defined by the LICENSE file found in the
# root directory of this source tree.
#
# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
"""
Generate a complete table specification for the website
This script will generate JSON output as expected by the osquery website given
a directory of osquery schema specifications. Results will be printer to stdout.
Usage:
python tools/codegen/genwebsitejson.py --specs=./specs
"""
# Copyright (c) 2014-present, The osquery authors
#
# This source code is licensed as defined by the LICENSE file found in the
# root directory of this source tree.
#
# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
import json
import os
import re
import sys
from gentable import *
# In the specs/ directory of the osquery repository, specification files are put
# in certain directories based on what platforms they are meant to be built on.
# This data structure represents the directories in specs/ and how they map to
# the operating systems which support tables found in those directories
PLATFORM_DIRS = {
"specs": ["darwin", "linux", "windows"],
"utility": ["darwin", "linux", "windows"],
"yara": ["darwin", "linux", "windows"],
"smart": ["darwin", "linux"],
"darwin": ["darwin"],
"kernel": ["darwin"],
"linwin": ["linux", "windows"],
"linux": ["linux"],
"macwin": ["darwin", "windows"],
"posix": ["darwin", "linux"],
"sleuthkit": ["darwin", "linux"],
"windows": ["windows"],
}
BASE_SOURCE_URL = "https://github.com/osquery/osquery/blob/master"
def platform_for_spec(path):
"""Given a path to a table specification, return a list of what osquery
platforms that table will work on. In the event that no match is found, it
will be assumed that the table is found on all platforms.
"""
full_path = os.path.abspath(path)
directory_list = os.path.dirname(full_path).split("/")
directory = directory_list[len(directory_list)-1]
return PLATFORM_DIRS[directory]
def remove_prefix(text, prefix):
# python 3.9 has `removeprefix`, but I don't want to add that requirement.
if text.startswith(prefix):
return text[len(prefix):]
return text
def url_for_spec(specs_dir, path):
"""Given a path to a table specification, return the URL that would take you
to the specification on GitHub.
"""
path_fragment = remove_prefix(path, specs_dir).lstrip("/ ")
url = os.path.join(BASE_SOURCE_URL, "specs", path_fragment)
return url
def generate_table_metadata(specs_dir, full_path):
"""This function generates a dictionary of table metadata for a spec file
found at a given path."""
with open(full_path, "r") as file_handle:
# Each osquery table specification is a syntactically correct python file
# because we imported `from gentable import *`, we imported all of the
# functions that you use in an osquery specification. a global "table"
# is then modified based on the python that has just executed.
tree = ast.parse(file_handle.read())
exec(compile(tree, "<string>", "exec"))
# Now that the `table` variable is accessible, we can access attributes
# of the table
t = {}
t["name"] = table.table_name
t["description"] = table.description
t["url"] = url_for_spec(specs_dir, full_path)
t["platforms"] = platform_for_spec(full_path)
t["evented"] = "event_subscriber" in table.attributes
t["cacheable"] = "cacheable" in table.attributes
t["notes"] = table.notes
t["examples"] = table.examples
# Now we must iterate through `table.columns` to collect information
# about each column
t["columns"] = []
for col in table.columns():
c = {}
c["name"] = col.name
c["description"] = col.description
c["type"] = col.type.affinity.replace("_TYPE", "").lower()
c["notes"] = col.notes
c["hidden"] = col.options.get("hidden", False)
c["required"] = col.options.get("required", False)
c["index"] = col.options.get("index", False)
if col.platforms != []:
c["platforms"] = col.platforms
t["columns"].append(c)
return t
def main(argc, argv):
parser = argparse.ArgumentParser(
"Generate minmal JSON from a table spec")
parser.add_argument("--specs", help="Path to spec directory", required=True)
args = parser.parse_args()
specs_dir = os.path.abspath(args.specs)
tables = {}
for subdir, dirs, files in os.walk(specs_dir):
for filename in files:
# Skip the example spec in the spec/ dir.
# There is no actual example table in osquery so it should not be generated into the docs.
if filename == "example.table":
continue
if filename.endswith(".table"):
full_path = os.path.join(subdir, filename)
metadata = generate_table_metadata(specs_dir, full_path)
tables[metadata["name"]] = metadata
# Print the JSON output to stdout
print(json.dumps([value for key, value in sorted(tables.items())], indent=2, separators=(',', ':')))
if __name__ == "__main__":
main(len(sys.argv), sys.argv)