Skip to content

Commit

Permalink
Merge branch 'dev' into driverless
Browse files Browse the repository at this point in the history
  • Loading branch information
Luca Marturana committed Jan 6, 2017
2 parents 38ecc61 + 82969c8 commit db38999
Show file tree
Hide file tree
Showing 41 changed files with 1,487 additions and 915 deletions.
8 changes: 4 additions & 4 deletions docker/dev/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ RUN echo "deb http://httpredir.debian.org/debian jessie main" > /etc/apt/sources
gcc-4.9 \
libelf1 less && rm -rf /var/lib/apt/lists/*

# Terribly terrible hacks: since our base Debian image ships with GCC 5.0 which breaks older kernels,
# revert the default to gcc-4.9. Also, since some customers use some very old distributions whose kernel
# makefile is hardcoded for gcc-4.6 or so (e.g. Debian Wheezy), we pretend to have gcc 4.6/4.7 by symlinking
# it to 4.9
# Since our base Debian image ships with GCC 5.0 which breaks older kernels, revert the
# default to gcc-4.9. Also, since some customers use some very old distributions whose kernel
# makefile is hardcoded for gcc-4.6 or so (e.g. Debian Wheezy), we pretend to have gcc 4.6/4.7
# by symlinking it to 4.9

RUN rm -rf /usr/bin/gcc \
&& ln -s /usr/bin/gcc-4.9 /usr/bin/gcc \
Expand Down
8 changes: 4 additions & 4 deletions docker/stable/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ RUN echo "deb http://httpredir.debian.org/debian jessie main" > /etc/apt/sources
gcc-4.9 \
libelf1 less && rm -rf /var/lib/apt/lists/*

# Terribly terrible hacks: since our base Debian image ships with GCC 5.0 which breaks older kernels,
# revert the default to gcc-4.9. Also, since some customers use some very old distributions whose kernel
# makefile is hardcoded for gcc-4.6 or so (e.g. Debian Wheezy), we pretend to have gcc 4.6/4.7 by symlinking
# it to 4.9
# Since our base Debian image ships with GCC 5.0 which breaks older kernels, revert the
# default to gcc-4.9. Also, since some customers use some very old distributions whose kernel
# makefile is hardcoded for gcc-4.6 or so (e.g. Debian Wheezy), we pretend to have gcc 4.6/4.7
# by symlinking it to 4.9

RUN rm -rf /usr/bin/gcc \
&& ln -s /usr/bin/gcc-4.9 /usr/bin/gcc \
Expand Down
303 changes: 153 additions & 150 deletions scripts/kernel-crawler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,148 +13,144 @@
# structure
#
repos = {
"CentOS" : [
{
# This is the root path of the repository in which the script will
# look for distros (HTML page)
"root" : "http://mirrors.kernel.org/centos/",

# This is the XPath + Regex (optional) for analyzing the `root`
# page and discover possible distro versions. Use the regex if you
# want to limit the version release
"discovery_pattern" : "/html/body//pre/a[regex:test(@href, '^6|7.*$')]/@href",

# Once we have found every version available, we need to know were
# to go inside the tree to find packages we need (HTML pages)
"subdirs" : [
"os/x86_64/Packages/",
"updates/x86_64/Packages/"
],

# Finally, we need to inspect every page for packages we need.
# Again, this is a XPath + Regex query so use the regex if you want
# to limit the number of packages reported.
"page_pattern" : "/html/body//a[regex:test(@href, '^kernel-(devel-)?[0-9].*\.rpm$')]/@href"
},

{
"root" : "http://vault.centos.org/",
"discovery_pattern" : "//body//table/tr/td/a[regex:test(@href, '^6|7.*$')]/@href",
"subdirs" : [
"os/x86_64/Packages/",
"updates/x86_64/Packages/"
],
"page_pattern" : "//body//table/tr/td/a[regex:test(@href, '^kernel-(devel-)?[0-9].*\.rpm$')]/@href"
},

{
"root" : "http://vault.centos.org/centos/",
"discovery_pattern" : "//body//table/tr/td/a[regex:test(@href, '^6|7.*$')]/@href",
"subdirs" : [
"os/x86_64/Packages/",
"updates/x86_64/Packages/"
],
"page_pattern" : "//body//table/tr/td/a[regex:test(@href, '^kernel-(devel-)?[0-9].*\.rpm$')]/@href"
}
],

"Ubuntu" : [
{
# Had to split the URL because, unlikely other repos for which the
# script was first created, Ubuntu puts everything into a single
# folder. The real URL is be:
# http://mirrors.us.kernel.org/ubuntu/pool/main/l/linux/
"root" : "https://mirrors.kernel.org/ubuntu/pool/main/l/",
"discovery_pattern" : "/html/body//a[@href = 'linux/']/@href",
"subdirs" : [""],
"page_pattern" : "/html/body//a[regex:test(@href, '^linux-(image|headers)-[3-9].*-generic.*amd64.deb$')]/@href",
"exclude_patterns": ["image-4.9", "headers-4.9"]
},

{
"root" : "https://mirrors.kernel.org/ubuntu/pool/main/l/",
"discovery_pattern" : "/html/body//a[@href = 'linux/']/@href",
"subdirs" : [""],
"page_pattern" : "/html/body//a[regex:test(@href, '^linux-headers-[3-9].*_all.deb$')]/@href",
"exclude_patterns": ["image-4.9", "headers-4.9"]
},

{
"root" : "http://security.ubuntu.com/ubuntu/pool/main/l/",
"discovery_pattern" : "/html/body//a[@href = 'linux/']/@href",
"subdirs" : [""],
"page_pattern" : "/html/body//a[regex:test(@href, '^linux-(image|headers)-[3-9].*-generic.*amd64.deb$')]/@href",
"exclude_patterns": ["image-4.9", "headers-4.9"]
},

{
"root" : "http://security.ubuntu.com/ubuntu/pool/main/l/",
"discovery_pattern" : "/html/body//a[@href = 'linux/']/@href",
"subdirs" : [""],
"page_pattern" : "/html/body//a[regex:test(@href, '^linux-headers-[3-9].*_all.deb$')]/@href",
"exclude_patterns": ["image-4.9", "headers-4.9"]
}
],

"Fedora" : [
{
"root" : "https://mirrors.kernel.org/fedora/releases/",
"discovery_pattern": "/html/body//a[regex:test(@href, '^2[2-9]/$')]/@href",
"subdirs" : [
"Everything/x86_64/os/Packages/k/"
],
"page_pattern" : "/html/body//a[regex:test(@href, '^kernel-(core|devel)-[0-9].*\.rpm$')]/@href"
},

{
"root" : "https://mirrors.kernel.org/fedora/updates/",
"discovery_pattern": "/html/body//a[regex:test(@href, '^2[2-9]/$')]/@href",
"subdirs" : [
"x86_64/k/"
],
"page_pattern" : "/html/body//a[regex:test(@href, '^kernel-(core|devel)-[0-9].*\.rpm$')]/@href"
},

# {
# "root" : "https://mirrors.kernel.org/fedora/development/",
# "discovery_pattern": "/html/body//a[regex:test(@href, '^2[2-9]/$')]/@href",
# "subdirs" : [
# "x86_64/os/Packages/k/"
# ],
# "page_pattern" : "/html/body//a[regex:test(@href, '^kernel-(core|devel)-[0-9].*\.rpm$')]/@href"
# }
],

"CoreOS" : [
{
"root" : "http://alpha.release.core-os.net/",
"discovery_pattern": "/html/body//a[regex:test(@href, 'amd64-usr')]/@href",
"subdirs" : [
""
],
"page_pattern" : "/html/body//a[regex:test(@href, '^[5-9][0-9][0-9]|current|[1][0-9]{3}')]/@href"
},

{
"root" : "http://beta.release.core-os.net/",
"discovery_pattern": "/html/body//a[regex:test(@href, 'amd64-usr')]/@href",
"subdirs" : [
""
],
"page_pattern" : "/html/body//a[regex:test(@href, '^[5-9][0-9][0-9]|current|[1][0-9]{3}')]/@href"
},

{
"root" : "http://stable.release.core-os.net/",
"discovery_pattern": "/html/body//a[regex:test(@href, 'amd64-usr')]/@href",
"subdirs" : [
""
],
"page_pattern" : "/html/body//a[regex:test(@href, '^[4-9][0-9][0-9]|current|[1][0-9]{3}')]/@href"
}
],

"Debian": [
"CentOS" : [
{
# This is the root path of the repository in which the script will
# look for distros (HTML page)
"root" : "http://mirrors.kernel.org/centos/",

# This is the XPath + Regex (optional) for analyzing the `root`
# page and discover possible distro versions. Use the regex if you
# want to limit the version release
"discovery_pattern" : "/html/body//pre/a[regex:test(@href, '^6|^7.*$')]/@href",

# Once we have found every version available, we need to know were
# to go inside the tree to find packages we need (HTML pages)
"subdirs" : [
"os/x86_64/Packages/",
"updates/x86_64/Packages/"
],

# Finally, we need to inspect every page for packages we need.
# Again, this is a XPath + Regex query so use the regex if you want
# to limit the number of packages reported.
"page_pattern" : "/html/body//a[regex:test(@href, '^kernel-(devel-)?[0-9].*\.rpm$')]/@href"
},

{
"root" : "http://vault.centos.org/",
"discovery_pattern" : "//body//table/tr/td/a[regex:test(@href, '^6|^7.*$')]/@href",
"subdirs" : [
"os/x86_64/Packages/",
"updates/x86_64/Packages/"
],
"page_pattern" : "//body//table/tr/td/a[regex:test(@href, '^kernel-(devel-)?[0-9].*\.rpm$')]/@href"
},

{
"root" : "http://vault.centos.org/centos/",
"discovery_pattern" : "//body//table/tr/td/a[regex:test(@href, '^6|^7.*$')]/@href",
"subdirs" : [
"os/x86_64/Packages/",
"updates/x86_64/Packages/"
],
"page_pattern" : "//body//table/tr/td/a[regex:test(@href, '^kernel-(devel-)?[0-9].*\.rpm$')]/@href"
}
],

"Ubuntu" : [
{
# Had to split the URL because, unlikely other repos for which the
# script was first created, Ubuntu puts everything into a single
# folder. The real URL is be:
# http://mirrors.us.kernel.org/ubuntu/pool/main/l/linux/
"root" : "https://mirrors.kernel.org/ubuntu/pool/main/l/",
"discovery_pattern" : "/html/body//a[@href = 'linux/']/@href",
"subdirs" : [""],
"page_pattern" : "/html/body//a[regex:test(@href, '^linux-(image|headers)-[3-9].*-generic.*amd64.deb$')]/@href"
},

{
"root" : "https://mirrors.kernel.org/ubuntu/pool/main/l/",
"discovery_pattern" : "/html/body//a[@href = 'linux/']/@href",
"subdirs" : [""],
"page_pattern" : "/html/body//a[regex:test(@href, '^linux-headers-[3-9].*_all.deb$')]/@href"
},

{
"root" : "http://security.ubuntu.com/ubuntu/pool/main/l/",
"discovery_pattern" : "/html/body//a[@href = 'linux/']/@href",
"subdirs" : [""],
"page_pattern" : "/html/body//a[regex:test(@href, '^linux-(image|headers)-[3-9].*-generic.*amd64.deb$')]/@href"
},

{
"root" : "http://security.ubuntu.com/ubuntu/pool/main/l/",
"discovery_pattern" : "/html/body//a[@href = 'linux/']/@href",
"subdirs" : [""],
"page_pattern" : "/html/body//a[regex:test(@href, '^linux-headers-[3-9].*_all.deb$')]/@href"
}
],

"Fedora" : [
{
"root" : "https://mirrors.kernel.org/fedora/releases/",
"discovery_pattern": "/html/body//a[regex:test(@href, '^2[2-9]/$')]/@href",
"subdirs" : [
"Everything/x86_64/os/Packages/k/"
],
"page_pattern" : "/html/body//a[regex:test(@href, '^kernel-(core|devel)-[0-9].*\.rpm$')]/@href"
},

{
"root" : "https://mirrors.kernel.org/fedora/updates/",
"discovery_pattern": "/html/body//a[regex:test(@href, '^2[2-9]/$')]/@href",
"subdirs" : [
"x86_64/k/"
],
"page_pattern" : "/html/body//a[regex:test(@href, '^kernel-(core|devel)-[0-9].*\.rpm$')]/@href"
},

# {
# "root" : "https://mirrors.kernel.org/fedora/development/",
# "discovery_pattern": "/html/body//a[regex:test(@href, '^2[2-9]/$')]/@href",
# "subdirs" : [
# "x86_64/os/Packages/k/"
# ],
# "page_pattern" : "/html/body//a[regex:test(@href, '^kernel-(core|devel)-[0-9].*\.rpm$')]/@href"
# }
],

"CoreOS" : [
{
"root" : "http://alpha.release.core-os.net/",
"discovery_pattern": "/html/body//a[regex:test(@href, 'amd64-usr')]/@href",
"subdirs" : [
""
],
"page_pattern" : "/html/body//a[regex:test(@href, '^[5-9][0-9][0-9]|current|[1][0-9]{3}')]/@href"
},

{
"root" : "http://beta.release.core-os.net/",
"discovery_pattern": "/html/body//a[regex:test(@href, 'amd64-usr')]/@href",
"subdirs" : [
""
],
"page_pattern" : "/html/body//a[regex:test(@href, '^[5-9][0-9][0-9]|current|[1][0-9]{3}')]/@href"
},

{
"root" : "http://stable.release.core-os.net/",
"discovery_pattern": "/html/body//a[regex:test(@href, 'amd64-usr')]/@href",
"subdirs" : [
""
],
"page_pattern" : "/html/body//a[regex:test(@href, '^[4-9][0-9][0-9]|current|[1][0-9]{3}')]/@href"
}
],

"Debian": [
{
"root": "https://mirrors.kernel.org/debian/pool/main/l/",
"discovery_pattern": "/html/body/pre/a[@href = 'linux/']/@href",
Expand Down Expand Up @@ -189,8 +185,8 @@
URL_TIMEOUT=30

if len(sys.argv) < 2 or not sys.argv[1] in repos:
sys.stderr.write("Usage: " + sys.argv[0] + " <distro>\n")
sys.exit(1)
sys.stderr.write("Usage: " + sys.argv[0] + " <distro>\n")
sys.exit(1)

#
# Navigate the `repos` tree and look for packages we need that match the
Expand All @@ -199,22 +195,29 @@
for repo in repos[sys.argv[1]]:
try:
root = urllib2.urlopen(repo["root"],timeout=URL_TIMEOUT).read()
versions = html.fromstring(root).xpath(repo["discovery_pattern"], namespaces = {"regex": "http://exslt.org/regular-expressions"})
for version in versions:
for subdir in repo["subdirs"]:
except:
continue
versions = html.fromstring(root).xpath(repo["discovery_pattern"], namespaces = {"regex": "http://exslt.org/regular-expressions"})
for version in versions:
for subdir in repo["subdirs"]:
# The try - except block is used because 404 errors and similar
# might happen (and actually happen because not all repos have
# packages we need)
try:
source = repo["root"] + version + subdir
page = urllib2.urlopen(source,timeout=URL_TIMEOUT).read()
rpms = html.fromstring(page).xpath(repo["page_pattern"], namespaces = {"regex": "http://exslt.org/regular-expressions"})

for rpm in rpms:
if "exclude_patterns" in repo and any(x in rpm for x in repo["exclude_patterns"]):
continue
else:
urls.add(source + str(urllib2.unquote(rpm)))
except:
continue
except:
continue

#
# Print URLs to stdout
#
for url in urls:
print(url)
print(url)
16 changes: 15 additions & 1 deletion userspace/libscap/scap-int.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,20 @@ struct scap
struct ppm_proclist_info* m_driver_procinfo;
bool refresh_proc_table_when_saving;
uint32_t m_fd_lookup_limit;
uint64_t m_unexpected_block_readsize;
};

typedef enum ppm_dumper_type
{
DT_FILE = 0,
DT_MEM = 0,
}ppm_dumper_type;

struct scap_dumper
{
gzFile m_f;
ppm_dumper_type m_type;
uint64_t m_off;
};

struct scap_ns_socket_list
Expand Down Expand Up @@ -153,7 +167,7 @@ int32_t scap_fd_info_to_string(scap_fdinfo* fdi, OUT char* str, uint32_t strlen)
// Calculate the length on disk of an fd entry's info
uint32_t scap_fd_info_len(scap_fdinfo* fdi);
// Write the given fd info to disk
int32_t scap_fd_write_to_disk(scap_t* handle, scap_fdinfo* fdi, gzFile f);
int32_t scap_fd_write_to_disk(scap_t* handle, scap_fdinfo* fdi, scap_dumper_t* dumper);
// Populate the given fd by reading the info from disk
uint32_t scap_fd_read_from_disk(scap_t* handle, OUT scap_fdinfo* fdi, OUT size_t* nbytes, gzFile f);
// Parse the headers of a trace file and load the tables
Expand Down
Loading

0 comments on commit db38999

Please sign in to comment.