-
Notifications
You must be signed in to change notification settings - Fork 523
/
pkg.rs
364 lines (325 loc) · 14.6 KB
/
pkg.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
// Copyright 2016 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::dist;
use std::io;
use std::fs;
use std::path::{Component, Path, PathBuf};
use std::str;
use crate::errors::*;
pub use self::toolchain_imp::*;
pub trait ToolchainPackager: Send {
fn write_pkg(self: Box<Self>, f: fs::File) -> Result<()>;
}
pub trait InputsPackager: Send {
fn write_inputs(self: Box<Self>, wtr: &mut dyn io::Write) -> Result<dist::PathTransformer>;
}
pub trait OutputsRepackager {
fn repackage_outputs(self: Box<Self>, wtr: &mut dyn io::Write) -> Result<dist::PathTransformer>;
}
#[cfg(not(all(target_os = "linux", target_arch = "x86_64")))]
mod toolchain_imp {
use std::fs;
use super::ToolchainPackager;
use crate::errors::*;
// Distributed client, but an unsupported platform for toolchain packaging so
// create a failing implementation that will conflict with any others.
impl<T: Send> ToolchainPackager for T {
fn write_pkg(self: Box<Self>, _f: fs::File) -> Result<()> {
bail!("Automatic packaging not supported on this platform")
}
}
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
mod toolchain_imp {
use std::collections::BTreeMap;
use std::io::{Read, Write};
use std::fs;
use std::path::{Component, Path, PathBuf};
use std::process;
use std::str;
use super::tarify_path;
use walkdir::WalkDir;
use crate::errors::*;
pub struct ToolchainPackageBuilder {
// Put dirs and file in a deterministic order (map from tar_path -> real_path)
dir_set: BTreeMap<PathBuf, PathBuf>,
file_set: BTreeMap<PathBuf, PathBuf>,
}
impl ToolchainPackageBuilder {
pub fn new() -> Self {
ToolchainPackageBuilder { dir_set: BTreeMap::new(), file_set: BTreeMap::new() }
}
pub fn add_common(&mut self) -> Result<()> {
self.add_dir(PathBuf::from("/tmp"))
}
pub fn add_executable_and_deps(&mut self, executable: PathBuf) -> Result<()> {
let mut remaining = vec![executable.to_owned()];
while let Some(obj_path) = remaining.pop() {
assert!(obj_path.is_absolute());
let tar_path = tarify_path(&obj_path)?;
// If file already in the set, assume we've analysed all deps
if self.file_set.contains_key(&tar_path) {
continue
}
let ldd_libraries = find_ldd_libraries(&obj_path)
.chain_err(|| format!("Failed to analyse {} with ldd", obj_path.display()))?;
remaining.extend(ldd_libraries);
self.file_set.insert(tar_path, obj_path);
}
Ok(())
}
pub fn add_dir(&mut self, dir_path: PathBuf) -> Result<()> {
assert!(dir_path.is_absolute());
if !dir_path.is_dir() {
bail!(format!("{} was not a dir when readying for tar", dir_path.to_string_lossy()))
}
if dir_path.components().next_back().expect("asserted absolute") == Component::RootDir {
return Ok(())
}
let tar_path = tarify_path(&dir_path)?;
self.dir_set.insert(tar_path, dir_path);
Ok(())
}
pub fn add_file(&mut self, file_path: PathBuf) -> Result<()> {
assert!(file_path.is_absolute());
if !file_path.is_file() {
bail!(format!("{} was not a file when readying for tar", file_path.to_string_lossy()))
}
let tar_path = tarify_path(&file_path)?;
self.file_set.insert(tar_path, file_path);
Ok(())
}
pub fn add_dir_contents(&mut self, dir_path: &Path) -> Result<()> {
// Although by not following symlinks we could break a custom
// constructed toolchain with links everywhere, this is just a
// best-effort auto packaging
for entry in WalkDir::new(&dir_path).follow_links(false) {
let entry = entry?;
let file_type = entry.file_type();
if file_type.is_dir() {
continue
} else if file_type.is_symlink() {
let metadata = fs::metadata(entry.path())?;
if !metadata.file_type().is_file() {
continue
}
} else if !file_type.is_file() {
// Device or other oddity
continue
}
trace!("walkdir add_file {}", entry.path().display());
// It's either a file, or a symlink pointing to a file
self.add_file(entry.path().to_owned())?
}
Ok(())
}
pub fn into_compressed_tar<W: Write>(self, writer: W) -> Result<()> {
use flate2::write::GzEncoder;
let ToolchainPackageBuilder { dir_set, file_set } = self;
let mut builder = tar::Builder::new(GzEncoder::new(writer, flate2::Compression::default()));
for (tar_path, dir_path) in dir_set.into_iter() {
builder.append_dir(tar_path, dir_path)?
}
for (tar_path, file_path) in file_set.into_iter() {
let file = &mut fs::File::open(file_path)?;
builder.append_file(tar_path, file)?
}
builder.finish().map_err(Into::into)
}
}
// The dynamic linker is the only thing that truly knows how dynamic libraries will be
// searched for, so we need to ask it directly.
//
// This function will extract any absolute paths from output like the following:
// $ ldd /bin/ls
// linux-vdso.so.1 => (0x00007ffeb41f6000)
// libselinux.so.1 => /lib/x86_64-linux-gnu/libselinux.so.1 (0x00007f6877f4f000)
// libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f6877b85000)
// libpcre.so.3 => /lib/x86_64-linux-gnu/libpcre.so.3 (0x00007f6877915000)
// libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f6877711000)
// /lib64/ld-linux-x86-64.so.2 (0x00007f6878171000)
// libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f68774f4000)
//
// Elf executables can be statically or dynamically linked, and position independant (PIE) or not:
// - dynamic + PIE = ET_DYN, ldd stdouts something like the list above and exits with code 0
// - dynamic + non-PIE = ET_EXEC, ldd stdouts something like the list above and exits with code 0
// - static + PIE = ET_DYN, ldd stdouts something like "\tstatically linked" or
// "\tldd (0x7f79ef662000)" and exits with code 0
// - static + non-PIE = ET_EXEC, ldd stderrs something like "\tnot a dynamic executable" or
// "ldd: a.out: Not a valid dynamic program" and exits with code 1
//
fn find_ldd_libraries(executable: &Path) -> Result<Vec<PathBuf>> {
let process::Output { status, stdout, stderr } = process::Command::new("ldd").arg(executable).output()?;
// Not a file ldd can handle. This can be a non-executable, or a static non-PIE
if !status.success() {
// Best-effort detection of static non-PIE
let mut elf = fs::File::open(executable)?;
let mut elf_bytes = [0; 0x12];
elf.read_exact(&mut elf_bytes)?;
if elf_bytes[..0x4] != [0x7f, 0x45, 0x4c, 0x46] {
bail!("Elf magic not found")
}
let little_endian = match elf_bytes[0x5] {
1 => true,
2 => false,
_ => bail!("Invalid endianness in elf header"),
};
let e_type = if little_endian {
(elf_bytes[0x11] as u16) << 8 | elf_bytes[0x10] as u16
} else {
(elf_bytes[0x10] as u16) << 8 | elf_bytes[0x11] as u16
};
if e_type != 0x02 {
bail!("ldd failed on a non-ET_EXEC elf")
}
// It appears to be an ET_EXEC, good enough for us
return Ok(vec![])
}
if !stderr.is_empty() {
trace!("ldd stderr non-empty: {:?}", String::from_utf8_lossy(&stderr))
}
let stdout = str::from_utf8(&stdout).map_err(|_| "ldd output not utf8")?;
Ok(parse_ldd_output(stdout))
}
// If it's a static PIE the output will be a line like "\tstatically linked", so be forgiving
// in the parsing here and treat parsing oddities as an empty list.
fn parse_ldd_output(stdout: &str) -> Vec<PathBuf> {
let mut libs = vec![];
for line in stdout.lines() {
let line = line.trim();
let mut parts: Vec<_> = line.split_whitespace().collect();
// Remove a possible "(0xdeadbeef)" or assume this isn't a library line
match parts.pop() {
Some(s) if s.starts_with('(') && s.ends_with(')') => (),
Some(_) |
None => continue,
}
if parts.len() > 3 {
continue
}
let libpath = match (parts.get(0), parts.get(1), parts.get(2)) {
// "linux-vdso.so.1 => (0x00007ffeb41f6000)"
(Some(_libname), Some(&"=>"), None) => continue,
// "libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f6877b85000)"
(Some(_libname), Some(&"=>"), Some(libpath)) => PathBuf::from(libpath),
// "/lib64/ld-linux-x86-64.so.2 (0x00007f6878171000)"
(Some(libpath), None, None) => PathBuf::from(libpath),
_ => continue,
};
if !libpath.is_absolute() {
continue
}
libs.push(libpath)
}
libs
}
#[test]
fn test_ldd_parse() {
let ubuntu_ls_output = "\tlinux-vdso.so.1 => (0x00007fffcfffe000)
\tlibselinux.so.1 => /lib/x86_64-linux-gnu/libselinux.so.1 (0x00007f69caa6b000)
\tlibc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f69ca6a1000)
\tlibpcre.so.3 => /lib/x86_64-linux-gnu/libpcre.so.3 (0x00007f69ca431000)
\tlibdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f69ca22d000)
\t/lib64/ld-linux-x86-64.so.2 (0x00007f69cac8d000)
\tlibpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f69ca010000)
";
assert_eq!(parse_ldd_output(ubuntu_ls_output).iter().map(|p| p.to_str().unwrap()).collect::<Vec<_>>(), &[
"/lib/x86_64-linux-gnu/libselinux.so.1",
"/lib/x86_64-linux-gnu/libc.so.6",
"/lib/x86_64-linux-gnu/libpcre.so.3",
"/lib/x86_64-linux-gnu/libdl.so.2",
"/lib64/ld-linux-x86-64.so.2",
"/lib/x86_64-linux-gnu/libpthread.so.0",
])
}
#[test]
fn test_ldd_parse_static() {
let static_outputs = &[
"\tstatically linked", // glibc ldd output
"\tldd (0x7f79ef662000)", // musl ldd output
];
for static_output in static_outputs {
assert_eq!(parse_ldd_output(static_output).len(), 0)
}
}
}
pub fn make_tar_header(src: &Path, dest: &str) -> io::Result<tar::Header> {
let metadata_res = fs::metadata(&src);
let mut file_header = tar::Header::new_ustar();
// TODO: test this works
if let Ok(metadata) = metadata_res {
// TODO: if the source file is a symlink, I think this does bad things
file_header.set_metadata(&metadata);
} else {
warn!("Couldn't get metadata of file {:?}, falling back to some defaults", src);
file_header.set_mode(0o644);
file_header.set_uid(0);
file_header.set_gid(0);
file_header.set_mtime(0);
file_header.set_device_major(0).expect("expected a ustar header");
file_header.set_device_minor(0).expect("expected a ustar header");
file_header.set_entry_type(tar::EntryType::file());
}
// tar-rs imposes that `set_path` takes a relative path
assert!(dest.starts_with("/"));
let dest = dest.trim_start_matches("/");
assert!(!dest.starts_with("/"));
// `set_path` converts its argument to a Path and back to bytes on Windows, so this is
// a bit of an inefficient round-trip. Windows path separators will also be normalised
// to be like Unix, and the path is (now) relative so there should be no funny results
// due to Windows
// TODO: should really use a `set_path_str` or similar
file_header.set_path(&dest)?;
Ok(file_header)
}
/// Simplify the path and strip the leading slash
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn tarify_path(path: &Path) -> Result<PathBuf> {
let final_path = simplify_path(path)?;
let mut components = final_path.components();
assert_eq!(components.next(), Some(Component::RootDir));
Ok(components.as_path().to_owned())
}
/// Simplify a path to one without any relative components, erroring if it looks
/// like there could be any symlink complexity that means a simplified path is not
/// equivalent to the original (see the documentation of `fs::canonicalize` for an
/// example).
///
/// So why avoid resolving symlinks? Any path that we are trying to simplify has
/// (usually) been added to an archive because something will try access it, but
/// resolving symlinks (be they for the actual file or directory components) can
/// make the accessed path 'disappear' in favour of the canonical path.
pub fn simplify_path(path: &Path) -> Result<PathBuf> {
let mut final_path = PathBuf::new();
for component in path.components() {
match component {
c @ Component::RootDir |
c @ Component::Prefix(_) |
c @ Component::Normal(_) => final_path.push(c),
Component::ParentDir => {
// If the path is doing funny symlink traversals, just give up
let is_symlink = fs::symlink_metadata(&final_path)
.chain_err(|| "Missing directory while simplifying path")?
.file_type()
.is_symlink();
if is_symlink {
bail!("Cannot handle symlinks in parent paths")
}
final_path.pop();
},
Component::CurDir => continue,
}
}
Ok(final_path)
}