From 0459a8b0621b887ef7bdc8881d9592bb8256fb35 Mon Sep 17 00:00:00 2001 From: Vasili Novikov Date: Sat, 17 Aug 2019 01:36:49 +0200 Subject: [PATCH] implement upstream diffs, implement shellcheck * stop ever relying on current dir, current environment etc. This opens possibility for future concurrent workflow. * add logging * stop enforcing of target/ being the PKGDEST. Use more flexible by-file tar review. * resolve dependencies via RAUR library instead of relying on .SRCINFO-s * add a separate `shellcheck` command for RUA, with built-in handling of PKGBUILD variables * move all `include_bytes!` macros to one place fixes GH-1, fixes GH-41, replaces GH-37, supersedes GH-42. --- Cargo.lock | 7 + Cargo.toml | 1 + README.md | 10 +- res/shellcheck-wrapper | 64 +++++++ res/{wrap_args.sh => wrap_args.sh.example} | 0 src/action_install.rs | 192 +++++++++++++++------ src/action_jailbuild.rs | 16 +- src/aur_download.rs | 108 ------------ src/cli_args.rs | 7 + src/git_utils.rs | 70 ++++++++ src/main.rs | 25 ++- src/pacman.rs | 2 +- src/print_format.rs | 2 +- src/print_package_info.rs | 4 +- src/reviewing.rs | 70 ++++++++ src/rua_dirs.rs | 11 -- src/rua_files.rs | 31 ++++ src/tar_check.rs | 24 ++- src/terminal_util.rs | 11 +- src/wrapped.rs | 171 ++++++++---------- 20 files changed, 515 insertions(+), 311 deletions(-) create mode 100644 res/shellcheck-wrapper rename res/{wrap_args.sh => wrap_args.sh.example} (100%) delete mode 100644 src/aur_download.rs create mode 100644 src/git_utils.rs create mode 100644 src/reviewing.rs delete mode 100644 src/rua_dirs.rs create mode 100644 src/rua_files.rs diff --git a/Cargo.lock b/Cargo.lock index c0d1454..ab1460d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -468,6 +468,11 @@ dependencies = [ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fs_extra" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -1333,6 +1338,7 @@ dependencies = [ "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "libalpm-fork 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2021,6 +2027,7 @@ dependencies = [ "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" "checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" "checksum fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +"checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" diff --git a/Cargo.toml b/Cargo.toml index 83bc02c..43a1ee2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ directories = "2.0.1" env_logger = "0.6.2" failure = "0.1.5" fs2 = "0.4.3" +fs_extra = "1.1.0" itertools = "0.8.0" lazy_static = "1.3.0" libalpm = { package = "libalpm-fork", version = "0.1.4" } diff --git a/README.md b/README.md index 3430320..20871ca 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,15 @@ RUA is a build tool for ArchLinux, AUR. Its features: -- Uses a namespace [jail](https://github.com/projectatomic/bubblewrap) to build packages: +- Uses a security namespace [jail](https://github.com/projectatomic/bubblewrap) to build packages: * supports "offline" builds (network namespace) * builds in isolated filesystem, see [safety](#Safety) section below * PKGBUILD script is run under seccomp rules (e.g. the build cannot call `ptrace`) * filesystem is mounted with "nosuid" (e.g. the build cannot call `sudo`) -- Show the user what they are about to install: - * warn if SUID files are present, and show them +- Provides detailed information: + * upstream diff is shown before building, or full diff if the package is new + * warn if SUID files are present in the already built package, and show them + * see code problems in PKGBUILD via `shellcheck` (taking care of special variables) * show INSTALL script (if present), executable and file list preview - Minimize user interaction: * verify all PKGBUILD-s once, build without interruptions @@ -28,6 +30,8 @@ Planned features include AUR upstream git diff and local patch application. `rua info xcalib freecad` # shows information on packages +`rua shellcheck path/to/my/PKGBUILD` # run `shellcheck` on a PKGBUILD, discovering potential problems with the build instruction. Takes special care for PKGBUILD variables. + `rua tarcheck xcalib.pkg.tar` # if you already have a *.pkg.tar package built, run RUA checks on it (SUID, executable list, INSTALL script review etc). `rua jailbuild --offline /path/to/pkgbuild/directory` # build a directory. Don't fetch any dependencies. Assumes a clean directory. diff --git a/res/shellcheck-wrapper b/res/shellcheck-wrapper new file mode 100644 index 0000000..0046442 --- /dev/null +++ b/res/shellcheck-wrapper @@ -0,0 +1,64 @@ +#!/bin/bash -euET + +## use as: +## shellcheck --check-sourced --norc -x < shellcheck-wrapper + +# declare variables used by PKGBUILD +srcdir= +pkgdir= + +# source it! +source PKGBUILD + +# ensure that obligatory PKGBUILD values are defined, and avoid "unused" warning for them: +test "${#pkgname[@]}" -gt 0 +test "${#pkgver[@]}" -gt 0 +test "${#pkgrel[@]}" -gt 0 +test "${#arch[@]}" -gt 0 + +# avoid "unused" warning for optional PKGBUILD variables: +export epoch +export pkgdesc +export url +export license +export install +export changelog +export source +export source_x86_64 +export source_i686 +export validpgpkeys +export noextract +export md5sums +export sha1sums +export sha224sums +export sha256sums +export sha384sums +export sha512sums +export groups +export backup +export depends +export depends_x86_64 +export depends_i686 +export makedepends +export makedepends_x86_64 +export makedepends_i686 +export checkdepends +export checkdepends_x86_64 +export checkdepends_i686 +export optdepends +export optdepends_x86_64 +export optdepends_i686 +export conflicts +export conflicts_x86_64 +export conflicts_i686 +export provides +export provides_x86_64 +export provides_i686 +export replaces +export replaces_x86_64 +export replaces_i686 +export options + +# avoid "unused" warning for variables defined _for_ PKGBUILD +export srcdir +export pkgdir diff --git a/res/wrap_args.sh b/res/wrap_args.sh.example similarity index 100% rename from res/wrap_args.sh rename to res/wrap_args.sh.example diff --git a/src/action_install.rs b/src/action_install.rs index bd6a72d..15d1a14 100644 --- a/src/action_install.rs +++ b/src/action_install.rs @@ -1,15 +1,22 @@ -use crate::rua_dirs::CHECKED_TARS; -use crate::rua_dirs::REVIEWED_BUILD_DIR; -use crate::rua_dirs::TARGET_SUBDIR; -use crate::tar_check; -use crate::{aur_download, wrapped}; use crate::{pacman, terminal_util}; +use crate::{reviewing, wrapped}; +use crate::{rua_files, tar_check}; +use core::cmp; use directories::ProjectDirs; +use fs_extra::dir::CopyOptions; use itertools::Itertools; +use lazy_static::lazy_static; +use libalpm::Alpm; +use log::debug; +use log::info; +use log::trace; +use raur::Package; +use regex::Regex; +use std::collections::HashMap; +use std::collections::HashSet; use std::fs; - -use std::collections::{HashMap, HashSet}; +use std::fs::ReadDir; use std::path::PathBuf; pub fn install(targets: Vec, dirs: &ProjectDirs, is_offline: bool, asdeps: bool) { @@ -17,9 +24,8 @@ pub fn install(targets: Vec, dirs: &ProjectDirs, is_offline: bool, asdep let mut aur_packages = HashMap::new(); let alpm = pacman::create_alpm(); for install_target in targets { - wrapped::prefetch_aur( + resolve_dependencies( &install_target, - dirs, &mut pacman_deps, &mut aur_packages, 0, @@ -29,7 +35,10 @@ pub fn install(targets: Vec, dirs: &ProjectDirs, is_offline: bool, asdep pacman_deps.retain(|name| !pacman::is_package_installed(&alpm, name)); show_install_summary(&pacman_deps, &aur_packages); for name in aur_packages.keys() { - aur_download::review_repo(name, dirs); + let dir = rua_files::review_dir(dirs, name); + fs::create_dir_all(&dir) + .unwrap_or_else(|err| panic!("Failed to create repository dir for {}, {}", name, err)); + reviewing::review_repo(&dir, name, dirs); } pacman::ensure_pacman_packages_installed(pacman_deps); install_all(dirs, aur_packages, is_offline, asdeps); @@ -53,7 +62,7 @@ fn show_install_summary(pacman_deps: &HashSet, aur_packages: &HashMap, offline: bool for (depth, packages) in &packages.iter().group_by(|pair| *pair.1) { let packages: Vec<_> = packages.map(|pair| pair.0).collect(); for name in &packages { + let review_dir = rua_files::review_dir(dirs, name); + let build_dir = rua_files::build_dir(dirs, name); + rm_rf::force_remove_all(&build_dir, true).expect("Failed to remove old build dir"); + std::fs::create_dir_all(&build_dir).expect("Failed to create build dir"); + fs_extra::copy_items( + &vec![review_dir], + rua_files::global_build_dir(dirs), + &CopyOptions::new(), + ) + .expect("failed to copy reviewed dir to build dir"); + rm_rf::force_remove_all(build_dir.join(".git"), true).expect("Failed to remove .git"); wrapped::build_directory( - dirs.cache_dir() - .join(&name) - .join(REVIEWED_BUILD_DIR) - .to_str() - .unwrap_or_else(|| { - panic!( - "{}:{} Failed to resolve build path for {}", - file!(), - line!(), - name - ) - }), + &build_dir.to_str().expect("Non-UTF8 directory name"), dirs, offline, ); @@ -86,9 +95,9 @@ fn install_all(dirs: &ProjectDirs, packages: HashMap, offline: bool for name in &packages { check_tars_and_move(name, dirs); } - let mut packages_to_install: Vec<(String, PathBuf)> = Vec::new(); - for name in packages { - let checked_tars = dirs.cache_dir().join(name).join(CHECKED_TARS); + let mut files_to_install: Vec<(String, PathBuf)> = Vec::new(); + for name in &packages { + let checked_tars = rua_files::checked_tars_dir(dirs, &name); let read_dir_iterator = fs::read_dir(checked_tars).unwrap_or_else(|e| { panic!( "Failed to read 'checked_tars' directory for {}, {}", @@ -96,53 +105,124 @@ fn install_all(dirs: &ProjectDirs, packages: HashMap, offline: bool ) }); for file in read_dir_iterator { - packages_to_install.push(( - name.to_owned(), + files_to_install.push(( + name.to_string(), file.expect("Failed to open file for tar_check analysis") .path(), )); } } - pacman::ensure_aur_packages_installed(packages_to_install, asdeps || depth > 0); + pacman::ensure_aur_packages_installed(files_to_install, asdeps || depth > 0); } } pub fn check_tars_and_move(name: &str, dirs: &ProjectDirs) { - let build_target_dir = dirs - .cache_dir() - .join(name) - .join(REVIEWED_BUILD_DIR) - .join(TARGET_SUBDIR); - let checked_tars_dir = dirs.cache_dir().join(name).join(CHECKED_TARS); - rm_rf::force_remove_all(&checked_tars_dir, true).unwrap_or_else(|err| { - panic!( - "{}:{} Failed to clean checked tar files dir {:?}, {}", - file!(), - line!(), - CHECKED_TARS, - err, - ) - }); - let target_dir = fs::read_dir(&build_target_dir); - let target_dir = target_dir.unwrap_or_else(|err| { + debug!("{}:{} checking tars for package {}", file!(), line!(), name); + let build_dir = rua_files::build_dir(dirs, name); + let dir_items: ReadDir = build_dir.read_dir().unwrap_or_else(|err| { panic!( - "target directory not found for package {}: {:?}. \ - \nDoes the PKGBUILD respect the environment variable PKGDEST ?\ - \n{}", - name, &build_target_dir, err, + "Failed to read directory contents for {:?}, {}", + &build_dir, err ) }); - for file in target_dir { + let checked_files = dir_items.flat_map(|file| { tar_check::tar_check( &file .expect("Failed to open file for tar_check analysis") .path(), - ); - } - fs::rename(&build_target_dir, &checked_tars_dir).unwrap_or_else(|e| { + ) + }); + debug!("all package (tar) files checked, moving them",); + let checked_tars_dir = rua_files::checked_tars_dir(dirs, name); + rm_rf::force_remove_all(&checked_tars_dir, true).unwrap_or_else(|err| { panic!( - "Failed to move {:?} (build artifacts) to {:?} for package {}, {}", - &build_target_dir, &checked_tars_dir, name, e, + "Failed to clean checked tar files dir {:?}, {}", + checked_tars_dir, err, ) }); + fs::create_dir_all(&checked_tars_dir).unwrap_or_else(|err| { + panic!( + "Failed to create checked_tars dir {:?}, {}", + &checked_tars_dir, err + ); + }); + + for file in checked_files { + let file_name = file.file_name().expect("Failed to parse package tar name"); + let file_name = file_name + .to_str() + .expect("Non-UTF8 characters in tar file name"); + fs::rename(&file, checked_tars_dir.join(file_name)).unwrap_or_else(|e| { + panic!( + "Failed to move {:?} (build artifact) to {:?}, {}", + &file, &checked_tars_dir, e, + ) + }); + } +} + +/// Check that the package name is easy to work with in shell +fn check_package_name(name: &str) { + lazy_static! { + static ref NAME_REGEX: Regex = Regex::new(r"[a-zA-Z][a-zA-Z._-]*") + .unwrap_or_else(|_| panic!("{}:{} Failed to parse regexp", file!(), line!())); + } + if !NAME_REGEX.is_match(name) { + eprintln!("Unexpected package name {}", name); + std::process::exit(1) + } +} + +fn resolve_dependencies( + name: &str, + pacman_deps: &mut HashSet, + aur_packages: &mut HashMap, + depth: i32, + alpm: &Alpm, +) { + check_package_name(&name); + if let Some(old_depth) = aur_packages.get(name) { + let old_depth = *old_depth; + aur_packages.insert(name.to_owned(), cmp::max(depth + 1, old_depth)); + info!("Skipping already resolved package {}", name); + } else { + aur_packages.insert(name.to_owned(), depth); + let info = raur_info(&name); + let deps = info + .depends + .iter() + .chain(info.make_depends.iter()) + .collect::>(); + for dep in deps.into_iter() { + if pacman::is_package_installed(alpm, &dep) { + // skip if already installed + } else if !pacman::is_package_installable(alpm, &dep) { + info!( + "{} depends on AUR package {}. Trying to resolve it...", + name, &dep + ); + resolve_dependencies(&dep, pacman_deps, aur_packages, depth + 1, alpm); + } else { + pacman_deps.insert(dep.to_owned()); + } + } + } +} + +fn raur_info(pkg: &str) -> Package { + trace!( + "{}:{} Fetching AUR information for package {}", + file!(), + line!(), + pkg + ); + let info = raur::info(&[pkg]); + let info = info.unwrap_or_else(|e| panic!("Failed to fetch info for package {}, {}", &pkg, e)); + match info.into_iter().next() { + Some(pkg) => pkg, + None => { + eprintln!("Package {} not found in AUR", pkg); + std::process::exit(1) + } + } } diff --git a/src/action_jailbuild.rs b/src/action_jailbuild.rs index 8f07993..101cf8a 100644 --- a/src/action_jailbuild.rs +++ b/src/action_jailbuild.rs @@ -1,23 +1,19 @@ -use crate::rua_dirs::TARGET_SUBDIR; use crate::{tar_check, wrapped}; use directories::ProjectDirs; -use std::fs; use std::path::PathBuf; -pub fn action_jailbuild(offline: bool, target: PathBuf, dirs: &ProjectDirs) { - let target_str = target +pub fn action_jailbuild(offline: bool, dir: PathBuf, dirs: &ProjectDirs) { + let dir_str = dir .to_str() .unwrap_or_else(|| panic!("{}:{} Cannot parse CLI target directory", file!(), line!())); - wrapped::build_directory(target_str, &dirs, offline); - for file in fs::read_dir(TARGET_SUBDIR).expect("'target' directory not found") { + wrapped::build_directory(dir_str, &dirs, offline); + for file in dir.read_dir().expect("'target' directory not found") { tar_check::tar_check( &file .expect("Failed to open file for tar_check analysis") .path(), ); } - eprintln!( - "Package built and checked in: {:?}", - target.join(TARGET_SUBDIR) - ); + eprintln!("Package built and checked in: {}", dir_str); + eprintln!("If you want to install the built artifacts, do it manually."); } diff --git a/src/aur_download.rs b/src/aur_download.rs deleted file mode 100644 index 6f53942..0000000 --- a/src/aur_download.rs +++ /dev/null @@ -1,108 +0,0 @@ -use crate::rua_dirs::PREFETCH_DIR; -use crate::rua_dirs::REVIEWED_BUILD_DIR; -use crate::terminal_util; - -use std::path::Path; -use std::process::{Command, Output}; -use std::{env, fs}; - -use directories::ProjectDirs; -use lazy_static::lazy_static; -use regex::Regex; -use rm_rf; - -fn assert_command_success(command: &Output) { - assert!( - command.status.success(), - "Command failed with exit code {:?}\nStderr: {}\nStdout: {}", - command.status.code(), - String::from_utf8_lossy(&command.stderr), - String::from_utf8_lossy(&command.stdout), - ); -} - -pub fn fresh_download(name: &str, dirs: &ProjectDirs) { - lazy_static! { - static ref NAME_REGEX: Regex = Regex::new(r"[a-zA-Z][a-zA-Z._-]*") - .unwrap_or_else(|_| panic!("{}:{} Failed to parse regexp", file!(), line!())); - } - assert!( - NAME_REGEX.is_match(name), - "{}:{} unexpected package name {}", - file!(), - line!(), - name - ); - let path = dirs.cache_dir().join(name).join(PREFETCH_DIR); - rm_rf::force_remove_all(&path, true).unwrap_or_else(|err| { - panic!( - "{}:{} Failed to clean cache dir {:?}, {}", - file!(), - line!(), - path, - err, - ) - }); - fs::create_dir_all(dirs.cache_dir().join(name)) - .unwrap_or_else(|err| panic!("Failed to create cache dir for {}, {}", name, err)); - env::set_current_dir(dirs.cache_dir().join(name)) - .unwrap_or_else(|err| panic!("Failed to cd into {}, {}", name, err)); - let git_http_ref = format!("https://aur.archlinux.org/{}.git", name); - let command = Command::new("git") - .args(&["clone", &git_http_ref, PREFETCH_DIR]) - .output() - .unwrap_or_else(|err| panic!("Failed to git-clone repository {}, {}", name, err)); - assert_command_success(&command); - assert!( - Path::new(PREFETCH_DIR).join(".SRCINFO").exists(), - "Repository {} does not have an SRCINFO file. Does this package exist in AUR?", - name - ); -} - -pub fn review_repo(name: &str, dirs: &ProjectDirs) { - env::set_current_dir(dirs.cache_dir().join(name).join(PREFETCH_DIR)) - .unwrap_or_else(|err| panic!("Failed to cd into build dir for {}, {}", name, err)); - loop { - eprint!( - "Verifying package {}. [V]=view PKGBUILD, [E]=edit PKGBUILD, \ - [I]=run shell to inspect, [O]=ok, use package: ", - name - ); - let string = terminal_util::console_get_line(); - - if string == "v" { - terminal_util::run_env_command("PAGER", "less", &["PKGBUILD"]); - } else if string == "e" { - terminal_util::run_env_command("EDITOR", "nano", &["PKGBUILD"]); - } else if string == "i" { - eprintln!("Exit the shell with `logout` or Ctrl-D..."); - terminal_util::run_env_command("SHELL", "bash", &[]); - } else if string == "o" { - break; - } - } - env::set_current_dir("..").unwrap_or_else(|err| { - panic!( - "{}:{} Failed to move to parent repo after review, {}", - file!(), - line!(), - err, - ) - }); - rm_rf::force_remove_all(REVIEWED_BUILD_DIR, true).unwrap_or_else(|err| { - panic!( - "{}:{} Failed to clean build dir {:?}, {}", - file!(), - line!(), - REVIEWED_BUILD_DIR, - err, - ) - }); - fs::rename(PREFETCH_DIR, REVIEWED_BUILD_DIR).unwrap_or_else(|err| { - panic!( - "Failed to move temporary directory '{}' to 'build', {}", - PREFETCH_DIR, err, - ) - }); -} diff --git a/src/cli_args.rs b/src/cli_args.rs index 46137d8..ba450b2 100644 --- a/src/cli_args.rs +++ b/src/cli_args.rs @@ -62,6 +62,13 @@ pub enum Action { #[structopt(help = "Target to show for", multiple = true, required = true)] target: Vec, }, + #[structopt( + about = "Run shellcheck on a PKGBUILD, taking care of PKGBUILD-specific variables" + )] + Shellcheck { + #[structopt(help = "PKGBUILD to check (or ./PKGBUILD if not provided)")] + target: Option, + }, #[structopt(about = "Check *.tar or *.tar.xz archive")] Tarcheck { #[structopt(help = "Archive to check", required = true)] diff --git a/src/git_utils.rs b/src/git_utils.rs new file mode 100644 index 0000000..77a13ec --- /dev/null +++ b/src/git_utils.rs @@ -0,0 +1,70 @@ +use colored::*; +use std::path::PathBuf; +use std::process::{Command, Stdio}; + +/// Note that we're using `git init` instead of `git clone`-like command +/// to let the user review the initial diff. +/// Also, the local branch does NOT track the remote one -- +/// instead it's being merged upon each review. +pub fn init_repo(pkg: &str, dir: &PathBuf) { + silently_run_panic_if_error("git", &["init", "-q"], dir); + let http_ref = format!("https://aur.archlinux.org/{}.git", pkg); + silently_run_panic_if_error("git", &["remote", "add", "upstream", &http_ref], dir); + fetch(dir); +} + +pub fn fetch(dir: &PathBuf) { + silently_run_panic_if_error("git", &["fetch", "-q", "upstream"], dir); +} + +pub fn is_upstream_merged(dir: &PathBuf) -> bool { + let rev_parse_head = git(dir) + .args(&["rev-parse", "HEAD"]) + .stderr(Stdio::null()) + .stdout(Stdio::null()) + .status() + .expect("failed to run git"); + if !rev_parse_head.success() { + false + } else { + git(dir) + .args(&["merge-base", "--is-ancestor", "upstream/master", "HEAD"]) + .status() + .expect("failed to run git") + .success() + } +} + +pub fn show_upstream_diff(dir: &PathBuf) { + git(dir) + .args(&["diff", "-R", "upstream/master"]) + .status() + .ok(); +} + +pub fn merge_upstream(dir: &PathBuf) { + git(dir).args(&["merge", "upstream/master"]).status().ok(); +} + +fn silently_run_panic_if_error(first_arg: &str, other_args: &[&str], directory: &PathBuf) { + let command = Command::new(first_arg) + .args(other_args) + .current_dir(directory) + .output() + .unwrap_or_else(|err| panic!("Failed to execute process {}, {}", first_arg, err)); + assert!( + command.status.success(), + "Command {} {} failed with exit code {:?}\nStderr: {}\nStdout: {}", + first_arg, + other_args.join(" "), + command.status.code(), + String::from_utf8_lossy(&command.stderr).red(), + String::from_utf8_lossy(&command.stdout), + ); +} + +fn git(dir: &PathBuf) -> Command { + let mut command = Command::new("git"); + command.current_dir(dir); + command +} diff --git a/src/main.rs b/src/main.rs index 67894c3..5eee863 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,13 +4,14 @@ static GLOBAL: std::alloc::System = std::alloc::System; mod action_install; mod action_jailbuild; mod action_search; -mod aur_download; mod cli_args; +mod git_utils; mod pacman; mod print_format; mod print_package_info; mod print_package_table; -mod rua_dirs; +mod reviewing; +mod rua_files; mod srcinfo_to_pkgbuild; mod tar_check; mod terminal_util; @@ -19,13 +20,14 @@ mod wrapped; use std::fs::{File, OpenOptions, Permissions}; use std::io::Write; use std::os::unix::fs::PermissionsExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::exit; use std::process::Command; use std::{env, fs}; use crate::cli_args::CLIColorType; use crate::print_package_info::info; +use crate::wrapped::shellcheck; use chrono::Utc; use cli_args::{Action, CliArgs}; use directories::ProjectDirs; @@ -157,6 +159,15 @@ fn main() { Action::Info { ref target } => { info(target, false).unwrap(); } + Action::Shellcheck { target } => { + let result = shellcheck(&target.unwrap_or_else(|| PathBuf::from("./PKGBUILD"))); + result + .map_err(|err| { + eprintln!("{}", err); + exit(1); + }) + .ok(); + } Action::Tarcheck { target } => { tar_check::tar_check(&target); eprintln!("Finished checking pachage: {:?}", target); @@ -189,11 +200,11 @@ fn prepare_for_jailed_action(dirs: &ProjectDirs) { .expect("Failed to create project config directory"); overwrite_file( &dirs.config_dir().join(".system/seccomp-i686.bpf"), - include_bytes!("../res/seccomp-i686.bpf"), + rua_files::SECCOMP_I686, ); overwrite_file( &dirs.config_dir().join(".system/seccomp-x86_64.bpf"), - include_bytes!("../res/seccomp-x86_64.bpf"), + rua_files::SECCOMP_X86_64, ); let seccomp_path = format!( ".system/seccomp-{}.bpf", @@ -207,10 +218,10 @@ fn prepare_for_jailed_action(dirs: &ProjectDirs) { ); overwrite_script( &dirs.config_dir().join(wrapped::WRAP_SCRIPT_PATH), - include_bytes!("../res/wrap.sh"), + rua_files::WRAP_SH, ); ensure_script( &dirs.config_dir().join(".system/wrap_args.sh.example"), - include_bytes!("../res/wrap_args.sh"), + rua_files::WRAP_ARGS_EXAMPLE, ); } diff --git a/src/pacman.rs b/src/pacman.rs index 5d2528c..af13c18 100644 --- a/src/pacman.rs +++ b/src/pacman.rs @@ -83,7 +83,7 @@ fn ensure_packages_installed(mut packages: Vec<(String, PathBuf)>, base_args: &[ eprint!("or install manually and enter M when done. "); } attempt += 1; - let string = terminal_util::console_get_line(); + let string = terminal_util::read_line_lowercase(); if string == "s" { let exit_status = Command::new("sudo") .arg("pacman") diff --git a/src/print_format.rs b/src/print_format.rs index e5091b1..242b4b3 100644 --- a/src/print_format.rs +++ b/src/print_format.rs @@ -44,5 +44,5 @@ pub fn print_indent<'a>( _ => print!("{}", v.collect::>().join(" ")), } - println!(); + eprintln!(); } diff --git a/src/print_package_info.rs b/src/print_package_info.rs index fc36311..faaf3ce 100644 --- a/src/print_package_info.rs +++ b/src/print_package_info.rs @@ -21,7 +21,7 @@ pub fn info(pkgs: &[String], verbose: bool) -> Result<(), Error> { if let Some(pkg) = hm.get(pkg) { all_pkgs.push(pkg) } else { - println!("{} package '{}' was not found", "error:".red(), pkg); + eprintln!("{} package '{}' was not found", "error:".red(), pkg); } } @@ -64,7 +64,7 @@ pub fn info(pkgs: &[String], verbose: bool) -> Result<(), Error> { print("Snapshot URL", &pkg.url_path); } - println!(); + eprintln!(); } Ok(()) diff --git a/src/reviewing.rs b/src/reviewing.rs new file mode 100644 index 0000000..a1066fc --- /dev/null +++ b/src/reviewing.rs @@ -0,0 +1,70 @@ +use crate::git_utils; +use crate::rua_files; +use crate::terminal_util; +use crate::wrapped; +use directories::ProjectDirs; +use log::debug; +use std::path::PathBuf; + +pub fn review_repo(dir: &PathBuf, pkg_name: &str, dirs: &ProjectDirs) { + let mut dir_contents = dir.read_dir().unwrap_or_else(|err| { + panic!( + "{}:{} Failed to read directory for reviewing, {}", + file!(), + line!(), + err + ) + }); + if dir_contents.next().is_none() { + debug!("Directory {:?} is empty, using git clone", &dir); + git_utils::init_repo(pkg_name, &dir); + } else { + debug!("Directory {:?} is not empty, fetching new version", &dir); + git_utils::fetch(&dir); + } + + let build_dir = rua_files::build_dir(dirs, pkg_name); + if build_dir.exists() && git_utils::is_upstream_merged(&dir) { + eprintln!("WARNING: your AUR repo is up-to-date."); + eprintln!( + "If you continue, the build directory will be removed and the build will be re-run." + ); + eprintln!("If you don't want that, consider resolving the situation manually,"); + eprintln!("for example: rua jailbuild {:?}", build_dir); + eprintln!(); + } + + loop { + eprintln!("Reviewing {:?}. ", dir); + let is_upstream_merged = git_utils::is_upstream_merged(&dir); + if is_upstream_merged { + eprint!("[S]=run shellcheck linter on PKGBUILD, "); + } else { + eprint!("[D]=view changes since your last review, "); + eprint!("[M]=accept/merge upstream changes, "); + eprint!("[S]=(shellcheck not available until you merge), "); + } + eprint!("[I]=run shell to edit/inspect, "); + if is_upstream_merged { + eprint!("[O]=ok, use package "); + } else { + eprint!("[O]=(cannot use the package until you merge) "); + } + let string = terminal_util::read_line_lowercase(); + + if string == "i" { + eprintln!("Exit the shell with `logout` or Ctrl-D..."); + terminal_util::run_env_command(&dir, "SHELL", "bash", &[]); + } else if string == "s" && is_upstream_merged { + wrapped::shellcheck(&dir.join("PKGBUILD")) + .map_err(|err| eprintln!("{}", err)) + .ok(); + } else if string == "d" && !is_upstream_merged { + git_utils::show_upstream_diff(dir); + } else if string == "m" && !is_upstream_merged { + git_utils::merge_upstream(dir); + } else if string == "o" && is_upstream_merged { + break; + } + } +} diff --git a/src/rua_dirs.rs b/src/rua_dirs.rs deleted file mode 100644 index ed6a0eb..0000000 --- a/src/rua_dirs.rs +++ /dev/null @@ -1,11 +0,0 @@ -/// Directory to `git clone` into, first step of the build pipeline -pub const PREFETCH_DIR: &str = "aur.tmp"; - -/// Directory from AUR that passed user review -pub const REVIEWED_BUILD_DIR: &str = "build"; - -/// Directory where built package artifacts are stored, *.pkg.tar.xz -pub const TARGET_SUBDIR: &str = "target"; - -/// Directory where built and user-reviewed package artifacts are stored, -pub const CHECKED_TARS: &str = "checked_tars"; diff --git a/src/rua_files.rs b/src/rua_files.rs new file mode 100644 index 0000000..5804b7a --- /dev/null +++ b/src/rua_files.rs @@ -0,0 +1,31 @@ +use directories::ProjectDirs; +use std::path::PathBuf; + +/// subdirectory of ~/.config/rua where the package is reviewed by user, and changes are kept +pub fn global_review_dir(dirs: &ProjectDirs) -> PathBuf { + dirs.config_dir().join("pkg") +} + +pub fn review_dir(dirs: &ProjectDirs, pkg_name: &str) -> PathBuf { + global_review_dir(dirs).join(pkg_name) +} + +/// Directory where packages are built after review +pub fn global_build_dir(dirs: &ProjectDirs) -> PathBuf { + dirs.cache_dir().join("build") +} + +pub fn build_dir(dirs: &ProjectDirs, pkg_name: &str) -> PathBuf { + global_build_dir(dirs).join(pkg_name) +} + +/// Directory where built and user-reviewed package artifacts are stored, +pub fn checked_tars_dir(dirs: &ProjectDirs, pkg_name: &str) -> PathBuf { + dirs.cache_dir().join("checked_tars").join(pkg_name) +} + +pub const SHELLCHECK_WRAPPER_BYTES: &str = include_str!("../res/shellcheck-wrapper"); +pub const SECCOMP_I686: &[u8] = include_bytes!("../res/seccomp-i686.bpf"); +pub const SECCOMP_X86_64: &[u8] = include_bytes!("../res/seccomp-x86_64.bpf"); +pub const WRAP_SH: &[u8] = include_bytes!("../res/wrap.sh"); +pub const WRAP_ARGS_EXAMPLE: &[u8] = include_bytes!("../res/wrap_args.sh.example"); diff --git a/src/tar_check.rs b/src/tar_check.rs index e7018a5..81ba653 100644 --- a/src/tar_check.rs +++ b/src/tar_check.rs @@ -1,27 +1,31 @@ use crate::terminal_util; use colored::*; +use log::debug; +use log::trace; use std::fs::File; use std::io::Read; -use std::path::Path; +use std::path::{Path, PathBuf}; use tar::*; use xz2::read::XzDecoder; -pub fn tar_check(tar_file: &Path) { +pub fn tar_check(tar_file: &Path) -> Option { let tar_str = tar_file .to_str() .unwrap_or_else(|| panic!("{}:{} Failed to parse tar file name", file!(), line!())); let archive = File::open(&tar_file).unwrap_or_else(|_| panic!("cannot open file {}", tar_str)); if tar_str.ends_with(".tar.xz") { tar_check_archive(Archive::new(XzDecoder::new(archive)), tar_str); + debug!("Checked package tar file {}", tar_str); + Some(tar_file.to_path_buf()) } else if tar_str.ends_with(".tar") { tar_check_archive(Archive::new(archive), tar_str); + debug!("Checked package tar file {}", tar_str); + Some(tar_file.to_path_buf()) } else { - panic!( - "Unsupported file format for tar_check function: {}", - tar_str - ) - }; + trace!("Skipping non-tar file {}", tar_str); + None + } } fn tar_check_archive(mut archive: Archive, path_str: &str) { @@ -87,7 +91,7 @@ fn tar_check_archive(mut archive: Archive, path_str: &str) { eprint!("{}", "!!! [S]=list SUID files!!!, ".red()) }; eprint!("[O]=ok, proceed. "); - let string = terminal_util::console_get_line(); + let string = terminal_util::read_line_lowercase(); eprintln!(); if string == "s" && !suid_files.is_empty() { for path in &suid_files { @@ -104,8 +108,10 @@ fn tar_check_archive(mut archive: Archive, path_str: &str) { } else if string == "i" && has_install { eprintln!("{}", &install_file); } else if string == "t" { + let dir = PathBuf::from(path_str); + let dir = dir.parent().unwrap_or_else(|| Path::new(".")); eprintln!("Exit the shell with `logout` or Ctrl-D..."); - terminal_util::run_env_command("SHELL", "bash", &[]); + terminal_util::run_env_command(&dir.to_path_buf(), "SHELL", "bash", &[]); } else if string == "o" { break; } else if string == "q" { diff --git a/src/terminal_util.rs b/src/terminal_util.rs index c100b6f..78181b0 100644 --- a/src/terminal_util.rs +++ b/src/terminal_util.rs @@ -1,8 +1,9 @@ use std::env; use std::io; +use std::path::PathBuf; use std::process::Command; -pub fn console_get_line() -> String { +pub fn read_line_lowercase() -> String { let mut string = String::new(); io::stdin() .read_line(&mut string) @@ -11,7 +12,12 @@ pub fn console_get_line() -> String { } /// For example: SHELL, PAGER, EDITOR. -pub fn run_env_command(env_variable_name: &str, alternative_executable: &str, arguments: &[&str]) { +pub fn run_env_command( + dir: &PathBuf, + env_variable_name: &str, + alternative_executable: &str, + arguments: &[&str], +) { let command = env::var(env_variable_name) .ok() .map(|s| s.trim().to_owned()); @@ -29,6 +35,7 @@ pub fn run_env_command(env_variable_name: &str, alternative_executable: &str, ar Command::new(alternative_executable) }; command.args(arguments); + command.current_dir(dir); let command = command.status(); if let Some(err) = command.err() { eprintln!("Failed to run command, error: {}", err); diff --git a/src/wrapped.rs b/src/wrapped.rs index 2840b20..d01af21 100644 --- a/src/wrapped.rs +++ b/src/wrapped.rs @@ -1,24 +1,17 @@ // Commands that are run inside "bubblewrap" jail -use crate::aur_download; -use crate::pacman; -use crate::pacman::PACMAN_ARCH; -use crate::rua_dirs::PREFETCH_DIR; -use crate::rua_dirs::TARGET_SUBDIR; +use crate::rua_files; +use crate::srcinfo_to_pkgbuild; use directories::ProjectDirs; -use libalpm::Alpm; use log::debug; -use srcinfo::Srcinfo; - -use std::cmp; -use std::collections::{HashMap, HashSet}; +use log::info; +use std::fs; use std::fs::File; use std::io::Write; -use std::path::Path; -use std::process::Command; +use std::path::PathBuf; +use std::process::{Command, Stdio}; use std::str; -use std::{env, fs}; pub const WRAP_SCRIPT_PATH: &str = ".system/wrap.sh"; @@ -26,42 +19,46 @@ fn wrap_yes_internet(dirs: &ProjectDirs) -> Command { Command::new(dirs.config_dir().join(WRAP_SCRIPT_PATH)) } -fn download_srcinfo_sources(dirs: &ProjectDirs) { - let dir = env::current_dir().unwrap().canonicalize().unwrap(); - let dir = dir.to_str().unwrap(); - let mut file = File::create("PKGBUILD.static") +fn download_srcinfo_sources(dir: &str, dirs: &ProjectDirs) { + let dir_path = PathBuf::from(dir).join("PKGBUILD.static"); + let mut file = File::create(&dir_path) .unwrap_or_else(|err| panic!("Cannot create {}/PKGBUILD.static, {}", dir, err)); - let srcinfo_path = Path::new(".SRCINFO") + let srcinfo_path = PathBuf::from(dir) + .join(".SRCINFO") .canonicalize() .unwrap_or_else(|e| panic!("Cannot resolve .SRCINFO path in {}, {}", dir, e)); - file.write_all(crate::srcinfo_to_pkgbuild::static_pkgbuild(&srcinfo_path).as_bytes()) + file.write_all(srcinfo_to_pkgbuild::static_pkgbuild(&srcinfo_path).as_bytes()) .expect("cannot write to PKGBUILD.static"); - eprintln!("Downloading sources using .SRCINFO..."); + info!("Downloading sources using .SRCINFO..."); let command = wrap_yes_internet(dirs) .args(&["--bind", dir, dir]) .args(&["makepkg", "-f", "--verifysource"]) .args(&["-p", "PKGBUILD.static"]) + .current_dir(dir) .status() .unwrap_or_else(|e| panic!("Failed to fetch dependencies in directory {}, {}", dir, e)); assert!(command.success(), "Failed to download PKGBUILD sources"); - fs::remove_file("PKGBUILD.static").expect("Failed to clean up PKGBUILD.static"); + fs::remove_file(PathBuf::from(dir).join("PKGBUILD.static")) + .expect("Failed to clean up PKGBUILD.static"); } -fn build_local(dirs: &ProjectDirs, is_offline: bool) { - let dir = env::current_dir() - .unwrap_or_else(|e| panic!("{}:{} Failed to get current dir, {}", file!(), line!(), e)); - let dir = dir.to_str().unwrap(); +fn build_local(dir: &str, dirs: &ProjectDirs, is_offline: bool) { + debug!( + "{}:{} Building package in directory {}", + file!(), + line!(), + dir + ); let mut command = wrap_yes_internet(dirs); + command.current_dir(dir); if is_offline { command.arg("--unshare-net"); } command.args(&["--bind", dir, dir]); - let command = command.args(&["makepkg"]).status().unwrap_or_else(|e| { - panic!( - "Failed to build package (jailed makepkg) in directory {}, {}", - dir, e, - ) - }); + let command = command + .args(&["makepkg"]) + .status() + .unwrap_or_else(|e| panic!("Failed to execute ~/.config/rua/.system/wrap.sh, {}", e,)); if !command.success() { eprintln!( "Build failed with exit code {} in {}", @@ -75,80 +72,52 @@ fn build_local(dirs: &ProjectDirs, is_offline: bool) { } pub fn build_directory(dir: &str, project_dirs: &ProjectDirs, offline: bool) { - env::set_current_dir(dir) - .unwrap_or_else(|e| panic!("cannot change the current directory to {}, {}", dir, e)); - env::set_var( - "PKGDEST", - Path::new(".") - .canonicalize() - .unwrap_or_else(|e| panic!("Failed to canonize target directory {}, {}", dir, e)) - .join(TARGET_SUBDIR), - ); if offline { - download_srcinfo_sources(project_dirs); + download_srcinfo_sources(dir, project_dirs); } - build_local(project_dirs, offline); + build_local(dir, project_dirs, offline); } -pub fn prefetch_aur( - name: &str, - dirs: &ProjectDirs, - pacman_deps: &mut HashSet, - aur_packages: &mut HashMap, - depth: i32, - alpm: &Alpm, -) { - if let Some(old_depth) = aur_packages.get(name) { - let old_depth = *old_depth; - aur_packages.insert(name.to_owned(), cmp::max(depth + 1, old_depth)); - eprintln!("Skipping already fetched package {}", name); - return; - } - aur_packages.insert(name.to_owned(), depth); - aur_download::fresh_download(&name, &dirs); - let srcinfo_path = dirs - .cache_dir() - .join(name) - .join(PREFETCH_DIR) - .join(".SRCINFO"); - let info = Srcinfo::parse_file(&srcinfo_path).unwrap_or_else(|err| { - panic!( - "{}:{} Failed to parse {:?}, {}", - file!(), - line!(), - srcinfo_path, - err, +pub fn shellcheck(target: &PathBuf) -> Result<(), String> { + if !target.exists() { + return Err("Cannot find target for shellcheck, aborting".into()); + }; + let mut command = Command::new("bwrap"); + command.args(&["--ro-bind", "/", "/"]); + command.args(&["--proc", "/proc", "--dev", "/dev"]); + command.args(&["--unshare-all"]); + command.args(&[ + "shellcheck", + "--check-sourced", + "--norc", + "--external-sources", + "/dev/stdin", + ]); + command.stdin(Stdio::piped()); + let mut child = command.spawn().map_err(|_| { + "Failed to spawn shellcheck process. Do you have shellcheck installed?\ + sudo pacman -S --needed shellcheck" + })?; + let stdin: &mut std::process::ChildStdin = child + .stdin + .as_mut() + .map_or(Err("Failed to open stdin for shellcheck"), Ok)?; + let target = target.to_str().expect("Failed to parse shellcheck target"); + let target = target.replace("'", "'\\''"); + let target = format!("source '{}'", target); + let bytes = rua_files::SHELLCHECK_WRAPPER_BYTES.replace("source PKGBUILD", &target); + stdin.write_all(bytes.as_bytes()).map_err(|err| { + format!( + "Failed to write shellcheck wrapper script to shellcheck-s stdin, {}", + err ) - }); - let deps = info - .pkg(name) - .unwrap_or_else(|| { - panic!( - "{}:{} pkgname {} not found in {:?}", - file!(), - line!(), - name, - &srcinfo_path - ) - }) - .depends - .iter() - .chain(&info.base.makedepends) - .chain(&info.base.checkdepends) - .filter(|deps_vector| deps_vector.supports(PACMAN_ARCH.as_str())) - .flat_map(|deps_vector| &deps_vector.vec) - .collect::>(); - debug!("package {} has dependencies: {:?}", name, &deps); - for dep in deps.into_iter() { - if pacman::is_package_installed(alpm, &dep) { - } else if !pacman::is_package_installable(alpm, &dep) { - eprintln!( - "{} depends on AUR package {}. Trying to fetch it...", - name, &dep - ); - prefetch_aur(&dep, dirs, pacman_deps, aur_packages, depth + 1, alpm); - } else { - pacman_deps.insert(dep.to_owned()); - } + })?; + let child = child + .wait_with_output() + .map_err(|e| format!("Failed waiting for shellcheck to exit: {}", e))?; + if child.status.success() { + Ok(()) + } else { + Err("".into()) } }