diff --git a/.gitignore b/.gitignore index 220040f55..613083218 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ target *.css *.css.map .sass-cache +.vagrant diff --git a/INSTALLATION.md b/INSTALLATION.md deleted file mode 100644 index 1f2016e2c..000000000 --- a/INSTALLATION.md +++ /dev/null @@ -1,172 +0,0 @@ -# Installation Guide - -## Requirements - -`cratesfyi` is written in Rust and requires Rust compiler. It is developed under -Linux, it may not work or compile under other operating systems. List of -requirements to compile `cratesfyi`: - -* rustc compiler -* gcc -* cmake -* pkg-config -* openssl (development files) -* libmagic (development files) -* git -* postgresql -* libc (development files) -* lxc - -## `cratesfyi-prefix` directory - -`cratesfyi` is using a prefix directory for: - -* Clone of `crates.io-index` -* `documentations` directory to copy generated documentation -* `sources` directory -* `public_html` directory to serve some static files -* `cratesfyi-container` a Linux container (LXC) to build crates in a safe - environment - -An example script to create cratesfyi-prefix directory (note: you don't need -setup a lxc-container if you are gonna run only web interface, -skip to setting up database): - - -```sh -#!/bin/sh -# Creates cratesfyi-prefix directory for cratesfyi -# This script is designed to run on Debian based operating systems, -# and tested under Debian jessie and sid - -set -e - -PREFIX=$(pwd)/cratesfyi-prefix -DIST_TEMPLATE=debian -DIST_RELEASE=jessie -DIST_MIRROR=http://httpredir.debian.org/debian - -mkdir $PREFIX -mkdir -p $PREFIX/sources $PREFIX/documentations -git clone https://github.com/rust-lang/crates.io-index.git $PREFIX/crates.io-index - -# Create debian8 lxc container into cratesfyi-container directory -# Use your own distribution template and release name -sudo LANG=C MIRROR=$DIST_MIRROR \ - lxc-create -n cratesfyi-container -P $PREFIX \ - -t $DIST_TEMPLATE -- -r $DIST_RELEASE - -# Due to some bug in lxc-attach this container -# must have a symbolic link in /var/lib/lxc -sudo ln -s $PREFIX/cratesfyi-container /var/lib/lxc - -# Container directory must be accessible by current user -sudo chmod 755 $PREFIX/cratesfyi-container - -# Setup lxc network -echo 'USE_LXC_BRIDGE="true" -LXC_BRIDGE="lxcbr0" -LXC_ADDR="10.0.3.1" -LXC_NETMASK="255.255.255.0" -LXC_NETWORK="10.0.3.0/24" -LXC_DHCP_RANGE="10.0.3.2,10.0.3.254" -LXC_DHCP_MAX="253" -LXC_DHCP_CONFILE="" -LXC_DOMAIN=""' | sudo tee /etc/default/lxc-net - -# Start network interface -sudo service lxc-net restart - -# Setup network for container -sudo sed -i 's/lxc.network.type.*/lxc.network.type = veth\nlxc.network.link = lxcbr0/' \ - $PREFIX/cratesfyi-container/config - -# Start lxc container -sudo lxc-start -n cratesfyi-container - -# Add user accounts into container -# cratesfyi is using multiple user accounts to run cargo simultaneously -for user in $(whoami) cratesfyi updater; do - sudo lxc-attach -n cratesfyi-container -- \ - adduser --disabled-login --disabled-password --gecos "" $user -done - -# Install required packages for rust installation -sudo lxc-attach -n cratesfyi-container -- apt-get update -sudo lxc-attach -n cratesfyi-container -- apt-get install -y file git curl sudo ca-certificates - -# Install rust nightly into container -sudo lxc-attach -n cratesfyi-container -- \ - su - -c 'curl -sSf https://static.rust-lang.org/rustup.sh | sh -s -- --channel=nightly' -``` - - -The last step is to install cratesfyi into the guest machine -(or build in guest machine). If your host and guest -operating system is same simply build cratesfyi in release mode and copy into -`/usr/local/bin` directory of guest system: - -```sh -cargo build --release -cp target/release/cratesfyi CRATESFYI_PREFIX_DIR/rootfs/usr/local/bin/ -``` - -cratesfyi is only using `lxd-attach` command with sudo. Make sure your user -account can use this command without root password. Example `sudoers` entry: - -```text -cratesfyi ALL=(ALL) NOPASSWD: /usr/bin/lxc-attach -``` - -### Setting up database - -cratesfyi is using postgresql database to store crate and build -information. You need to set up database before using chroot builder. To do -this: - -```text -$ sudo su - postgres -c psql -# First create a user -postgres=# CREATE USER cratesfyi WITH PASSWORD 'password'; -postgres=# CREATE DATABASE cratesfyi OWNER cratesfyi; -postgres=# \q -# Initialize database with cratesfyi -$ CRATESFYI_DATABASE_URL=postgresql://cratesfyi:password@localhost cratesfyi database init -``` - -## Environment variables - -`cratesfyi` is using various environment variables to run. Those are: - -```sh -HOME=/srv/cratesfyi -PATH=/srv/cratesfyi/.cargo/bin:/usr/local/bin:/usr/bin:/bin -CRATESFYI_PREFIX=/srv/cratesfyi/cratesfyi-prefix -CRATESFYI_DATABASE_URL='postgresql://cratesfyi@localhost' -CRATESFYI_GITHUB_USERNAME='USERNAME' -CRATESFYI_GITHUB_ACCESSTOKEN='TOKEN' -``` - -You can also specify `RUST_LOG=cratesfyi` to see all log messages. It will only -show `INFO` or more leveled log messages without any `RUST_LOG` environment -variable. - -Example systemd service file for cratesfyi: - -```text -[Unit] -Description=Cratesfyi daemon -After=network.target postgresql.service - -[Service] -User=cratesfyi -Group=cratesfyi -Type=forking -PIDFile=/srv/cratesfyi/cratesfyi-prefix/cratesfyi.pid -EnvironmentFile=/srv/cratesfyi/env -ExecStart=/bin/sh -c '/srv/cratesfyi/cratesfyi daemon > /srv/cratesfyi/cratesfyi-prefix/cratesfyi.log 2>&1' -WorkingDirectory=/srv/cratesfyi - -[Install] -WantedBy=multi-user.target -``` diff --git a/README.md b/README.md index 970983c87..c29a299ea 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ crates' documentation easily. Example of URL redirections for `clap` crate: | URL | Redirects to documentation of | |------------------------------|------------------------------------------------| | | Latest version of clap | -| | 2.* version | -| | 2.9.* version | +| | 2.* version | +| | 2.9.* version | | | 2.9.3 version (you don't need = unlike semver) | The crates.fyi domain will redirect to docs.rs, supporting all of the @@ -47,6 +47,108 @@ Example badges for `mio` crate: | Version 0.1.0: | ![mio](https://docs.rs/mio/badge.svg?version=0.1.0) | +## Development + +We strongly recommend using vagrant, this will give you a virtual machine +already configured and ready to start developing on. + +### Getting started + +Make sure you have vagrant, virtualbox and a ssh client and you need +to able to download ~800MB data on the first run. + + +```sh +git clone https://github.com/onur/docs.rs.git docs.rs +cd docs.rs +vagrant up # This may take a little while on the first run +``` + +You can always run `vagrant provision` to reconfigure virtual machine. +Provision will install required dependencies and nightly rust compiler +into virtual machine. It will also configure lxc-container inside +virtual machine. + +### CLI + +Make sure you are running every listed command inside `/vagrant` directory +in virtual machine. You can connect to virtual machine with `vagrant ssh` and +switch current working directory with: `cd /vagrant` inside virtual machine. + + +#### Starting web server + +This command will start web interface of docs.rs and you can access it from: +`http://localhost:3000/` + +``` +cargo run -- start-web-server +``` + + +#### `build` subcommand + +```sh +# Builds and adds it into database +# This is the main command to build and add a documentation into docs.rs. +cargo run -- build crate + + +# Adds essential files (css and fonts) into database to avoid duplication +# This command needs to be run after each rustc update +cargo run -- build add-essential-files + + +# Builds every crate and adds them into database +# (beware: this may take months to finish) +cargo run -- build world +``` + + +#### `database` subcommand + +```sh +# Initializes database. Currently, only creates tables in database. +cargo run -- database init + + +# Adds a directory into database to serve with `staticfile` crate. +cargo run -- database add-directory [PREFIX] + + +# Updates github stats for crates. +# You need to set CRATESFYI_GITHUB_USERNAME, CRATESFYI_GITHUB_ACCESSTOKEN +# environment variables in order to run this command. +# You can set this environment variables in ~/.cratesfyi.env file. +cargo run -- database update-github-fields + + +# Updates search-index. +# daemon is running this command occasionally, and this command must be +# run to update recent-version of a crate index and search index. +# If you are having any trouble with accessing right version of a crate, +# run this command. Otherwise it's not required. +cargo run -- database update-search-index + + +# Updates release activitiy chart +cargo run -- database update-release-activity +``` + +If you want to explore or edit database manually, you can connect database +with `psql` command. + + +#### `doc` subcommand + +This subcommand will only build documentation of a crate. +It is designed to run inside a secure container. + +``` +cargo run -- doc +``` + + #### Contributors * [Onur Aslan](https://github.com/onur) diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 000000000..e079b0aa0 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,146 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.box = "docs.rs" + config.vm.box_url = "https://docs.rs/vagrant.box" + config.vm.box_download_checksum = "02554eb16ac72367980a883c4ff05f17d9988406" + config.vm.box_download_checksum_type = "sha1" + + config.ssh.username = "cratesfyi" + + config.vm.network "forwarded_port", guest: 3000, host: 3000 + + # Use 25% of available system memory and all CPU's + config.vm.provider "virtualbox" do |vb| + host = RbConfig::CONFIG['host_os'] + + if host =~ /darwin/ + cpus = `sysctl -n hw.ncpu`.to_i + mem = `sysctl -n hw.memsize`.to_i / 1024 / 1024 / 4 + elsif host =~ /linux/ + cpus = `nproc`.to_i + mem = `grep 'MemTotal' /proc/meminfo | sed -e 's/MemTotal://' -e 's/ kB//'`.to_i / 1024 / 4 + else + cpus = 2 + mem = 1024 + end + + vb.memory = mem + vb.cpus = cpus + end + + + # docs.rs vagrant image comes with only a pre-configured cratesfyi-container + # installing rest with provision + config.vm.provision "shell", inline: <<-SHELL + set -ev + + ############################################################ + # Installing docs.rs dependencies # + ############################################################ + apt-get update + apt-get install -y --no-install-recommends cmake curl cmake gcc g++ git libmagic-dev libssl-dev + + ############################################################ + # Installing rustc into cratesfyi-container # + ############################################################ + lxc-attach -n cratesfyi-container -- apt-get update + lxc-attach -n cratesfyi-container -- apt-get install -y --no-install-recommends curl ca-certificates binutils gcc libc6-dev libmagic1 + lxc-attach -n cratesfyi-container -- su - cratesfyi -c 'curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly-2016-12-19' + + ############################################################ + # Creating rustc links for cratesfyi user # + ############################################################ + for directory in .cargo .rustup .multirust; do + [[ -h /home/cratesfyi/$directory ]] || sudo -u cratesfyi ln -vs /var/lib/lxc/cratesfyi-container/rootfs/home/cratesfyi/$directory /home/cratesfyi/ + done + + ############################################################ + # Setting up environment variables # + ############################################################ + [[ -f /home/cratesfyi/.cratesfyi.env ]] || sudo -u cratesfyi tee /home/cratesfyi/.cratesfyi.env <<\EOF +CRATESFYI_PREFIX=/home/cratesfyi/cratesfyi-prefix +CRATESFYI_DATABASE_URL=postgresql://cratesfyi@localhost +CRATESFYI_GITHUB_USERNAME= +CRATESFYI_GITHUB_ACCESSTOKEN= +RUST_LOG=cratesfyi +EOF + + ############################################################ + # Loading environment variables # + ############################################################ + source /home/cratesfyi/.cratesfyi.env + + ############################################################ + # Preparing cratesfyi-prefix # + ############################################################ + sudo -u cratesfyi mkdir -vp /home/cratesfyi/cratesfyi-prefix/documentations \ + /home/cratesfyi/cratesfyi-prefix/public_html + + ############################################################ + # Getting external css files from docs.rs # + ############################################################ + sudo -u cratesfyi wget -qcP /home/cratesfyi/cratesfyi-prefix/public_html \ + https://docs.rs/rustdoc-20160526-1.10.0-nightly-97e3a2401.css \ + https://docs.rs/main-20160526-1.10.0-nightly-97e3a2401.css + + ############################################################ + # Cloning crates.io-index # + ############################################################ + if [ ! -d /home/cratesfyi/cratesfyi-prefix/crates.io-index ]; then + sudo -u cratesfyi git clone https://github.com/rust-lang/crates.io-index /home/cratesfyi/cratesfyi-prefix/crates.io-index + else + sudo -u cratesfyi git --git-dir=/home/cratesfyi/cratesfyi-prefix/crates.io-index/.git pull + fi + + # Create `crates-index-diff_last-seen` branch for tracking new crates + sudo -u cratesfyi git --git-dir=/home/cratesfyi/cratesfyi-prefix/crates.io-index/.git branch crates-index-diff_last-seen || true + + ############################################################ + # Building docs.rs # + ############################################################ + su - cratesfyi -c "cd /vagrant && cargo build" 2>&1 + + ############################################################ + # Copying docs.rs into container # + ############################################################ + cp -v /vagrant/target/debug/cratesfyi /var/lib/lxc/cratesfyi-container/rootfs/usr/local/bin + + ############################################################ + # Re-creating database # + ############################################################ + echo 'DROP DATABASE cratesfyi; CREATE DATABASE cratesfyi OWNER cratesfyi' | sudo -u postgres psql + + ############################################################ + # Initializing database scheme # + ############################################################ + su - cratesfyi -c "cd /vagrant && cargo run -- database init" + + ############################################################ + # Add essential files for downloaded nigthly # + ############################################################ + su - cratesfyi -c "cd /vagrant && cargo run -- build add-essential-files" 2>&1 + + ############################################################ + # Populating database by building some crates # + ############################################################ + su - cratesfyi -c "cd /vagrant && cargo run -- build crate rand 0.3.15" 2>&1 + su - cratesfyi -c "cd /vagrant && cargo run -- build crate log 0.3.6" 2>&1 + su - cratesfyi -c "cd /vagrant && cargo run -- build crate regex 0.1.80" 2>&1 + + ############################################################ + # Update search index and release activity # + ############################################################ + su - cratesfyi -c "cd /vagrant && cargo run -- database update-search-index" 2>&1 + su - cratesfyi -c "cd /vagrant && cargo run -- database update-release-activity" 2>&1 + + + ############################################################ + # docs.rs vagrant box is ready! # + #----------------------------------------------------------# + # You can connect to virtual machine with `vagrant ssh`. # + # docs.rs is available in `/vagrant` folder! # + ############################################################ + SHELL +end diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index a0c38e886..da46ebd98 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -9,7 +9,6 @@ extern crate time; use std::env; -use std::process; use std::path::PathBuf; use clap::{Arg, App, SubCommand}; @@ -91,7 +90,13 @@ pub fn main() { .required(true) .help("Version of crate"))) .subcommand(SubCommand::with_name("add-essential-files") - .about("Adds essential files for rustc"))) + .about("Adds essential files for rustc")) + .subcommand(SubCommand::with_name("lock") + .about("Locks cratesfyi daemon to stop \ + building new crates")) + .subcommand(SubCommand::with_name("unlock") + .about("Unlocks cratesfyi daemon to continue \ + building new crates"))) .subcommand(SubCommand::with_name("start-web-server") .about("Starts web server") .arg(Arg::with_name("SOCKET_ADDR") @@ -118,7 +123,11 @@ pub fn main() { .index(2) .help("Prefix of files in \ database"))) - .subcommand(SubCommand::with_name("update-release-activity"))) + .subcommand(SubCommand::with_name("update-release-activity")) + .about("Updates montly release activity \ + chart") + .subcommand(SubCommand::with_name("update-search-index")) + .about("Updates search index")) .get_matches(); @@ -169,41 +178,44 @@ pub fn main() { let mut docbuilder = DocBuilder::new(docbuilder_opts); - docbuilder.load_cache().expect("Failed to load cache"); - if let Some(_) = matches.subcommand_matches("world") { + docbuilder.load_cache().expect("Failed to load cache"); docbuilder.build_world().expect("Failed to build world"); + docbuilder.save_cache().expect("Failed to save cache"); } else if let Some(matches) = matches.subcommand_matches("crate") { + docbuilder.load_cache().expect("Failed to load cache"); docbuilder.build_package(matches.value_of("CRATE_NAME").unwrap(), matches.value_of("CRATE_VERSION").unwrap()) .expect("Building documentation failed"); + docbuilder.save_cache().expect("Failed to save cache"); } else if let Some(_) = matches.subcommand_matches("add-essential-files") { docbuilder.add_essential_files().expect("Failed to add essential files"); + } else if let Some(_) = matches.subcommand_matches("lock") { + docbuilder.lock().expect("Failed to lock"); + } else if let Some(_) = matches.subcommand_matches("unlock") { + docbuilder.unlock().expect("Failed to unlock"); } - docbuilder.save_cache().expect("Failed to save cache"); } else if let Some(matches) = matches.subcommand_matches("database") { if let Some(_) = matches.subcommand_matches("init") { - use std::io::Write; - use std::io; let conn = db::connect_db().unwrap(); - if let Err(err) = db::create_tables(&conn) { - writeln!(&mut io::stderr(), "Failed to initialize database: {}", err).unwrap(); - process::exit(1); - } + db::create_tables(&conn).expect("Failed to initialize database"); } else if let Some(_) = matches.subcommand_matches("update-github-fields") { cratesfyi::utils::github_updater().expect("Failed to update github fields"); } else if let Some(matches) = matches.subcommand_matches("add-directory") { add_path_into_database(&db::connect_db().unwrap(), matches.value_of("PREFIX").unwrap_or(""), matches.value_of("DIRECTORY").unwrap()) - .unwrap(); + .expect("Failed to add directory into database"); } else if let Some(_) = matches.subcommand_matches("update-release-activity") { // FIXME: This is actually util command not database cratesfyi::utils::update_release_activity().expect("Failed to update release activity"); + } else if let Some(_) = matches.subcommand_matches("update-search-index") { + let conn = db::connect_db().unwrap(); + db::update_search_index(&conn).expect("Failed to update search index"); } } else if let Some(matches) = matches.subcommand_matches("start-web-server") { - start_web_server(matches.value_of("SOCKET_ADDR")); + start_web_server(Some(matches.value_of("SOCKET_ADDR").unwrap_or("0.0.0.0:3000"))); } else if let Some(_) = matches.subcommand_matches("daemon") { cratesfyi::utils::start_daemon(); } else { diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index 2dfbb7979..0f56fb438 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -81,4 +81,27 @@ impl DocBuilder { } Ok(()) } + + + fn lock_path(&self) -> PathBuf { + self.options.prefix.join("cratesfyi.lock") + } + + /// Creates a lock file. Daemon will check this lock file and stop operating if its exists. + pub fn lock(&self) -> Result<()> { + let path = self.lock_path(); + if !path.exists() { + try!(fs::OpenOptions::new().write(true).create(true).open(path)); + } + Ok(()) + } + + /// Removes lock file. + pub fn unlock(&self) -> Result<()> { + let path = self.lock_path(); + if path.exists() { + try!(fs::remove_file(path)); + } + Ok(()) + } } diff --git a/templates/about.hbs b/templates/about.hbs index 2997a0223..ef3e1a10c 100644 --- a/templates/about.hbs +++ b/templates/about.hbs @@ -46,11 +46,11 @@ Latest version of clap - docs.rs/clap/^2 + docs.rs/clap/~2 2.* version - docs.rs/clap/^2.9 + docs.rs/clap/~2.9 2.9.* version