Skip to content

Commit

Permalink
Readme cleanup and automation improvements
Browse files Browse the repository at this point in the history 
* Clear README

* Reformat .gitignore
  • Loading branch information
@davvore33
davvore33 committed Sep 17, 2019
1 parent e106ff4 commit d80daba
Show file tree
Hide file tree
Showing 7 changed files with 224 additions and 163 deletions.
25 changes: 13 additions & 12 deletions .gitignore
View file
@@ -1,11 +1,11 @@
# Object directories
/build/

# OSX files
.DS_Store
# Protobuf generated files
/api/generated/*

# Temp emacs files
*~
# Python Venv
/warp17-venv/

# cscope and etags files
TAGS
Expand All @@ -26,20 +26,21 @@ cscope.in.out
*.lib
*.a

# Sublime-text files
*.sublime-project
*.sublime-workspace

# Protobuf generated files
/api/generated/*
# OSX files
.DS_Store

# Python Venv
/warp17-venv/
# Temp emacs files
*~

# Perl autogen
/_Inline/

# IDEs files and folders
*.sublime-project
*.sublime-workspace
/.vscode/
/.idea/

# Cmake generated files
/cmake-build-debug/
CMakeLists.txt
7 changes: 3 additions & 4 deletions .travis.yml
View file
Expand Up @@ -67,9 +67,8 @@ cache:
- $HOME/.cache/pip

before_install:
- sudo apt update
- sudo apt install linux-headers-$(uname -r) libnuma-dev ncurses-dev protobuf-compiler libprotobuf-dev python-protobuf libprotobuf-c0 libprotobuf-c0-dev libprotobuf8 libprotoc8 protobuf-c-compiler -y
- sudo bash ./build_dpdk.sh -v 17.11.3 -j 2 -i
- sudo ./dep_install.sh
- sudo bash ./build_dpdk.sh -v 17.11.6 -j 2 -i
- sudo pip install -r python/requirements.txt
- sudo sysctl -w vm.nr_hugepages=1024
- sudo mkdir -p /mnt/huge
Expand All @@ -87,4 +86,4 @@ after_failure:
- sudo cat /tmp/warp17*/*.out
- sudo cat /tmp/warp17*/*.log
notifications:
slack: warp17:4NA10mDvXakkJ8I4PGMOxlLg
slack: warp17:slrgF4MwJdIwf49Upb0BSIam
157 changes: 14 additions & 143 deletions README.md
View file
Expand Up @@ -106,157 +106,28 @@ payload) from 10 million clients which are processed on the receing side by
## Prerequisites

Any 64 bit Linux distribution will do, however we have been testing this with
Ubuntu Server 14.04 LTS. In addition we have made an OVF virtual machine image
Ubuntu Server 16.04 LTS. In addition we have made an OVF virtual machine image
available, details can be found in the respective [documentation](ovf/README.md).

### Install build essential, lib Numa, python and ncurses
```
sudo apt-get install build-essential libnuma-dev python ncurses-dev
```

### Install DPDK 17.11.3

* Download [DPDK 17.11.3](http://fast.dpdk.org/rel/dpdk-17.11.3.tar.xz)

```
tar xf dpdk-17.11.3.tar.xz
cd dpdk-stable-17.11.3
```

* Install DPDK:

```
make config T=x86_64-native-linuxapp-gcc
make
sudo make install
```

* Load the `igb_uio` DPDK module, either as shown below or by running the
`$RTE_SDK/usertools/dpdk-setup.sh` script and selecting option
`Insert IGB UIO module`:

- add the following modules to `/etc/modules`:

```
#
# DPDK additions
#
uio
igb_uio
rte_kni
```

- reload all modules:

```
sudo depmod -a
sudo modprobe uio
sudo modprobe igb_uio
sudo modprobe rte_kni
```
### Install DPDK

* Enable at least 32 1G hugepages and configure them (see section 2.3.2.1 from
the [DPDK Guide](http://dpdk.org/doc/guides/linux_gsg/sys_reqs.html)):
- add the following line to `/etc/default/grub`:

```
GRUB_CMDLINE_LINUX="default_hugepagesz=1G hugepagesz=1G hugepages=32"
```

- update grub:

```
sudo update-grub
```

- reboot the machine

```
sudo reboot
```

* Mount hugepages (see section 2.3.2.2 from the
[DPDK Guide](http://dpdk.org/doc/guides/linux_gsg/sys_reqs.html)):
- add the mountpoint:

```
sudo mkdir /mnt/huge_1GB
```

- make the mountpoint permanent by adding to `/etc/fstab`:

```
nodev /mnt/huge_1GB hugetlbfs pagesize=1GB 0 0
```
Run the automated script with `<version>` as 17.11.6 (the latest LTS supported by warp17)

- remount:

```
sudo mount /mnt/huge_1GB
```

* Export the path to the DPDK SDK (where DPDK was installed) into the variable
RTE_SDK. For example:

```
export RTE_SDK=/usr/local/share/dpdk
```

* Export the target of the DPDK SDK into the variable RTE_TARGET. For example:

```
export RTE_TARGET=x86_64-native-linuxapp-gcc
```
```
# build_dpdk.sh -v <version>
```

### Install Google Protocol Buffers

* First install the protobuf compilers and python libraries.

```
sudo apt-get install protobuf-compiler libprotobuf-dev python-protobuf
```

* If using Ubuntu Server 14.04 LTS then just install:

```
sudo apt-get install libprotobuf-c0 libprotobuf-c0-dev libprotobuf8 libprotoc8 protobuf-c-compiler
```

* Otherwise (Ubuntu version >= 15.10):
* Install [libprotobuf-c](http://packages.ubuntu.com/trusty/amd64/libprotobuf-c0/download),
[libprotobuf-c-dev](http://packages.ubuntu.com/trusty/amd64/libprotobuf-c0-dev/download)
from Ubuntu 14.04LTS:

```
sudo dpkg -i libprotobuf-c0_0.15-1build1_amd64.deb
sudo dpkg -i libprotobuf-c0-dev_0.15-1build1_amd64.deb
```

* Install [libprotobuf8](http://packages.ubuntu.com/trusty/amd64/libprotobuf8/download)
from Ubuntu 14.04LTS:

```
sudo dpkg -i libprotobuf8_2.5.0-9ubuntu1_amd64.deb
```

* Install [libprotoc8](http://packages.ubuntu.com/trusty/amd64/libprotoc8/download)
from Ubuntu 14.04LTS:
Run `dep_install.sh` as root from the source folder

```
sudo dpkg -i libprotoc8_2.5.0-9ubuntu1_amd64.deb
```

* Install [protobuf-c-compiler](http://packages.ubuntu.com/trusty/amd64/protobuf-c-compiler/download)
from ubuntu 14.04LTS:

```
sudo dpkg -i protobuf-c-compiler_0.15-1build1_amd64.deb
```

__ATTENTION: Since Ubuntu trusty (14) is been dropped you can't download
these packages anymore, although you can run the script in the code root
`dep_install.sh` which will provide a full installation of protobuf2
untill we will support protobuf3.__
```
# dep_install.sh
```

__ATTENTION: This will download and install packpackagesets from unsigned source.__
__This is like this because we support only protobuf2 which is dropped__
__since ubuntu14.04 (we are planning to move on protobuf3)__

## Get WARP17
Get the `warp17-<ver>.tgz` archive or clone the desired
Expand Down
34 changes: 33 additions & 1 deletion build_dpdk.sh
View file
Expand Up @@ -115,13 +115,41 @@ function install {
if [[ -z $interactive ]]; then
confirm "Installing the dpdk lib"
fi
check_root
exec_cmd "Copying igb_uio in $kernel folder" cp $1/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko /lib/modules/$kernel/kernel/drivers/uio/
exec_cmd "Generating modules and map files." depmod -a
exec_cmd "Add uio mod" modprobe uio
exec_cmd "Add igb_uio mod" modprobe igb_uio
}

# Install dpdk dependecies
# ATTENTION: update at every new dpdk release supported [current state 17.11.5]
function get_deps {
exec_cmd "Installing required dependecies" sudo apt install -y build-essential libnuma-dev python ncurses-dev
}

function exports {
RTE_SDK="$dest/$name"

if [[ -n $SUDO_USER ]]; then
echo "ATTENTION"
echo "Run this script using command \"sudo -E\" or I won't able to check your env"
if [[ -z $interactive ]]; then
confirm "Writing the RTE_SDK in your profile?"
fi
fi

for line in $(env); do
if [[ -n $(echo $line | grep $RTE_SDK) ]]; then
die "you've $RTE_SDK already exported"
fi
if [[ -n $(cat $HOME/.bash_profile 2>/dev/null | grep $RTE_SDK) ||
-n $(cat $HOME/.bashrc 2>/dev/null | grep $RTE_SDK) ]]; then
die "you've $RTE_SDK already written"
fi
done
exec_cmd "" "echo RTE_SDK=$RTE_SDK >> $HOME/.bash_profile"
}

# Skipping in case dpdk is already there
if [[ -d "$dest/$name/x86_64-native-linuxapp-gcc/build" ]]; then
echo dpdk-$ver is already there
Expand All @@ -132,5 +160,9 @@ else

fi

check_root
update
get_deps
install "$dest/$name"
exports
exit
14 changes: 14 additions & 0 deletions common/common.sh
View file
Expand Up @@ -101,3 +101,17 @@ function usage {
die ${usage}
exit 1
}

# Update ubuntu
function update {
confirm "Do you want to upgrade you packages?"
apt update
apt upgrade -y
}

# Debug print function that adds a counter
# You can also provide your message at the end
function count_debug_print() {
let i=i+1
echo -e "I'm $i \t $@"
}
6 changes: 3 additions & 3 deletions dep_install.sh
View file
Expand Up @@ -86,9 +86,8 @@ files=(
)

# Warp17 and protobuf2 dependencies
function update {
apt-get update
apt-get install build-essential libnuma-dev python ncurses-dev zlib1g-dev python-pkg-resources
function dep_install {
apt install -y build-essential libnuma-dev python ncurses-dev zlib1g-dev python-pkg-resources
}

function get_packets {
Expand All @@ -112,6 +111,7 @@ function freeze_updates {

function install {
update
dep_install
exec_cmd "Preparing the working directory" mkdir -p $workdir
get_packets

Expand Down

0 comments on commit d80daba

Please sign in to comment.