diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 1c8fd04ed..3383bf25e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,149 +1,149 @@ -# name: Benchmark - -# on: -# issue_comment: -# types: created - -# jobs: -# benchmark_base: -# # On pull requests and if the comment starts with `/run-benchmark` -# if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/run-benchmark') -# runs-on: self-hosted - -# steps: -# - name: Checkout code -# uses: actions/checkout@v2 -# with: -# ref: main - -# - name: Get base branch commit ID -# id: get_base_commit -# run: echo "BASE_COMMIT_ID=$(git rev-parse HEAD)" > base_commit_id.txt - -# - name: Upload base commit ID -# uses: actions/upload-artifact@v3 -# with: -# name: base-commit-id -# path: base_commit_id.txt - -# - name: Set up Python -# uses: actions/setup-python@v2 -# with: -# python-version: '3.9' - -# - name: Create virtual environment -# run: python -m venv bitblas_benchmark - -# - name: Activate virtual environment and install dependencies -# run: | -# source bitblas_benchmark/bin/activate -# python -m pip install --upgrade pip -# if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi - -# - name: Install project in wheel mode -# run: | -# source bitblas_benchmark/bin/activate -# python -m pip install . +name: Benchmark + +on: + issue_comment: + types: created + +jobs: + benchmark_base: + # On pull requests and if the comment starts with `/run-benchmark` + if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/run-benchmark') + runs-on: self-hosted + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: main + + - name: Get base branch commit ID + id: get_base_commit + run: echo "BASE_COMMIT_ID=$(git rev-parse HEAD)" > base_commit_id.txt + + - name: Upload base commit ID + uses: actions/upload-artifact@v4 + with: + name: base-commit-id + path: base_commit_id.txt + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Create virtual environment + run: python -m venv bitblas_benchmark + + - name: Activate virtual environment and install dependencies + run: | + source bitblas_benchmark/bin/activate + python -m pip install --upgrade pip + if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi + + - name: Install project in wheel mode + run: | + source bitblas_benchmark/bin/activate + python -m pip install . -# - name: Matmul Benchmark -# run: | -# source bitblas_benchmark/bin/activate -# cd benchmark/operators -# python ./benchmark_ops_matmul.py - -# benchmark_head: -# # On pull requests and if the comment starts with `/run-benchmark` -# if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/run-benchmark') -# runs-on: self-hosted -# needs: [benchmark_base] - -# steps: -# - name: Checkout PR branch code -# uses: actions/checkout@v2 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.pull_request.head.ref }} - -# - name: Get PR branch commit ID -# id: get_pr_commit -# run: echo "PR_COMMIT_ID=$(git rev-parse HEAD)" > pr_commit_id.txt - -# - name: Upload PR commit ID -# uses: actions/upload-artifact@v3 -# with: -# name: pr-commit-id -# path: pr_commit_id.txt - -# - name: Set up Python -# uses: actions/setup-python@v2 -# with: -# python-version: '3.9' - -# - name: Create virtual environment -# run: python -m venv bitblas_benchmark - -# - name: Activate virtual environment and install dependencies -# run: | -# source bitblas_benchmark/bin/activate -# python -m pip install --upgrade pip -# if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi - -# - name: Install project in wheel mode -# run: | -# source bitblas_benchmark/bin/activate -# python -m pip install . + - name: Matmul Benchmark + run: | + source bitblas_benchmark/bin/activate + cd benchmark/operators + python ./benchmark_ops_matmul.py + + benchmark_head: + # On pull requests and if the comment starts with `/run-benchmark` + if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/run-benchmark') + runs-on: self-hosted + needs: [benchmark_base] + + steps: + - name: Checkout PR branch code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + + - name: Get PR branch commit ID + id: get_pr_commit + run: echo "PR_COMMIT_ID=$(git rev-parse HEAD)" > pr_commit_id.txt + + - name: Upload PR commit ID + uses: actions/upload-artifact@v4 + with: + name: pr-commit-id + path: pr_commit_id.txt + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Create virtual environment + run: python -m venv bitblas_benchmark + + - name: Activate virtual environment and install dependencies + run: | + source bitblas_benchmark/bin/activate + python -m pip install --upgrade pip + if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi + + - name: Install project in wheel mode + run: | + source bitblas_benchmark/bin/activate + python -m pip install . -# - name: Matmul Benchmark -# run: | -# source bitblas_benchmark/bin/activate -# cd benchmark/operators -# python ./benchmark_ops_matmul.py - -# benchmark_compare: -# if: github.event.issue.pull_request != null && contains(github.event.comment.body, '/run-benchmark') -# needs: [benchmark_base, benchmark_head] -# runs-on: self-hosted - -# steps: -# - name: Download commit IDs -# uses: actions/download-artifact@v4.1.7 -# with: -# name: base-commit-id -# path: . - -# - name: Download PR commit ID -# uses: actions/download-artifact@v4.1.7 -# with: -# name: pr-commit-id -# path: . - -# - name: Set up Python -# uses: actions/setup-python@v2 -# with: -# python-version: '3.9' - -# - name: Create virtual environment -# run: python -m venv bitblas_benchmark - -# - name: Activate virtual environment and install dependencies -# run: | -# source bitblas_benchmark/bin/activate -# python -m pip install --upgrade pip -# if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi - -# - name: Compare benchmark results -# run: | -# source bitblas_benchmark/bin/activate -# cd benchmark/operators -# python ./compare_benchmark.py --base $(cat base_commit_id.txt) --head $(cat pr_commit_id.txt) 2>&1 | tee compare_results.txt - -# - name: Authenticate GitHub CLI -# env: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# run: | -# echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token - -# - name: Post benchmark results -# run: | -# cat compare_results.txt -# gh pr comment ${{ github.event.issue.number }} --body "$(cat compare_results.txt)" + - name: Matmul Benchmark + run: | + source bitblas_benchmark/bin/activate + cd benchmark/operators + python ./benchmark_ops_matmul.py + + benchmark_compare: + if: github.event.issue.pull_request != null && contains(github.event.comment.body, '/run-benchmark') + needs: [benchmark_base, benchmark_head] + runs-on: self-hosted + + steps: + - name: Download commit IDs + uses: actions/download-artifact@v4.1.7 + with: + name: base-commit-id + path: . + + - name: Download PR commit ID + uses: actions/download-artifact@v4.1.7 + with: + name: pr-commit-id + path: . + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Create virtual environment + run: python -m venv bitblas_benchmark + + - name: Activate virtual environment and install dependencies + run: | + source bitblas_benchmark/bin/activate + python -m pip install --upgrade pip + if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi + + - name: Compare benchmark results + run: | + source bitblas_benchmark/bin/activate + cd benchmark/operators + python ./compare_benchmark.py --base $(cat base_commit_id.txt) --head $(cat pr_commit_id.txt) 2>&1 | tee compare_results.txt + + - name: Authenticate GitHub CLI + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token + + - name: Post benchmark results + run: | + cat compare_results.txt + gh pr comment ${{ github.event.issue.number }} --body "$(cat compare_results.txt)" diff --git a/README.md b/README.md index 24da9f8f9..ecf76ba5b 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Some of the key features of BitBLAS include: - Support customizing mixed-precision DNN operations for your specific scenarios via the flexible DSL (TIR Script). ## Latest News -- 11/04/2024 🚀🚀: We've supported high performance A INT4 x W INT4/INT2 Matmul. +- 11/04/2024 🚀🚀: We've supported high performance A INT4 x W INT4/INT2 Matmul for [BitNet a4.8](https://arxiv.org/pdf/2411.04965). - 10/02/2024 🚀🚀: We've added initial Flash Attention Ops and its implementation in Tilelang! Please refer to [PythonAPI](https://github.com/microsoft/BitBLAS/blob/main/docs/PythonAPI.md) and [QuickStart](https://github.com/microsoft/BitBLAS/blob/main/docs/QuickStart.md) docs and [PR #202](https://github.com/microsoft/BitBLAS/pull/202). - 08/12/2024 🚀🚀: We've improved performance for contiguous batching. To enable it, you'll need to set specific flags. For more details, please refer to [PR #133](https://github.com/microsoft/BitBLAS/pull/133). - 07/11/2024 ✨: Ladder is published and presented in OSDI'24. Please find [Ladder paper and presentation](https://www.usenix.org/conference/osdi24/presentation/wang-lei) if you are interested in the technical details of BitBLAS. diff --git a/docker/Dockerfile.cu120 b/docker/Dockerfile.cu120 new file mode 100644 index 000000000..46ebe7416 --- /dev/null +++ b/docker/Dockerfile.cu120 @@ -0,0 +1,28 @@ +FROM nvcr.io/nvidia/pytorch:23.01-py3 + +WORKDIR /root + +RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential git wget \ + libgtest-dev libprotobuf-dev protobuf-compiler libgflags-dev libsqlite3-dev llvm-dev \ + && apt-get clean autoclean && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/* + +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.5.2-0-Linux-x86_64.sh -O install_miniconda.sh && \ + bash install_miniconda.sh -b -p /opt/conda && rm install_miniconda.sh + +ENV PATH="/opt/conda/bin:${PATH}" + +ENV LIBGL_ALWAYS_INDIRECT=1 + +RUN conda install pip cmake && conda clean --all + +RUN apt-get install -y python3 python3-dev python3-setuptools gcc libtinfo-dev zlib1g-dev build-essential cmake libedit-dev libxml2-dev + +RUN git clone https://github.com/microsoft/BitBLAS.git --recursive -b main BitBLAS \ + && cd BitBLAS && ./install.sh + +CMD bash diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..563649e4f --- /dev/null +++ b/docker/README.md @@ -0,0 +1,10 @@ +To ease the process of installing all the dependencies, baseline software, and Ladder code, we provide a Dockerfile and a simple guideline to build a Docker image with all of above installed. The Docker image is built on top of Ubuntu 20.04, and it contains all the dependencies required to run the experiments. We only provide the Dockerfile for NVIDIA GPU, and the Dockerfile for AMD GPU will be provided upon request. + +```bash +git clone --recursive https://github.com/microsoft/BitBLAS BitBLAS +cd BitBLAS/docker +# build the image, this may take a while (around 10+ minutes on our test machine) +docker build -t bitblas_cuda -f Dockerfile.cu120 . +# run the container +docker run -it --cap-add=SYS_ADMIN --network=host --gpus all --cap-add=SYS_PTRACE --shm-size=4G --security-opt seccomp=unconfined --security-opt apparmor=unconfined --name bitblas_test bitblas_cuda bash +```