Skip to content

Commit

Permalink
Squashed commit of the following:
Browse files Browse the repository at this point in the history
commit 14cdc04
Author: Stephen Buttolph <stephen@avalabs.org>
Date:   Thu Mar 21 11:43:13 2024 -0400

    Remove useless bootstrapping metric (#2858)

commit 4d9bfdd
Author: marun <maru.newby@avalabs.org>
Date:   Thu Mar 21 16:42:00 2024 +0100

    `tmpnet`: Reuse dynamically-allocated API port across restarts (#2857)

commit 23e5417
Author: Stephen Buttolph <stephen@avalabs.org>
Date:   Wed Mar 20 01:48:04 2024 -0400

    Remove fallback validator height indexing (#2801)

commit 0118611
Author: Alberto Benegiamo <alberto.benegiamo@gmail.com>
Date:   Tue Mar 19 18:58:15 2024 +0100

    X-Chain - repackaged wallet backends (#2762)

    Co-authored-by: Stephen Buttolph <stephen@avalabs.org>

commit 9cef7d3
Author: Alberto Benegiamo <alberto.benegiamo@gmail.com>
Date:   Tue Mar 19 02:51:27 2024 +0100

    P-Chain - repackaged wallet backends (#2757)

    Co-authored-by: Stephen Buttolph <stephen@avalabs.org>

commit e88e565
Author: marun <maru.newby@avalabs.org>
Date:   Mon Mar 18 19:18:53 2024 +0100

    `tmpnet`: Enable collection of logs and metrics (#2820)

commit 6249bab
Author: Alberto Benegiamo <alberto.benegiamo@gmail.com>
Date:   Mon Mar 18 19:11:00 2024 +0100

    Dynamic Fees - Add E Upgrade boilerplate (#2597)

    Co-authored-by: Stephen Buttolph <stephen@avalabs.org>

commit 6a3661b
Author: aaronbuchwald <aaron.buchwald56@gmail.com>
Date:   Sun Mar 17 01:45:33 2024 -0400

    Remove verify height index (#2634)

    Co-authored-by: Stephen Buttolph <stephen@avalabs.org>

commit a18c4a3
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Date:   Sat Mar 16 16:38:40 2024 +0000

    Bump bufbuild/buf-setup-action from 1.29.0 to 1.30.0 (#2842)

    Signed-off-by: dependabot[bot] <support@github.com>
    Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
    Co-authored-by: Stephen Buttolph <stephen@avalabs.org>

commit b62846f
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Date:   Fri Mar 15 15:28:36 2024 +0000

    Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 (#2849)

    Signed-off-by: dependabot[bot] <support@github.com>
    Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
    Co-authored-by: Stephen Buttolph <stephen@avalabs.org>

commit 5eaa9c2
Author: Derek Rada <Dirrk@users.noreply.github.com>
Date:   Fri Mar 15 08:00:29 2024 -0700

    packer build (#2806)

    Co-authored-by: yevhenvolchenko <yevhenvolchenko@users.noreply.github.com>

commit 598018b
Author: marun <maru.newby@avalabs.org>
Date:   Thu Mar 14 20:38:07 2024 -0700

    `tmpnet`: Add a UUID to temporary networks to support metrics collection (#2763)

commit 4e2d005
Author: Dan Laine <daniel.laine@avalabs.org>
Date:   Thu Mar 14 16:22:59 2024 -0400

    update merkledb readme to specify key length is in bits (#2840)

Signed-off-by: Joshua Kim <20001595+joshua-kim@users.noreply.github.com>
  • Loading branch information
joshua-kim committed Mar 25, 2024
1 parent 93f90d9 commit 1acc175
Show file tree
Hide file tree
Showing 150 changed files with 2,419 additions and 3,232 deletions.
60 changes: 0 additions & 60 deletions .github/packer/ubuntu-jammy-x86_64-public-ami.json

This file was deleted.

81 changes: 81 additions & 0 deletions .github/packer/ubuntu-jammy-x86_64-public-ami.pkr.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
packer {
required_plugins {
amazon = {
source = "github.com/hashicorp/amazon"
version = "~> 1"
}
ansible = {
source = "github.com/hashicorp/ansible"
version = "~> 1"
}
}
}

variable "skip_create_ami" {
type = string
default = "${env("SKIP_CREATE_AMI")}"
}

variable "tag" {
type = string
default = "${env("TAG")}"
}

variable "version" {
type = string
default = "jammy-22.04"
}

data "amazon-ami" "autogenerated_1" {
filters = {
architecture = "x86_64"
name = "ubuntu/images/*ubuntu-${var.version}-*-server-*"
root-device-type = "ebs"
virtualization-type = "hvm"
}
most_recent = true
owners = ["099720109477"]
region = "us-east-1"
}

locals {
skip_create_ami = var.skip_create_ami == "True"
timestamp = regex_replace(timestamp(), "[- TZ:]", "")
clean_name = regex_replace(timestamp(), "[^a-zA-Z0-9-]", "-")
}

source "amazon-ebs" "autogenerated_1" {
ami_groups = ["all"]
ami_name = "public-avalanche-ubuntu-${var.version}-${var.tag}-${local.timestamp}"
instance_type = "c5.large"
region = "us-east-1"
skip_create_ami = local.skip_create_ami
source_ami = "${data.amazon-ami.autogenerated_1.id}"
ssh_username = "ubuntu"
tags = {
Base_AMI_Name = "{{ .SourceAMIName }}"
Name = "public-avalanche-ubuntu-${var.version}-${var.tag}-${local.clean_name}"
Release = "${var.version}"
}
}

build {
sources = ["source.amazon-ebs.autogenerated_1"]

provisioner "shell" {
inline = ["while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done", "wait_apt=$(ps aux | grep apt | wc -l)", "while [ \"$wait_apt\" -gt \"1\" ]; do echo \"waiting for apt to be ready....\"; wait_apt=$(ps aux | grep apt | wc -l); sleep 5; done", "sudo apt-get -y update", "sudo apt-get install -y python3-boto3 golang"]
}

provisioner "ansible" {
extra_arguments = ["-e", "component=public-ami build=packer os_release=jammy tag=${var.tag}"]
playbook_file = ".github/packer/create_public_ami.yml"
roles_path = ".github/packer/roles/"
use_proxy = false
}

provisioner "shell" {
execute_command = "sudo bash -x {{ .Path }}"
script = ".github/packer/clean-public-ami.sh"
}

}
2 changes: 1 addition & 1 deletion .github/workflows/buf-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: bufbuild/buf-setup-action@v1.29.0
- uses: bufbuild/buf-setup-action@v1.30.0
- uses: bufbuild/buf-push-action@v1
with:
input: "proto"
Expand Down
20 changes: 19 additions & 1 deletion .github/workflows/build-public-ami.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ on:
tags:
- "*"

env:
PACKER_VERSION: "1.10.2"
PYTHON3_BOTO3_VERSION: "1.20.34+dfsg-1"

jobs:
build-public-ami-and-upload:
runs-on: ubuntu-22.04
Expand All @@ -26,7 +30,7 @@ jobs:
- name: Install aws cli
run: |
sudo apt update
sudo apt-get -y install packer python3-boto3
sudo apt-get -y install python3-boto3=${PYTHON3_BOTO3_VERSION}
- name: Get the tag
id: get_tag
Expand All @@ -53,6 +57,20 @@ jobs:
aws-secret-access-key: ${{ secrets.MARKETPLACE_KEY }}
aws-region: us-east-1

- name: Setup `packer`
uses: hashicorp/setup-packer@main
id: setup
with:
version: ${{ env.PACKER_VERSION }}

- name: Run `packer init`
id: init
run: "packer init ./.github/packer/ubuntu-jammy-x86_64-public-ami.pkr.hcl"

- name: Run `packer validate`
id: validate
run: "packer validate ./.github/packer/ubuntu-jammy-x86_64-public-ami.pkr.hcl"

- name: Create AMI and upload to marketplace
run: |
./.github/workflows/update-ami.py
Expand Down
97 changes: 91 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ concurrency:

env:
go_version: '~1.21.8'
tmpnet_data_path: ~/.tmpnet/networks/1000
grafana_url: https://grafana-experimental.avax-dev.network/d/kBQpRdWnk/avalanche-main-dashboard?orgId=1&refresh=10s&var-filter=is_ephemeral_node%7C%3D%7Cfalse&var-filter=gh_repo%7C%3D%7Cava-labs%2Favalanchego&var-filter=gh_run_id%7C%3D%7C${{ github.run_id }}&var-filter=gh_run_attempt%7C%3D%7C${{ github.run_attempt }}

jobs:
Unit:
Expand Down Expand Up @@ -67,15 +67,44 @@ jobs:
- name: Build AvalancheGo Binary
shell: bash
run: ./scripts/build.sh -r
- name: Start prometheus
shell: bash
run: bash -x ./scripts/run_prometheus.sh
env:
PROMETHEUS_ID: ${{ secrets.PROMETHEUS_ID }}
PROMETHEUS_PASSWORD: ${{ secrets.PROMETHEUS_PASSWORD }}
- name: Start promtail
shell: bash
run: bash -x ./scripts/run_promtail.sh
env:
LOKI_ID: ${{ secrets.LOKI_ID }}
LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
- name: Notify of metrics availability
shell: bash
run: .github/workflows/notify-metrics-availability.sh
env:
GRAFANA_URL: ${{ env.grafana_url }}
GH_JOB_ID: ${{ github.job }}
FILTER_BY_OWNER: avalanchego-e2e
- name: Run e2e tests
shell: bash
run: E2E_SERIAL=1 ./scripts/tests.e2e.sh
env:
GH_REPO: ${{ github.repository }}
GH_WORKFLOW: ${{ github.workflow }}
GH_RUN_ID: ${{ github.run_id }}
GH_RUN_NUMBER: ${{ github.run_number }}
GH_RUN_ATTEMPT: ${{ github.run_attempt }}
GH_JOB_ID: ${{ github.job }}
- name: Upload tmpnet network dir
uses: actions/upload-artifact@v4
if: always()
with:
name: e2e-tmpnet-data
path: ${{ env.tmpnet_data_path }}
path: |
~/.tmpnet/networks
~/.tmpnet/prometheus/prometheus.log
~/.tmpnet/promtail/promtail.log
if-no-files-found: error
e2e_existing_network:
runs-on: ubuntu-latest
Expand All @@ -88,15 +117,43 @@ jobs:
- name: Build AvalancheGo Binary
shell: bash
run: ./scripts/build.sh -r
- name: Start prometheus
shell: bash
run: bash -x ./scripts/run_prometheus.sh
env:
PROMETHEUS_ID: ${{ secrets.PROMETHEUS_ID }}
PROMETHEUS_PASSWORD: ${{ secrets.PROMETHEUS_PASSWORD }}
- name: Start promtail
shell: bash
run: bash -x ./scripts/run_promtail.sh
env:
LOKI_ID: ${{ secrets.LOKI_ID }}
LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
- name: Notify of metrics availability
shell: bash
run: .github/workflows/notify-metrics-availability.sh
env:
GRAFANA_URL: ${{ env.grafana_url }}
GH_JOB_ID: ${{ github.job }}
- name: Run e2e tests with existing network
shell: bash
run: E2E_SERIAL=1 ./scripts/tests.e2e.existing.sh
env:
GH_REPO: ${{ github.repository }}
GH_WORKFLOW: ${{ github.workflow }}
GH_RUN_ID: ${{ github.run_id }}
GH_RUN_NUMBER: ${{ github.run_number }}
GH_RUN_ATTEMPT: ${{ github.run_attempt }}
GH_JOB_ID: ${{ github.job }}
- name: Upload tmpnet network dir
uses: actions/upload-artifact@v4
if: always()
with:
name: e2e-existing-network-tmpnet-data
path: ${{ env.tmpnet_data_path }}
path: |
~/.tmpnet/networks
~/.tmpnet/prometheus/prometheus.log
~/.tmpnet/promtail/promtail.log
if-no-files-found: error
Upgrade:
runs-on: ubuntu-latest
Expand All @@ -109,15 +166,43 @@ jobs:
- name: Build AvalancheGo Binary
shell: bash
run: ./scripts/build.sh
- name: Start prometheus
shell: bash
run: bash -x ./scripts/run_prometheus.sh
env:
PROMETHEUS_ID: ${{ secrets.PROMETHEUS_ID }}
PROMETHEUS_PASSWORD: ${{ secrets.PROMETHEUS_PASSWORD }}
- name: Start promtail
shell: bash
run: bash -x ./scripts/run_promtail.sh
env:
LOKI_ID: ${{ secrets.LOKI_ID }}
LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
- name: Notify of metrics availability
shell: bash
run: .github/workflows/notify-metrics-availability.sh
env:
GRAFANA_URL: ${{ env.grafana_url }}
GH_JOB_ID: ${{ github.job }}
- name: Run e2e tests
shell: bash
run: ./scripts/tests.upgrade.sh
env:
GH_REPO: ${{ github.repository }}
GH_WORKFLOW: ${{ github.workflow }}
GH_RUN_ID: ${{ github.run_id }}
GH_RUN_NUMBER: ${{ github.run_number }}
GH_RUN_ATTEMPT: ${{ github.run_attempt }}
GH_JOB_ID: ${{ github.job }}
- name: Upload tmpnet network dir
uses: actions/upload-artifact@v4
if: always()
with:
name: upgrade-tmpnet-data
path: ${{ env.tmpnet_data_path }}
path: |
~/.tmpnet/networks
~/.tmpnet/prometheus/prometheus.log
~/.tmpnet/promtail/promtail.log
if-no-files-found: error
Lint:
runs-on: ubuntu-latest
Expand All @@ -138,7 +223,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: bufbuild/buf-setup-action@v1.29.0
- uses: bufbuild/buf-setup-action@v1.30.0
with:
github_token: ${{ github.token }}
- uses: bufbuild/buf-lint-action@v1
Expand All @@ -153,7 +238,7 @@ jobs:
with:
go-version: ${{ env.go_version }}
check-latest: true
- uses: bufbuild/buf-setup-action@v1.29.0
- uses: bufbuild/buf-setup-action@v1.30.0
- shell: bash
run: scripts/protobuf_codegen.sh
- shell: bash
Expand Down
19 changes: 19 additions & 0 deletions .github/workflows/notify-metrics-availability.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env bash

set -euo pipefail

# Timestamps are in seconds
from_timestamp="$(date '+%s')"
monitoring_period=900 # 15 minutes
to_timestamp="$((from_timestamp + monitoring_period))"

# Grafana expects microseconds, so pad timestamps with 3 zeros
metrics_url="${GRAFANA_URL}&var-filter=gh_job_id%7C%3D%7C${GH_JOB_ID}&from=${from_timestamp}000&to=${to_timestamp}000"

# Optionally ensure that the link displays metrics only for the shared
# network rather than mixing it with the results for private networks.
if [[ -n "${FILTER_BY_OWNER:-}" ]]; then
metrics_url="${metrics_url}&var-filter=network_owner%7C%3D%7C${FILTER_BY_OWNER}"
fi

echo "::notice links::metrics ${metrics_url}"

0 comments on commit 1acc175

Please sign in to comment.