Skip to content

Commit

Permalink
Merge commit '474d0646f9d8744138d38ce6b3b12277c71799df' into catchup/…
Browse files Browse the repository at this point in the history
…long_lived_vault_from_main_474d0646f9d8744138d38ce6b3b12277c71799df
  • Loading branch information
Quexington committed Dec 4, 2023
2 parents ed67e74 + 474d064 commit f54934c
Show file tree
Hide file tree
Showing 128 changed files with 2,076 additions and 1,874 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/build-linux-installer-deb.yml
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ jobs:
run: |
py3createtorrent -f -t udp://tracker.opentrackr.org:1337/announce "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain_${CHIA_INSTALLER_VERSION}_${{ matrix.os.arch }}.deb -o "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain_${CHIA_INSTALLER_VERSION}_${{ matrix.os.arch }}.deb.torrent --webseed https://download.chia.net/install/chia-blockchain_${CHIA_INSTALLER_VERSION}_${{ matrix.os.arch }}.deb
py3createtorrent -f -t udp://tracker.opentrackr.org:1337/announce "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain-cli_${CHIA_INSTALLER_VERSION}-1_${{ matrix.os.arch }}.deb -o "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain-cli_${CHIA_INSTALLER_VERSION}-1_${{ matrix.os.arch }}.deb.torrent --webseed https://download.chia.net/install/chia-blockchain-cli_${CHIA_INSTALLER_VERSION}-1_${{ matrix.os.arch }}.deb
gh release upload $RELEASE_TAG "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain_${CHIA_INSTALLER_VERSION}_${{ matrix.os.arch }}.deb.torrent
gh release upload --repo ${{ github.repository }} $RELEASE_TAG "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain_${CHIA_INSTALLER_VERSION}_${{ matrix.os.arch }}.deb.torrent
- name: Upload Dev Installer
if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/main'
Expand All @@ -326,6 +326,7 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload \
--repo ${{ github.repository }} \
$RELEASE_TAG \
build_scripts/final_installer/chia-blockchain_${CHIA_INSTALLER_VERSION}_${{ matrix.os.arch }}.deb \
build_scripts/final_installer/chia-blockchain-cli_${CHIA_INSTALLER_VERSION}-1_${{ matrix.os.arch }}.deb
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/build-linux-installer-rpm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ jobs:
run: |
py3createtorrent -f -t udp://tracker.opentrackr.org:1337/announce "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm -o "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm.torrent --webseed https://download.chia.net/install/chia-blockchain-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm
py3createtorrent -f -t udp://tracker.opentrackr.org:1337/announce "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain-cli-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm -o "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain-cli-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm.torrent --webseed https://download.chia.net/install/chia-blockchain-cli-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm
gh release upload $RELEASE_TAG "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm.torrent
gh release upload --repo ${{ github.repository }} $RELEASE_TAG "$GITHUB_WORKSPACE"/build_scripts/final_installer/chia-blockchain-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm.torrent
- name: Upload Dev Installer
if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/main'
Expand All @@ -314,6 +314,7 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload \
--repo ${{ github.repository }} \
$RELEASE_TAG \
build_scripts/final_installer/chia-blockchain-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm \
build_scripts/final_installer/chia-blockchain-cli-${CHIA_INSTALLER_VERSION}-1.x86_64.rpm
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/build-macos-installers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ jobs:
run: |
py3createtorrent -f -t udp://tracker.opentrackr.org:1337/announce ${{ github.workspace }}/build_scripts/final_installer/Chia-${{ env.CHIA_INSTALLER_VERSION }}${{ matrix.os.file-suffix }}.dmg -o ${{ github.workspace }}/build_scripts/final_installer/Chia-${{ env.CHIA_INSTALLER_VERSION }}${{ matrix.os.file-suffix }}.dmg.torrent --webseed https://download.chia.net/install/Chia-${{ env.CHIA_INSTALLER_VERSION }}${{ matrix.os.file-suffix }}.dmg
ls ${{ github.workspace }}/build_scripts/final_installer/
gh release upload $RELEASE_TAG ${{ github.workspace }}/build_scripts/final_installer/Chia-${{ env.CHIA_INSTALLER_VERSION }}${{ matrix.os.file-suffix }}.dmg.torrent
gh release upload --repo ${{ github.repository }} $RELEASE_TAG ${{ github.workspace }}/build_scripts/final_installer/Chia-${{ env.CHIA_INSTALLER_VERSION }}${{ matrix.os.file-suffix }}.dmg.torrent
- name: Upload Dev Installer
if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/main'
Expand All @@ -377,6 +377,7 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload \
--repo ${{ github.repository }} \
$RELEASE_TAG \
build_scripts/final_installer/*.dmg
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/build-windows-installer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ jobs:
run: |
py3createtorrent -f -t udp://tracker.opentrackr.org:1337/announce "${GITHUB_WORKSPACE}"/chia-blockchain-gui/release-builds/windows-installer/ChiaSetup-${{ env.CHIA_INSTALLER_VERSION }}.exe -o "${GITHUB_WORKSPACE}"/chia-blockchain-gui/release-builds/windows-installer/ChiaSetup-${{ env.CHIA_INSTALLER_VERSION }}.exe.torrent --webseed https://download.chia.net/install/ChiaSetup-${{ env.CHIA_INSTALLER_VERSION }}.exe
ls
gh release upload $env:RELEASE_TAG "${GITHUB_WORKSPACE}"/chia-blockchain-gui/release-builds/windows-installer/ChiaSetup-${{ env.CHIA_INSTALLER_VERSION }}.exe.torrent
gh release upload --repo ${{ github.repository }} $RELEASE_TAG "${GITHUB_WORKSPACE}"/chia-blockchain-gui/release-builds/windows-installer/ChiaSetup-${{ env.CHIA_INSTALLER_VERSION }}.exe.torrent
- name: Upload Dev Installer
if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/main'
Expand All @@ -394,7 +394,7 @@ jobs:
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload $env:RELEASE_TAG "${GITHUB_WORKSPACE}"/chia-blockchain-gui/release-builds/windows-installer/ChiaSetup-${{ env.CHIA_INSTALLER_VERSION }}.exe
gh release upload --repo ${{ github.repository }} $env:RELEASE_TAG "${GITHUB_WORKSPACE}"/chia-blockchain-gui/release-builds/windows-installer/ChiaSetup-${{ env.CHIA_INSTALLER_VERSION }}.exe
- uses: Chia-Network/actions/github/jwt@main
if: steps.check_secrets.outputs.HAS_GLUE_SECRET
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ jobs:
diff-cover --compare-branch=${{ env.compare-branch }} --fail-under=100 --html-report=coverage-reports/diff-cover.html --markdown-report=coverage-reports/diff-cover.md coverage-reports/coverage.xml | tee coverage-reports/diff-cover-stdout
COV_STATUS="${PIPESTATUS[0]}"
echo "COV_STATUS=$COV_STATUS" >> "$GITHUB_ENV"
if [[ $COV_STATUS != '0' ]]; then
if [[ $COV_STATUS != '0' && "$GITHUB_BASE_REF" != '' ]]; then
exit 1
fi
Expand Down
6 changes: 5 additions & 1 deletion SECURITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ currently being supported with security updates.
| 1.3.x | :x: |
| 1.4.x | :x: |
| 1.5.x | :x: |
| 1.6.x | :white_check_mark: |
| 1.6.x | :x: |
| 1.7.x | :x: |
| 1.8.x | :x: |
| 2.0.x | :x: |
| 2.1.x | :white_check_mark: |

## Reporting a Vulnerability

Expand Down
7 changes: 1 addition & 6 deletions benchmarks/block_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
from chia.types.blockchain_format.sized_bytes import bytes32
from chia.types.blockchain_format.sub_epoch_summary import SubEpochSummary
from chia.types.full_block import FullBlock
from chia.util.db_wrapper import DBWrapper2
from chia.util.ints import uint8, uint32, uint64, uint128

# to run this benchmark:
Expand All @@ -44,7 +43,6 @@

async def run_add_block_benchmark(version: int) -> None:
verbose: bool = "--verbose" in sys.argv
db_wrapper: DBWrapper2 = await setup_db("block-store-benchmark.db", version)

# keep track of benchmark total time
all_test_time = 0.0
Expand All @@ -54,7 +52,7 @@ async def run_add_block_benchmark(version: int) -> None:

header_hashes = []

try:
async with setup_db("block-store-benchmark.db", version) as db_wrapper:
block_store = await BlockStore.create(db_wrapper)

block_height = 1
Expand Down Expand Up @@ -495,9 +493,6 @@ async def run_add_block_benchmark(version: int) -> None:
db_size = os.path.getsize(Path("block-store-benchmark.db"))
print(f"database size: {db_size/1000000:.3f} MB")

finally:
await db_wrapper.close()


if __name__ == "__main__":
print("version 2")
Expand Down
64 changes: 64 additions & 0 deletions benchmarks/blockchains.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
from __future__ import annotations

import asyncio
import cProfile
import time
from contextlib import contextmanager
from subprocess import check_call
from typing import Iterator

from chia.simulator.block_tools import create_block_tools_async, test_constants
from chia.simulator.keyring import TempKeyring
from chia.util.keyring_wrapper import KeyringWrapper
from tests.util.blockchain import persistent_blocks


@contextmanager
def enable_profiler(profile: bool, name: str) -> Iterator[None]:
if not profile:
yield
return

with cProfile.Profile() as pr:
yield

pr.create_stats()
output_file = f"{name}"
pr.dump_stats(output_file + ".profile")
check_call(["gprof2dot", "-f", "pstats", "-o", output_file + ".dot", output_file + ".profile"])
with open(output_file + ".png", "w+") as f:
check_call(["dot", "-T", "png", output_file + ".dot"], stdout=f)
print(" output written to: %s.png" % output_file)


async def run_test_chain_benchmark() -> None:
with TempKeyring() as keychain:
bt = await create_block_tools_async(constants=test_constants, keychain=keychain)
with enable_profiler(True, "load-test-chain"):
start = time.monotonic()
for version in ["", "_hardfork"]:
for count, name in [
(400, "test_blocks_400_rc5"),
(1000, "test_blocks_1000_rc5"),
(1000, "pre_genesis_empty_slots_1000_blocksrc5"),
(1500, "test_blocks_1500_rc5"),
(10000, "test_blocks_10000_rc5"),
(758 + 320, "test_blocks_long_reorg_rc5"),
(2000, "test_blocks_2000_compact_rc5"),
(10000, "test_blocks_10000_compact_rc5"),
]:
persistent_blocks(count, f"{name}{version}.db", bt, seed=b"100")
end = time.monotonic()
KeyringWrapper.cleanup_shared_instance()

print(f"time to load test chains: {end - start:.2f}s")


if __name__ == "__main__":
import logging

logger = logging.getLogger()
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.WARNING)

asyncio.run(run_test_chain_benchmark())
15 changes: 5 additions & 10 deletions benchmarks/coin_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
from chia.full_node.coin_store import CoinStore
from chia.types.blockchain_format.coin import Coin
from chia.types.blockchain_format.sized_bytes import bytes32
from chia.util.db_wrapper import DBWrapper2
from chia.util.ints import uint32, uint64

# to run this benchmark:
Expand Down Expand Up @@ -41,12 +40,11 @@ def make_coins(num: int) -> Tuple[List[Coin], List[bytes32]]:

async def run_new_block_benchmark(version: int) -> None:
verbose: bool = "--verbose" in sys.argv
db_wrapper: DBWrapper2 = await setup_db("coin-store-benchmark.db", version)

# keep track of benchmark total time
all_test_time = 0.0

try:
async with setup_db("coin-store-benchmark.db", version) as db_wrapper:
coin_store = await CoinStore.create(db_wrapper)

all_unspent: List[bytes32] = []
Expand Down Expand Up @@ -74,7 +72,7 @@ async def run_new_block_benchmark(version: int) -> None:
await coin_store.new_block(
uint32(height),
uint64(timestamp),
{pool_coin, farmer_coin},
[pool_coin, farmer_coin],
additions,
removals,
)
Expand Down Expand Up @@ -114,7 +112,7 @@ async def run_new_block_benchmark(version: int) -> None:
await coin_store.new_block(
uint32(height),
uint64(timestamp),
{pool_coin, farmer_coin},
[pool_coin, farmer_coin],
additions,
removals,
)
Expand Down Expand Up @@ -164,7 +162,7 @@ async def run_new_block_benchmark(version: int) -> None:
await coin_store.new_block(
uint32(height),
uint64(timestamp),
{pool_coin, farmer_coin},
[pool_coin, farmer_coin],
additions,
removals,
)
Expand Down Expand Up @@ -212,7 +210,7 @@ async def run_new_block_benchmark(version: int) -> None:
await coin_store.new_block(
uint32(height),
uint64(timestamp),
{pool_coin, farmer_coin},
[pool_coin, farmer_coin],
additions,
removals,
)
Expand Down Expand Up @@ -301,9 +299,6 @@ async def run_new_block_benchmark(version: int) -> None:
all_test_time += total_time
print(f"all tests completed in {all_test_time:0.4f}s")

finally:
await db_wrapper.close()

db_size = os.path.getsize(Path("coin-store-benchmark.db"))
print(f"database size: {db_size/1000000:.3f} MB")

Expand Down
38 changes: 18 additions & 20 deletions benchmarks/utils.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
from __future__ import annotations

import contextlib
import enum
import os
import random
import subprocess
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Generic, Optional, Tuple, Type, TypeVar, Union
from typing import Any, AsyncIterator, Generic, Optional, Tuple, Type, TypeVar, Union

import aiosqlite
import click
from chia_rs import AugSchemeMPL, G1Element, G2Element

Expand Down Expand Up @@ -179,30 +178,29 @@ def rand_full_block() -> FullBlock:
return full_block


async def setup_db(name: Union[str, os.PathLike[str]], db_version: int) -> DBWrapper2:
@contextlib.asynccontextmanager
async def setup_db(name: Union[str, os.PathLike[str]], db_version: int) -> AsyncIterator[DBWrapper2]:
db_filename = Path(name)
try:
os.unlink(db_filename)
except FileNotFoundError:
pass
connection = await aiosqlite.connect(db_filename)

def sql_trace_callback(req: str) -> None:
sql_log_path = "sql.log"
timestamp = datetime.now().strftime("%H:%M:%S.%f")
log = open(sql_log_path, "a")
log.write(timestamp + " " + req + "\n")
log.close()

log_path: Optional[Path]
if "--sql-logging" in sys.argv:
await connection.set_trace_callback(sql_trace_callback)

await connection.execute("pragma journal_mode=wal")
await connection.execute("pragma synchronous=full")

ret = DBWrapper2(connection, db_version)
await ret.add_connection(await aiosqlite.connect(db_filename))
return ret
log_path = Path("sql.log")
else:
log_path = None

async with DBWrapper2.managed(
database=db_filename,
log_path=log_path,
db_version=db_version,
reader_count=1,
journal_mode="wal",
synchronous="full",
) as db_wrapper:
yield db_wrapper


def get_commit_hash() -> str:
Expand Down
Empty file.
16 changes: 16 additions & 0 deletions build_scripts/assets/systemd/chia-crawler@.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
[Unit]
Description=Chia Crawler Service for %i
Requires=chia-daemon@%i.service
After=chia-daemon@%i.service

[Service]
Type=simple
Environment=CHIA_ROOT=/home/%i/.chia/mainnet
ExecStart=/opt/chia/start_crawler
User=%i
Group=%i
LimitNOFILE=1048576
LimitNPROC=1048576

[Install]
WantedBy=multi-user.target
16 changes: 16 additions & 0 deletions build_scripts/assets/systemd/chia-daemon@.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
[Unit]
Description=Chia Daemon Service for %i
StopWhenUnneeded=true

[Service]
Type=simple
Environment=CHIA_ROOT=/home/%i/.chia/mainnet
ExecStart=/opt/chia/daemon
ExecStartPost=/bin/bash -c '(while ! nc -z -v -w1 localhost 55400 2>/dev/null; do echo "Waiting for the daemon to listen on port 55400..."; sleep 1; done); sleep 1'
User=%i
Group=%i
LimitNOFILE=1048576
LimitNPROC=1048576

[Install]
WantedBy=multi-user.target
17 changes: 17 additions & 0 deletions build_scripts/assets/systemd/chia-data-layer-http@.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
[Unit]
Description=Chia Data Layer HTTP Service for %i
Requires=chia-daemon@%i.service
After=chia-daemon@%i.service

[Service]
Type=simple
Environment=CHIA_ROOT=/home/%i/.chia/mainnet
ExecStart=/opt/chia/start_data_layer_http
User=%i
Group=%i
LimitNOFILE=1048576
LimitNPROC=1048576
TimeoutStopSec=15

[Install]
WantedBy=multi-user.target
17 changes: 17 additions & 0 deletions build_scripts/assets/systemd/chia-data-layer@.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
[Unit]
Description=Chia Data Layer Service for %i
Requires=chia-daemon@%i.service
After=chia-daemon@%i.service

[Service]
Type=simple
Environment=CHIA_ROOT=/home/%i/.chia/mainnet
ExecStart=/opt/chia/start_data_layer
User=%i
Group=%i
LimitNOFILE=1048576
LimitNPROC=1048576
TimeoutStopSec=15

[Install]
WantedBy=multi-user.target

0 comments on commit f54934c

Please sign in to comment.