Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

All payment tweaks #674

Merged
merged 36 commits into from
Aug 30, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
321cd19
feat!(protocol): remove chunk merkletree to simplify payment
joshuef Aug 16, 2023
f670c09
feat!(protocol): get price and pay for each chunk individually
joshuef Aug 16, 2023
9a9cb25
feat!(protocol): gets keys with GetStoreCost
joshuef Aug 16, 2023
38325c4
feat!: pay each chunk holder direct
joshuef Aug 16, 2023
9900db9
feat(node): store data if the majority of CLOSE_GROUP will
joshuef Aug 18, 2023
faafb20
test(node): reenable payment fail check
joshuef Aug 18, 2023
419623e
chore(node): clarify payment errors
joshuef Aug 18, 2023
7533454
chore(node): only store paid for data, ignore maj
joshuef Aug 18, 2023
c0539ee
chore(networking): return all GetStoreCost prices and use them
joshuef Aug 18, 2023
8ab67aa
chore(client): error out early for invalid transfers
joshuef Aug 20, 2023
940a885
feat: one transfer per data set, mapped dbcs to content addrs
joshuef Aug 20, 2023
422137b
test(node): data verification test refactors for readability
joshuef Aug 23, 2023
6cd33dc
chore(node): data verification log tweaks
joshuef Aug 23, 2023
5ac7129
fix: not check payment for relocated holder
maqi Aug 23, 2023
c3fbeca
fix: correct replicated spend validation
maqi Aug 23, 2023
7f21f29
fix(node): handling events should wait before connected to the network
RolandSherwin Jul 25, 2023
2ad6a61
fix(network): trigger bootstrap until we have enough peers
RolandSherwin Aug 23, 2023
27d4f0c
ci: add missing package specification to dbc spend tests
joshuef Aug 25, 2023
dff6293
ci: increase node mem limit with increased dbc processing
joshuef Aug 25, 2023
97d5187
fix(protocol): avoid panics
joshuef Aug 25, 2023
330936a
chore: increase concurrent fetches for replication data
joshuef Aug 25, 2023
3ed2589
test: parallelise churn data final query
joshuef Aug 25, 2023
5051481
chore(networking): ensure we're always driving forward replication if…
joshuef Aug 25, 2023
4f1237f
chore(client): reduce transferoutputs cloning
joshuef Aug 25, 2023
3d73477
chore(client): pass around content payments map mut ref
joshuef Aug 25, 2023
334c1e6
feat: refactor to allow greater upload parallelisation
joshuef Aug 25, 2023
c466c8e
chore(client): refactor client wallet to reduce dbc clones
joshuef Aug 26, 2023
b70223c
chore: mem_check test update
maqi Aug 28, 2023
0dd0452
chore(deps): bump tokio to 1.32.0
RolandSherwin Aug 29, 2023
6f23a27
fix(tokio): remove tokio fs
RolandSherwin Aug 29, 2023
d1329bb
chore: trival clean ups
maqi Aug 29, 2023
9fa1968
feat(protocol): add logs for `RecordHeader` serde
RolandSherwin Aug 25, 2023
f855706
chore(logs): add more spend PUT validation logs
RolandSherwin Aug 25, 2023
c02cf0f
ci: increase client mem usage limit
joshuef Aug 30, 2023
f56694f
test(node): refactor churn test order
joshuef Aug 30, 2023
d32eab4
chore(networking): increase FETCH_TIMEOUT to 10s
joshuef Aug 30, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/benchmark-prs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ jobs:
shell: bash
run: |
client_peak_mem_limit_mb="2000" # mb
client_avg_mem_limit_mb="700" # mb
client_avg_mem_limit_mb="1500" # mb

peak_mem_usage=$(
rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs/safe.* -o --no-line-number --no-filename |
Expand Down Expand Up @@ -237,7 +237,7 @@ jobs:
- name: Check node memory usage
shell: bash
run: |
node_peak_mem_limit_mb="100" # mb
node_peak_mem_limit_mb="250" # mb
peak_mem_usage=$(
rg '"memory_used_mb":[^,]*' $NODE_DATA_PATH/*/logs/* -o --no-line-number --no-filename |
awk -F':' '/"memory_used_mb":/{print $2}' |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/generate-benchmark-charts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ jobs:
- name: Check node memory usage
shell: bash
run: |
node_peak_mem_limit_mb="100" # mb
node_peak_mem_limit_mb="250" # mb
peak_mem_usage=$(
rg '"memory_used_mb":[^,]*' $NODE_DATA_PATH/*/logs/* -o --no-line-number --no-filename |
awk -F':' '/"memory_used_mb":/{print $2}' |
Expand Down Expand Up @@ -162,7 +162,7 @@ jobs:
shell: bash
run: |
client_peak_mem_limit_mb="2000" # mb
client_avg_mem_limit_mb="700" # mb
client_avg_mem_limit_mb="1500" # mb

peak_mem_usage=$(
rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs/safe.* -o --no-line-number --no-filename |
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/memcheck.yml
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,12 @@ jobs:
timeout-minutes: 25

- name: Chunks data integrity during nodes churn
run: cargo test --release -p sn_node --test data_with_churn -- --nocapture
run: cargo test --release -p sn_node --test data_with_churn -- --nocapture
env:
TEST_DURATION_MINS: 15
TEST_TOTAL_CHURN_CYCLES: 15
SN_LOG: "all"
timeout-minutes: 20
timeout-minutes: 30

- name: Verify restart of nodes using rg
shell: bash
Expand Down Expand Up @@ -151,7 +151,7 @@ jobs:
received_list_count=$(rg "Replicate list received from" $NODE_DATA_PATH -c --stats | \
rg "(\d+) matches" | rg "\d+" -o)
echo "Received $received_list_count replication lists"
fetching_attempt_count=$(rg "Fetching replication" $NODE_DATA_PATH -c --stats | \
fetching_attempt_count=$(rg "FetchingKeysForReplication" $NODE_DATA_PATH -c --stats | \
rg "(\d+) matches" | rg "\d+" -o)
echo "Carried out $fetching_attempt_count fetching attempts"
replication_attempt_count=$(rg "Replicating chunk" $NODE_DATA_PATH -c --stats | \
Expand Down Expand Up @@ -206,7 +206,7 @@ jobs:
# The memory usage here will be significantly higher here than in the benchmark test,
# where we don't have a bootstrap node.
run: |
node_peak_mem_limit_mb="160" # mb
node_peak_mem_limit_mb="300" # mb

peak_mem_usage=$(
rg '"memory_used_mb":[^,]*' $NODE_DATA_PATH/*/logs/* -o --no-line-number --no-filename |
Expand All @@ -226,7 +226,7 @@ jobs:
shell: bash
run: |
client_peak_mem_limit_mb="2000" # mb
client_avg_mem_limit_mb="700" # mb
client_avg_mem_limit_mb="1500" # mb

peak_mem_usage=$(
rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs/safe.* -o --no-line-number --no-filename |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -262,14 +262,14 @@ jobs:
fi

- name: execute the dbc spend tests
run: cargo test --release --features="local-discovery" --test sequential_transfers -- --nocapture
run: cargo test --release -p sn_node --features="local-discovery" --test sequential_transfers -- --nocapture
env:
SN_LOG: "all"
CARGO_TARGET_DIR: "./transfer-target"
timeout-minutes: 25

- name: execute the storage payment tests
run: cargo test --release --features="local-discovery" --test storage_payments -- --nocapture --test-threads=1
run: cargo test --release -p sn_node --features="local-discovery" --test storage_payments -- --nocapture --test-threads=1
env:
SN_LOG: "all"
CARGO_TARGET_DIR: "./transfer-target"
Expand Down
45 changes: 4 additions & 41 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion sn_cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ sn_transfers = { path = "../sn_transfers", version = "0.10.28" }
sn_logging = { path = "../sn_logging", version = "0.2.4" }
sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.1.5" }
sn_protocol = { path = "../sn_protocol", version = "0.5.3" }
tokio = { version = "1.17.0", features = ["fs", "io-util", "macros", "parking_lot", "rt", "sync", "time"] }
tokio = { version = "1.32.0", features = ["io-util", "macros", "parking_lot", "rt", "sync", "time"] }
tracing = { version = "~0.1.26" }
tracing-core = "0.1.30"
url = "2.4.0"
Expand Down
16 changes: 8 additions & 8 deletions sn_cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ async fn main() -> Result<()> {
println!("Built with git version: {}", sn_build_info::git_info());
println!("Instantiating a SAFE client...");

let client_data_dir_path = get_client_data_dir_path().await?;
let secret_key = get_client_secret_key(&client_data_dir_path).await?;
let client_data_dir_path = get_client_data_dir_path()?;
let secret_key = get_client_secret_key(&client_data_dir_path)?;

if opt.peers.peers.is_empty() {
if !cfg!(feature = "local-discovery") {
Expand Down Expand Up @@ -89,27 +89,27 @@ async fn main() -> Result<()> {
Ok(())
}

async fn get_client_secret_key(root_dir: &PathBuf) -> Result<SecretKey> {
fn get_client_secret_key(root_dir: &PathBuf) -> Result<SecretKey> {
// create the root directory if it doesn't exist
tokio::fs::create_dir_all(&root_dir).await?;
std::fs::create_dir_all(root_dir)?;
let key_path = root_dir.join(CLIENT_KEY);
let secret_key = if key_path.is_file() {
info!("Client key found. Loading from file...");
let secret_hex_bytes = tokio::fs::read(key_path).await?;
let secret_hex_bytes = std::fs::read(key_path)?;
bls_secret_from_hex(secret_hex_bytes)?
} else {
info!("No key found. Generating a new client key...");
let secret_key = SecretKey::random();
tokio::fs::write(key_path, hex::encode(secret_key.to_bytes())).await?;
std::fs::write(key_path, hex::encode(secret_key.to_bytes()))?;
secret_key
};
Ok(secret_key)
}

async fn get_client_data_dir_path() -> Result<PathBuf> {
fn get_client_data_dir_path() -> Result<PathBuf> {
let mut home_dirs = dirs_next::data_dir().expect("Data directory is obtainable");
home_dirs.push("safe");
home_dirs.push("client");
tokio::fs::create_dir_all(home_dirs.as_path()).await?;
std::fs::create_dir_all(home_dirs.as_path())?;
Ok(home_dirs)
}