Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: Parallel Decomp - Global pool in separate class and... #3556

Open
wants to merge 8 commits into
base: devel
Choose a base branch
from
Open

WIP: Parallel Decomp - Global pool in separate class and... #3556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
28fca19
add parallel decompression support
HereThereBeDragons Feb 6, 2024
59d4ae2
move all data processing from CallbackCurlData
HereThereBeDragons Feb 9, 2024
e0adb2d
have a global queue in downloadmanager to reuse/prealloc tube element…
HereThereBeDragons Feb 9, 2024
7b304d4
WIP global pool but encapsulated in its own class; memory footprint c…
HereThereBeDragons Mar 26, 2024
c353d82
try to optimize sequential download
HereThereBeDragons Apr 2, 2024
24b2a4f
Split VerifyAndFinalize into multiple funcs and use the split version…
HereThereBeDragons Apr 4, 2024
0e2c64c
small fixes + add correct logoutput to parallel download code sections
HereThereBeDragons Apr 15, 2024
092bcf1
fix typos
HereThereBeDragons Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cvmfs/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ if (BUILD_CVMFS OR BUILD_LIBCVMFS)
network/dns.cc
network/download.cc
network/jobinfo.cc
network/parallel_download_coordinator.cc
network/sink_file.cc
network/sink_mem.cc
network/sink_path.cc
Expand Down Expand Up @@ -599,6 +600,7 @@ if (BUILD_SERVER)
network/dns.cc
network/download.cc
network/jobinfo.cc
network/parallel_download_coordinator.cc
network/s3fanout.cc
network/sink_file.cc
network/sink_mem.cc
Expand Down Expand Up @@ -740,6 +742,7 @@ if (BUILD_SERVER)
network/dns.cc
network/download.cc
network/jobinfo.cc
network/parallel_download_coordinator.cc
network/s3fanout.cc
network/sink_file.cc
network/sink_mem.cc
Expand Down Expand Up @@ -833,6 +836,7 @@ if (BUILD_SERVER)
network/dns.cc
network/download.cc
network/jobinfo.cc
network/parallel_download_coordinator.cc
network/sink_file.cc
network/sink_mem.cc
network/sink_path.cc
Expand Down Expand Up @@ -1038,6 +1042,7 @@ if(BUILD_RECEIVER)
network/dns.cc
network/download.cc
network/jobinfo.cc
network/parallel_download_coordinator.cc
network/s3fanout.cc
network/sink_file.cc
network/sink_mem.cc
Expand Down Expand Up @@ -1198,6 +1203,7 @@ if (BUILD_PRELOADER)
network/dns.cc
network/download.cc
network/jobinfo.cc
network/parallel_download_coordinator.cc
network/s3fanout.cc
network/sink_file.cc
network/sink_mem.cc
Expand Down
39 changes: 39 additions & 0 deletions cvmfs/mountpoint.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1526,6 +1526,45 @@ bool MountPoint::CreateDownloadManagers() {
}
}

// parallel decompression
int64_t parallel_dwld_min_buffers = -1;
int64_t parallel_dwld_max_buffers = -1;
int64_t parallel_dwld_inflight_buffers = -1;

if (options_mgr_->GetValue("CVMFS_PARALLEL_DOWNLOAD_MIN_BUFFERS", &optarg)) {
parallel_dwld_min_buffers = String2Int64(optarg);
if (parallel_dwld_min_buffers < 0) {
LogCvmfs(kLogCvmfs, kLogSyslogErr | kLogDebug,
"Ignoring CVMFS_PARALLEL_DOWNLOAD_MIN_BUFFERS as it must be >= 0. "
"Value given %s.", optarg.c_str());
parallel_dwld_min_buffers = -1;
}
}

if (options_mgr_->GetValue("CVMFS_PARALLEL_DOWNLOAD_MAX_BUFFERS", &optarg)) {
parallel_dwld_max_buffers = String2Int64(optarg);
if (parallel_dwld_max_buffers < 0) {
LogCvmfs(kLogCvmfs, kLogSyslogErr | kLogDebug,
"Ignoring CVMFS_PARALLEL_DOWNLOAD_MAX_BUFFERS as it must be >= 0. "
"Value given %s", optarg.c_str());
parallel_dwld_max_buffers = -1;
}
}

if (options_mgr_->GetValue("CVMFS_PARALLEL_DOWNLOAD_INFLIGHT_BUFFERS",
&optarg)) {
parallel_dwld_inflight_buffers = String2Int64(optarg);
if (parallel_dwld_inflight_buffers < 1) {
LogCvmfs(kLogCvmfs, kLogSyslogErr | kLogDebug,
"Ignoring CVMFS_PARALLEL_DOWNLOAD_INFLIGHT_BUFFERS as it must be >= 1."
" Value given %s", optarg.c_str());
parallel_dwld_inflight_buffers = -1;
}
}
download_mgr_->InitParallelDownload(parallel_dwld_min_buffers,
parallel_dwld_max_buffers,
parallel_dwld_inflight_buffers);

return SetupExternalDownloadMgr(do_geosort);
}

Expand Down
Loading
Loading