Skip to content

Commit d60c817

Browse files
ttytmttytm
authored
vpm: implement multithreading (#19208)
1 parent 5cafdea commit d60c817

File tree

1 file changed

+122
-40
lines changed

1 file changed

+122
-40
lines changed

‎cmd/tools/vpm.v

Lines changed: 122 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import net.urllib
1111
import json
1212
import v.help
1313
import v.vmod
14+
import sync.pool
1415

1516
const (
1617
default_vpm_server_urls = ['https://vpm.vlang.io', 'https://vpm.url4e.com']
@@ -21,20 +22,20 @@ const (
2122
supported_vcs_systems = ['git', 'hg']
2223
supported_vcs_folders = ['.git', '.hg']
2324
supported_vcs_update_cmds = {
24-
'git': 'git pull --recurse-submodules' // pulling with `--depth=1` leads to conflicts, when the upstream is more than 1 commit newer
25-
'hg': 'hg pull --update'
25+
'git': 'pull --recurse-submodules' // pulling with `--depth=1` leads to conflicts, when the upstream is more than 1 commit newer
26+
'hg': 'pull --update'
2627
}
2728
supported_vcs_install_cmds = {
28-
'git': 'git clone --depth=1 --recursive --shallow-submodules'
29-
'hg': 'hg clone'
29+
'git': 'clone --depth=1 --recursive --shallow-submodules'
30+
'hg': 'clone'
3031
}
3132
supported_vcs_outdated_steps = {
32-
'git': ['git fetch', 'git rev-parse @', 'git rev-parse @{u}']
33-
'hg': ['hg incoming']
33+
'git': ['fetch', 'rev-parse @', 'rev-parse @{u}']
34+
'hg': ['incoming']
3435
}
3536
supported_vcs_version_cmds = {
36-
'git': 'git version'
37-
'hg': 'hg version'
37+
'git': 'version'
38+
'hg': 'version'
3839
}
3940
)
4041

@@ -231,7 +232,7 @@ fn vpm_install_from_vpm(module_names []string) {
231232
errors++
232233
eprintln('Errors while incrementing the download count for ${name}:')
233234
}
234-
vcs_install_cmd := supported_vcs_install_cmds[vcs]
235+
vcs_install_cmd := '${vcs} ${supported_vcs_install_cmds[vcs]}'
235236
cmd := '${vcs_install_cmd} "${mod.url}" "${minfo.final_module_path}"'
236237
verbose_println(' command: ${cmd}')
237238
cmdres := os.execute(cmd)
@@ -255,7 +256,7 @@ fn print_failed_cmd(cmd string, cmdres os.Result) {
255256

256257
fn ensure_vcs_is_installed(vcs string) bool {
257258
mut res := true
258-
cmd := supported_vcs_version_cmds[vcs]
259+
cmd := '${vcs} ${supported_vcs_version_cmds[vcs]}'
259260
cmdres := os.execute(cmd)
260261
if cmdres.exit_code != 0 {
261262
print_failed_cmd(cmd, cmdres)
@@ -299,7 +300,7 @@ fn vpm_install_from_vcs(module_names []string, vcs_key string) {
299300
continue
300301
}
301302
println('Installing module "${name}" from "${url}" to "${final_module_path}" ...')
302-
vcs_install_cmd := supported_vcs_install_cmds[vcs_key]
303+
vcs_install_cmd := '${vcs_key} ${supported_vcs_install_cmds[vcs_key]}'
303304
cmd := '${vcs_install_cmd} "${url}" "${final_module_path}"'
304305
verbose_println(' command: ${cmd}')
305306
cmdres := os.execute(cmd)
@@ -394,6 +395,45 @@ fn vpm_install(module_names []string, source Source) {
394395
}
395396
}
396397

398+
pub struct ModUpdateInfo {
399+
mut:
400+
name string
401+
final_path string
402+
has_err bool
403+
}
404+
405+
fn update_module(mut pp pool.PoolProcessor, idx int, wid int) &ModUpdateInfo {
406+
mut result := &ModUpdateInfo{
407+
name: pp.get_item[string](idx)
408+
}
409+
zname := url_to_module_name(result.name)
410+
result.final_path = valid_final_path_of_existing_module(result.name) or { return result }
411+
println('Updating module "${zname}" in "${result.final_path}" ...')
412+
vcs := vcs_used_in_dir(result.final_path) or { return result }
413+
if !ensure_vcs_is_installed(vcs[0]) {
414+
result.has_err = true
415+
println('VPM needs `${vcs}` to be installed.')
416+
return result
417+
}
418+
path_flag := if vcs[0] == 'hg' { '-R' } else { '-C' }
419+
vcs_cmd := '${vcs[0]} ${path_flag} "${result.final_path}" ${supported_vcs_update_cmds[vcs[0]]}'
420+
verbose_println(' command: ${vcs_cmd}')
421+
vcs_res := os.execute('${vcs_cmd}')
422+
if vcs_res.exit_code != 0 {
423+
result.has_err = true
424+
println('Failed updating module "${zname}" in "${result.final_path}".')
425+
print_failed_cmd(vcs_cmd, vcs_res)
426+
return result
427+
} else {
428+
verbose_println(' ${vcs_res.output.trim_space()}')
429+
increment_module_download_count(zname) or {
430+
result.has_err = true
431+
eprintln('Errors while incrementing the download count for ${zname}:')
432+
}
433+
}
434+
return result
435+
}
436+
397437
fn vpm_update(m []string) {
398438
mut module_names := m.clone()
399439
if settings.is_help {
@@ -403,19 +443,34 @@ fn vpm_update(m []string) {
403443
if module_names.len == 0 {
404444
module_names = get_installed_modules()
405445
}
446+
if settings.is_verbose {
447+
vpm_update_verbose(module_names)
448+
return
449+
}
450+
mut pp := pool.new_pool_processor(callback: update_module)
451+
pp.work_on_items(module_names)
452+
for res in pp.get_results[ModUpdateInfo]() {
453+
if res.has_err {
454+
exit(1)
455+
}
456+
resolve_dependencies(res.name, res.final_path, module_names)
457+
}
458+
}
459+
460+
fn vpm_update_verbose(module_names []string) {
406461
mut errors := 0
407-
for modulename in module_names {
408-
zname := url_to_module_name(modulename)
409-
final_module_path := valid_final_path_of_existing_module(modulename) or { continue }
410-
os.chdir(final_module_path) or {}
462+
for name in module_names {
463+
zname := url_to_module_name(name)
464+
final_module_path := valid_final_path_of_existing_module(name) or { continue }
411465
println('Updating module "${zname}" in "${final_module_path}" ...')
412466
vcs := vcs_used_in_dir(final_module_path) or { continue }
413467
if !ensure_vcs_is_installed(vcs[0]) {
414468
errors++
415469
println('VPM needs `${vcs}` to be installed.')
416470
continue
417471
}
418-
vcs_cmd := supported_vcs_update_cmds[vcs[0]]
472+
path_flag := if vcs[0] == 'hg' { '-R' } else { '-C' }
473+
vcs_cmd := '${vcs[0]} ${path_flag} "${final_module_path}" ${supported_vcs_update_cmds[vcs[0]]}'
419474
verbose_println(' command: ${vcs_cmd}')
420475
vcs_res := os.execute('${vcs_cmd}')
421476
if vcs_res.exit_code != 0 {
@@ -430,39 +485,66 @@ fn vpm_update(m []string) {
430485
eprintln('Errors while incrementing the download count for ${zname}:')
431486
}
432487
}
433-
resolve_dependencies(modulename, final_module_path, module_names)
488+
resolve_dependencies(name, final_module_path, module_names)
434489
}
435490
if errors > 0 {
436491
exit(1)
437492
}
438493
}
439494

495+
pub struct ModDateInfo {
496+
name string
497+
mut:
498+
outdated bool
499+
exec_err bool
500+
}
501+
502+
fn get_mod_date_info(mut pp pool.PoolProcessor, idx int, wid int) &ModDateInfo {
503+
mut result := &ModDateInfo{
504+
name: pp.get_item[string](idx)
505+
}
506+
final_module_path := valid_final_path_of_existing_module(result.name) or { return result }
507+
vcs := vcs_used_in_dir(final_module_path) or { return result }
508+
is_hg := vcs[0] == 'hg'
509+
vcs_cmd_steps := supported_vcs_outdated_steps[vcs[0]]
510+
mut outputs := []string{}
511+
for step in vcs_cmd_steps {
512+
path_flag := if is_hg { '-R' } else { '-C' }
513+
cmd := '${vcs[0]} ${path_flag} "${final_module_path}" ${step}'
514+
res := os.execute('${cmd}')
515+
if res.exit_code < 0 {
516+
verbose_println('Error command: ${cmd}')
517+
verbose_println('Error details:\n${res.output}')
518+
result.exec_err = true
519+
return result
520+
}
521+
if is_hg {
522+
if res.exit_code == 1 {
523+
result.outdated = true
524+
return result
525+
}
526+
} else {
527+
outputs << res.output
528+
}
529+
}
530+
// vcs[0] == 'git'
531+
if !is_hg && outputs[1] != outputs[2] {
532+
result.outdated = true
533+
}
534+
return result
535+
}
536+
440537
fn get_outdated() ![]string {
441538
module_names := get_installed_modules()
442539
mut outdated := []string{}
443-
for name in module_names {
444-
final_module_path := valid_final_path_of_existing_module(name) or { continue }
445-
os.chdir(final_module_path) or {}
446-
vcs := vcs_used_in_dir(final_module_path) or { continue }
447-
vcs_cmd_steps := supported_vcs_outdated_steps[vcs[0]]
448-
mut outputs := []string{}
449-
for step in vcs_cmd_steps {
450-
res := os.execute(step)
451-
if res.exit_code < 0 {
452-
verbose_println('Error command: ${step}')
453-
verbose_println('Error details:\n${res.output}')
454-
return error('Error while checking latest commits for "${name}" .')
455-
}
456-
if vcs[0] == 'hg' {
457-
if res.exit_code == 1 {
458-
outdated << name
459-
}
460-
} else {
461-
outputs << res.output
462-
}
463-
}
464-
if vcs[0] == 'git' && outputs[1] != outputs[2] {
465-
outdated << name
540+
mut pp := pool.new_pool_processor(callback: get_mod_date_info)
541+
pp.work_on_items(module_names)
542+
for res in pp.get_results[ModDateInfo]() {
543+
if res.exec_err {
544+
return error('Error while checking latest commits for "${res.name}" .')
545+
}
546+
if res.outdated {
547+
outdated << res.name
466548
}
467549
}
468550
return outdated

0 commit comments

Comments
 (0)