Skip to content
Permalink
Browse files

add new 'archive' and --autoarchive function to auto-cleanup pipelines

  • Loading branch information...
ssadedin committed Sep 8, 2019
1 parent 22a6d77 commit fbba5acf9e2b1cbe1f5be8a535293413686cfa2c
@@ -440,6 +440,7 @@ elif [ "$COMMAND" == "cleanup" ] || \
[ "$COMMAND" == "query" ] || \
[ "$COMMAND" == "preserve" ] || \
[ "$COMMAND" == "preallocate" ] || \
[ "$COMMAND" == "archive" ] || \
[ "$COMMAND" == "stats" ] || \
[ "$COMMAND" == "install" ] || \
[ "$COMMAND" == "diagram" ] || \
@@ -35,6 +35,7 @@ import org.apache.commons.cli.Option
import org.codehaus.groovy.runtime.StackTraceUtils;

import bpipe.agent.AgentRunner
import bpipe.cmd.ArchiveCommand
import bpipe.cmd.PreallocateCommand
import bpipe.cmd.Stop;
import bpipe.worx.WorxEventListener;
@@ -183,6 +184,12 @@ class Runner {
exit(0)
}
else
if(mode == "archive") {
log.info("Mode is archive")
new ArchiveCommand(args as List).run(System.out)
exit(0)
}
else
if(mode == "register") {
log.info("Mode is register")
cli = registerCli
@@ -276,6 +283,7 @@ class Runner {
cli.with {
h longOpt:'help', 'usage information'
d longOpt:'dir', 'output directory', args:1
a longOpt: 'autoarchive', 'clean up all internal files after run into given archive', args:1
t longOpt:'test', 'test mode'
f longOpt: 'filename', 'output file name of report', args:1
r longOpt:'report', 'generate an HTML report / documentation for pipeline'
@@ -485,6 +493,10 @@ class Runner {
EventManager.instance.signal(PipelineEvent.SHUTDOWN, "Shutting down process $pid")
exit(1)
}

if(opts.a) {
new ArchiveCommand(["-d", opts.a]).run(System.out)
}
}

static void exit(int code) {
@@ -0,0 +1,127 @@
/*
* Copyright (c) 2016 MCRI, authors
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package bpipe.cmd

import java.io.Writer;

import java.util.List
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream

import javax.swing.colorchooser.CenterLayout

import org.apache.ivy.util.FileUtil

import bpipe.CommandManager;
import bpipe.Config
import bpipe.ExecutorFactory
import bpipe.ExecutorPool
import bpipe.PooledExecutor
import bpipe.Runner
import bpipe.Utils
import groovy.io.FileType
import groovy.transform.CompileStatic;
import groovy.util.logging.Log;

/**
* Clean up all the temporary files and commands generated by the pipeline into an archive zip file
*
* @author Simon Sadedin
*/
@Log
class ArchiveCommand extends BpipeCommand {

public ArchiveCommand(List<String> args) {
super("archive", args);
}

@Override
public void run(Writer out) {
parse('archive <zip file>') {
d 'Remove archived files', longOpt:'delete'
}

String zipPath = opts.arguments()[0]
if(!zipPath)
throw new IllegalArgumentException("Please provide the path to a zip file to archive into")

if(!zipPath.endsWith('.zip')) {
new File(zipPath).mkdirs()
zipPath = computeZipPath(zipPath)
println "MSG: Archiving to computed file path ${zipPath}"
}

if(!zipPath.endsWith('.zip'))
throw new IllegalArgumentException("The archive file name should end with .zip")

List archivedFiles
new File(zipPath).withOutputStream {
archivedFiles = createArchiveZip(new ZipOutputStream(it))
}

out.println "MSG: Archived ${archivedFiles.size()} files"
List<File> dirs = []
if(opts.d) {
for(File f in archivedFiles) {
if(!f.delete()) {
out.println "WARNING: could not delete $f"
}
if(f.parentFile)
dirs << f.parentFile
}

if(opts.d) {
new File('.bpipe').deleteDir()
}
}
}

/**
* Archive the local bpipe commands into the given archive file and return
* the list of archived files
* @param zos
* @return
*/
@CompileStatic
List<File> createArchiveZip(ZipOutputStream zos) {
List<File> toDelete = []
new File('.bpipe').eachFileRecurse(FileType.FILES) { File f ->
zos.putNextEntry(new ZipEntry(f.path))
f.withInputStream { zos << it }
zos.closeEntry()
toDelete << f
}
File commandlog = new File("commandlog.txt")
zos.putNextEntry(new ZipEntry(commandlog.path))
zos.write(commandlog.readBytes())
zos.close()
toDelete << commandlog
}

String computeZipPath(String dir) {
Date now = new Date()
return new File(dir, "bpipe_archive_" + now.format('YMMdd_HHmmss') + '.zip').path
}
}
@@ -38,6 +38,10 @@ abstract class BpipeCommand {
try {
this.run(writer)
}
catch(IllegalArgumentException exArg) {
System.err.println("One or more arguments was incorrect: please check the options and try again")
cli.usage()
}
finally {
writer.flush()
}
@@ -82,6 +86,18 @@ abstract class BpipeCommand {
}
}

OptionAccessor parse(String usage, @DelegatesTo(CliBuilder) Closure builder) {
cli = new CliBuilder(usage:usage)
builder.delegate = cli
builder()
opts = cli.parse(args)
if(!opts) {
System.exit(1)
}
return opts
}


@CompileStatic
OptionAccessor parse() {
opts = cli.parse(args)

0 comments on commit fbba5ac

Please sign in to comment.
You can’t perform that action at this time.