Skip to content

Commit

Permalink
Initial framing of relion_refine command from appion
Browse files Browse the repository at this point in the history
  • Loading branch information
@mcianfrocco
mcianfrocco committed May 11, 2017
1 parent b9d2840 commit 3cdf04f
Showing 1 changed file with 3 additions and 150 deletions.
153 changes: 3 additions & 150 deletions relion/run_relionAWS_fromAppion.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,118 +53,6 @@ def s3_to_ebs(IP,keypair,bucketname,dironebs,rclonepath,keyid,secretid,region,nu
exec_remote_cmd('mv %s/%s tmp.mrcs' %(dironebs,fileonly))
exec_remote_cmd('rm -rf /%s/' %(dironebs))
exec_remote_cmd('mv tmp.mrcs /data/%s' %(fileonly))
#==========================
def s3_to_ebs_movie(IP,keypair,bucketname,dironebs,rclonepath,keyid,secretid,region,numfilesAtATime):
#Copy rclone onto instance
cmd='scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s ubuntu@%s:~/'%(keypair,rclonepath,IP)
subprocess.Popen(cmd,shell=True).wait()

#Write rclone config file
homedir='/home/ubuntu/'
rclonename='ebss3'
if os.path.exists('.rclone.conf'):
os.remove('.rclone.conf')
r1=open('rclone.conf','w')
r1.write('[rclonename]\n')
r1.write('type = s3\n')
r1.write('env_auth = false\n')
r1.write('access_key_id = %s\n' %(keyid))
r1.write('secret_access_key = %s\n' %(secretid))
r1.write('region = %s\n' %(region))
r1.write('endpoint = \n')
r1.write('location_constraint = %s\n' %(region))
r1.write('acl = authenticated-read\n')
r1.write('server_side_encryption = \n')
r1.write('storage_class = STANDARD\n')
r1.close()

cmd='scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s rclone.conf ubuntu@%s:~/.rclone.conf' %(keypair,IP)
subprocess.Popen(cmd,shell=True).wait()

#Copy data down
env.host_string='ubuntu@%s' %(IP)
env.key_filename = '%s' %(keypair)
rcloneexe='rclone'
exec_remote_cmd('%s/%s copy rclonename:%s %s --min-size 500M --quiet --transfers %i' %(homedir,rcloneexe,bucketname.split('s3://')[-1],dironebs,numfilesAtATime))

fileonly=dironebs.split('/')[-1]
if dironebs.split('.')[-1] == 'mrcs' or dironebs.split('.')[-1] == 'spi':
exec_remote_cmd('mv %s/%s tmp.mrcs' %(dironebs,fileonly))
exec_remote_cmd('rm -rf /%s/' %(dironebs))
exec_remote_cmd('mv tmp.mrcs /data/%s' %(fileonly))


#=========================
def rclone_to_s3_mics(micstar,numfiles,region,keyid,secretid,rclonename,bucketname,awspath,project):

if subprocess.Popen('uname',shell=True, stdout=subprocess.PIPE).stdout.read().strip() == 'Linux':
rclonepath='%s/rclone' %(awspath)
if subprocess.Popen('uname',shell=True, stdout=subprocess.PIPE).stdout.read().strip() == 'Darwin':
rclonepath='%s/rclone_mac'%(awspath)

#Write .rclone.conf
homedir=subprocess.Popen('echo $HOME', shell=True, stdout=subprocess.PIPE).stdout.read().split()[0]
if os.path.exists('%s/.rclone.conf' %(homedir)):
os.remove('%s/.rclone.conf' %(homedir))

r1=open('%s/.rclone.conf' %(homedir),'w')
r1.write('[rclonename]\n')
r1.write('type = s3\n')
r1.write('env_auth = false\n')
r1.write('access_key_id = %s\n' %(keyid))
r1.write('secret_access_key = %s\n' %(secretid))
r1.write('region = %s\n' %(region))
r1.write('endpoint = \n')
r1.write('location_constraint = %s\n' %(region))
r1.write('acl = authenticated-read\n')
r1.write('server_side_encryption = \n')
r1.write('storage_class = STANDARD\n')
r1.close()

directoryToTransfer=micstar.split('/')
del directoryToTransfer[-1]
directoryToTransfer='/'.join(directoryToTransfer)

#Check first micro to see if it is in directoryToTransfer
#Get column number first
o1=open(micstar,'r')
for line in o1:
if len(line.split())> 0:
if line.split()[0]=='_rlnMicrographName':
micolnum=line.split()[1].split('#')[-1]
o1.close()
o1=open(micstar,'r')
flag=0
for line in o1:
if len(line.split())> 0:
if os.path.exists(line.split()[int(micolnum)-1]):
if flag == 0:
path=line.split()[int(micolnum)-1].split('/')
del path[-1]
path='/'.join(path)
flag=1
o1.close()
cmd='%s copy %s rclonename:%s --quiet --transfers %i > rclone.log' %(rclonepath,directoryToTransfer,bucketname,math.ceil(numfiles))
subprocess.Popen(cmd,shell=True).wait()
otherbucket=''
if path != directoryToTransfer:

#Get uploadlist
o13=open('uploadlist.txt','w')
for line in open(micstar,'r'):
if len(line.split()) > 0:
if os.path.exists(line.split()[int(micolnum)-1]):
o13.write('%s\n' %(line.split()[int(micolnum)-1].split('/')[-1]))
o13.close()
cmd='%s copy %s/ rclonename:%s-mic --include-from uploadlist.txt --quiet --transfers %i > rclone.log' %(rclonepath,path,bucketname,math.ceil(numfiles))
subprocess.Popen(cmd,shell=True).wait()

otherbucket='%s-mic' %(bucketname)
os.remove('uploadlist.txt')

os.remove('rclone.log')
return 's3://%s' %(bucketname),directoryToTransfer,'s3://%s' %(otherbucket),path

#=========================
def rclone_to_s3_movie(micstar,numfiles,region,keyid,secretid,rclonename,bucketname,awspath,project):

Expand Down Expand Up @@ -3958,43 +3846,8 @@ def relion_autopick_mpi(project):

#Read relion command from appion
in_cmd=sys.argv[1]

sys.exit()

jobtype,numjobsIn=getJobType(infile)

#checkConflicts()

if jobtype == 'None':
print 'Error: unrecognized relion command'
sys.exit()
project=''
#Get project name if exists
if os.path.exists('.aws_relion_project_info'):
project=linecache.getline('.aws_relion_project_info',1).split('=')[-1]

#Align movies
if jobtype == 'relion_run_motioncorr' or jobtype == 'relion_run_motioncorr_mpi':
relion_run_motioncorr(project)

#CTF estimation
if jobtype == 'relion_run_ctffind' or jobtype == 'relion_run_ctffind_mpi':
relion_run_ctffind(project)

#Extract particles
if numjobsIn == 1:
if jobtype == 'relion_preprocess' or jobtype == 'relion_preprocess_mpi':
relion_preprocess_mpi(project)

#Perform 2D or 3D classification / refinement
if jobtype == 'relion_refine' or jobtype == 'relion_refine_mpi':
relion_refine_mpi(project)

#Submit auto picking job to GPUs
if jobtype == 'relion_autopick' or jobtype == 'relion_autopick_mpi':
relion_autopick_mpi(project)

#Movie refinemnt
if numjobsIn==2:
relion_movie_refine(project)

print 'Error: Job submission not recognized. Exiting'
relion_refine_mpi(in_cmd)

0 comments on commit 3cdf04f

Please sign in to comment.