Jj test convert

reliontomo/convert/convert30_tomo.py

@@ -25,7 +25,6 @@
 
 from pwem.emlib.image import ImageHandler
 import pyworkflow.utils as pwutils
-from pyworkflow.object import String
 from relion.convert.convert_base import WriterBase
 from scipion.install.funcs import mkdir
 import numpy as np
@@ -52,6 +51,7 @@
 SHIFTX = 'rlnOriginX'
 SHIFTY = 'rlnOriginY'
 SHIFTZ = 'rlnOriginZ'
+NOT_FOUND = 'NotFound'
 
 RELION_TOMO_LABELS = [TOMO_NAME,
                       COORD_X,
@@ -127,17 +127,17 @@ def writeSetOfSubtomograms(self, subtomoSet, subtomosStar, isPyseg=False):
                            rlnOriginZ]
             if isPyseg:
                 fieldsToAdd = [rlnMicrographName,
-                              rlnCoordinateX,
-                              rlnCoordinateY,
-                              rlnCoordinateZ,
-                              rlnImageName,
-                              rlnCtfImage,
-                              rlnAngleRot,
-                              rlnAngleTilt,
-                              rlnAnglePsi,
-                              rlnOriginX,
-                              rlnOriginY,
-                              rlnOriginZ]
+                               rlnCoordinateX,
+                               rlnCoordinateY,
+                               rlnCoordinateZ,
+                               rlnImageName,
+                               rlnCtfImage,
+                               rlnAngleRot,
+                               rlnAngleTilt,
+                               rlnAnglePsi,
+                               rlnOriginX,
+                               rlnOriginY,
+                               rlnOriginZ]
 
             tomoTable.addRow(*fieldsToAdd)
 
@@ -168,9 +168,10 @@ def _createStarTomoTable(isPyseg):
     @ staticmethod
     def _getCTFFileFromSubtomo(subtomo):
         try:
-            return subtomo.getCoordinate3D()._3dcftMrcFile.get()
+            ctfFile = subtomo.getCoordinate3D()._3dcftMrcFile.get()
+            return ctfFile if ctfFile else NOT_FOUND
         except:
-            return ''
+            return NOT_FOUND
 
     @staticmethod
     def _getTransformInfoFromSubtomo(subtomo, calcInv=True):
@@ -180,11 +181,12 @@ def _getTransformInfoFromSubtomo(subtomo, calcInv=True):
 
         if T:  # Alignment performed before
             M = subtomo.getTransform().getMatrix()
-            shifts = translation_from_matrix(M)
+            # shifts = translation_from_matrix(M)
             if calcInv:
-                shifts = -shifts
+                # shifts = -shifts
                 M = np.linalg.inv(M)
 
             angles = -np.rad2deg(euler_from_matrix(M, axes='szyz'))
+            shifts = -translation_from_matrix(M)
 
         return angles, shifts

reliontomo/protocols/protocol_base_tomo.py

@@ -40,7 +40,7 @@
 from relion.convert import convertBinaryVol, MASK_FILL_ZERO, Table
 from relion import ANGULAR_SAMPLING_LIST
 from reliontomo import Plugin
-from reliontomo.convert import writeSetOfSubtomograms
+from reliontomo.convert import convert30_tomo
 
 
 class ProtRelionBaseTomo(EMProtocol):
@@ -498,9 +498,9 @@ def _defineComputeParams(form):
     def _insertAllSteps(self):
 
         self._initialize()
-        self._insertFunctionStep('convertInputStep')
+        self._insertFunctionStep(self.convertInputStep)
         self._insertRelionStep()
-        self._insertFunctionStep('createOutputStep')
+        self._insertFunctionStep(self.createOutputStep)
 
     def _insertRelionStep(self):
         """ Prepare the command line arguments before calling Relion. """
@@ -526,7 +526,8 @@ def convertInputStep(self):
         subtomoSet = self._getInputParticles()
         subtomosStar = self._getFileName('input_star')
         self.info("Converting set from '%s' into '%s'" % (subtomoSet.getFileName(), subtomosStar))
-        writeSetOfSubtomograms(subtomoSet, subtomosStar)
+        writer = convert30_tomo.Writer()
+        writer.writeSetOfSubtomograms(subtomoSet, subtomosStar)
 
     def runRelionStep(self, params):
         """ Execute the relion steps with the give params. """
@@ -794,7 +795,7 @@ def _setCTFArgs(self, args):
             args['--ctf_multiplied'] = ''
         if self.ctfPhaseFlipped.get():
             # --ctf_phase_flipped (false) : Have the data been CTF phase-flipped?
-            args['--ctf_multiplied'] = ''
+            args['--ctf_phase_flipped'] = ''
         if self.ctfFlipPhases.get():
             # --only_flip_phases (false) : Only perform CTF phase-flipping?
             # (default is full amplitude-correction)

reliontomo/protocols/protocol_classify_3d_tomo.py

@@ -28,7 +28,12 @@
 from os import remove
 from os.path import abspath, exists
 
+from emtable import Table
+
+from pwem import ALIGN_PROJ
 from pwem.protocols import ProtClassify3D, params
+from relion.convert import convert30
+
 from pyworkflow import BETA
 
 from .protocol_base_tomo import ProtRelionBaseTomo
@@ -70,6 +75,9 @@ def _setSamplingArgs(self, args):
 
     # --------------------------- STEPS functions --------------------------------------------
     def createOutputStep(self):
+
+        import time
+        time.sleep(10)
         subtomoSet = self._getInputParticles()
         classes3D = self._createSetOfClassesSubTomograms(subtomoSet)
         self._fillClassesFromIter(classes3D, self._lastIter())
@@ -89,9 +97,9 @@ def createOutputStep(self):
         self._defineOutputs(outputVolumes=volumes)
         self._defineSourceRelation(subtomoSet, volumes)
 
-        if not self.doContinue:
-            self._defineSourceRelation(self.referenceVolume, classes3D)
-            self._defineSourceRelation(self.referenceVolume, volumes)
+        # if not self.doContinue and not self.referenceVolume.hasValue():
+        #     self._defineSourceRelation(self.referenceVolume, classes3D)
+        #     self._defineSourceRelation(self.referenceVolume, volumes)
 
         if self.keepOnlyLastIterFiles:
             self._cleanUndesiredFiles()
@@ -169,9 +177,6 @@ def _loadClassifyInfo(self, iteration):
         with open(modelStar) as fid:
             self.modelTable.readStar(fid, 'model_general')
             self.claasesTable.readStar(fid, 'model_classes')
-        dataStar = self._getFileName('data', iter=iteration)
-        with open(dataStar) as fid:
-            self.dataTable.readStar(fid)
 
         # Model table has only one row, while classes table has the same number of rows as classes found
         self.nClasses = int(self.modelTable._rows[0].rlnNrClasses)
@@ -182,14 +187,24 @@ def _loadClassifyInfo(self, iteration):
     def _fillClassesFromIter(self, clsSet, iteration):
         """ Create the SetOfClasses3D from a given iteration. """
         self._loadClassifyInfo(iteration)
+
+        self.reader = convert30.Reader(alignType=ALIGN_PROJ,
+                                       pixelSize=clsSet.getSamplingRate())
+
+        dataStar = self._getFileName('data', iter=iteration)
+        dataIter = Table.iterRows(fileName =dataStar, key='rlnImageName')
         clsSet.classifyItems(updateItemCallback=self._updateParticle,
                              updateClassCallback=self._updateClass,
-                             itemDataIterator=self.dataTable.__iter__())
+                             itemDataIterator=dataIter,
+                             iterParams={"orderBy": "_filename"})
 
     def _updateParticle(self, item, row):
         item.setClassId(int(row.rlnClassNumber))#rlnGroupNumber))
         item._rlnLogLikeliContribution = params.Float(row.rlnLogLikeliContribution)
         item._rlnMaxValueProbDistribution = params.Float(row.rlnMaxValueProbDistribution)
+        if item.getFileName() != row.rlnImageName:
+            raise Exception("File missmatch: item has %s and row has %s" % (item.getFileName(), row.rlnImageName))
+        self.reader.setParticleTransform(item, row)
 
     def _updateClass(self, item):
         classId = item.getObjId()

reliontomo/protocols/protocol_ctf_3d_estimation.py

@@ -68,6 +68,7 @@ def __init__(self, **args):
         self._doseFromMdoc = None
         self.ctfMRCFileList = []
         self.ctfStarFileList = []
+        self.coordsSampling = []
 
     # --------------------------- DEFINE param functions --------------------------------------------
     def _defineParams(self, form):
@@ -115,12 +116,13 @@ def _defineParams(self, form):
     # --------------------------- INSERT steps functions --------------------------------------------
     def _insertAllSteps(self):
         program = "relion_reconstruct" if self.numberOfMpi == 1 else "relion_reconstruct_mpi"
+        self.coordsSampling = self.inputCoordinates.get().getPrecedents().getSamplingRate()
         # Insert the steps
-        writeDeps = self._insertFunctionStep("writeStarCtf3DStep")
+        writeDeps = self._insertFunctionStep(self.writeStarCtf3DStep)
         for ctfStarFile, ctfMRCFile in zip(self.ctfStarFileList, self.ctfMRCFileList):
-            self._insertFunctionStep("reconstructCtf3DStep",  program, ctfStarFile, ctfMRCFile,
+            self._insertFunctionStep(self.reconstructCtf3DStep,  program, ctfStarFile, ctfMRCFile,
                                      prerequisites=[writeDeps])
-        self._insertFunctionStep('createOutputStep')
+        self._insertFunctionStep(self.createOutputStep)
 
     # --------------------------- STEPS functions --------------------------------------------
     def writeStarCtf3DStep(self):
@@ -141,7 +143,7 @@ def writeStarCtf3DStep(self):
                 self._estimateCTF3DPerSubvolume(ts, setTsInfo)
 
     def reconstructCtf3DStep(self, program, ctfStarFile, ctfMRCFile):
-        param = {"sampling": self.getTSSetFromCTFSeries().getSamplingRate(),
+        param = {"sampling": self.coordsSampling,
                  "ctfStar": abspath(ctfStarFile),
                  "ctf3D": abspath(ctfMRCFile),
                  "boxSize": self.boxSize.get()
@@ -168,6 +170,7 @@ def createOutputStep(self):
                 coord._3dcftMrcFile = String(ctfMrc)
                 out_coords.append(coord)
 
+        out_coords.setBoxSize(self.boxSize.get())
         self._defineOutputs(outputCoordinates=out_coords)
         self._defineTransformRelation(self.inputCoordinates, out_coords)
 
@@ -400,8 +403,8 @@ def _estimateCTF3DPerSubvolume(self, tsExp, setTsInfo):
                 avgDefocus = (ctf.getDefocusU() + ctf.getDefocusV()) / 2
                 tiltAngleDegs = ti.getTiltAngle()
                 tiltAngleRads = np.deg2rad(tiltAngleDegs)
-                xTomo = float(coord.getX(BOTTOM_LEFT_CORNER) - (sizeX / 2)) * setTsInfo[SRATE]
-                zTomo = float(coord.getZ(BOTTOM_LEFT_CORNER) - (sizeZ / 2)) * setTsInfo[SRATE]
+                xTomo = float(coord.getX(BOTTOM_LEFT_CORNER) - (sizeX / 2)) * setTsInfo[SRATE]/self.coordsSampling
+                zTomo = float(coord.getZ(BOTTOM_LEFT_CORNER) - (sizeZ / 2)) * setTsInfo[SRATE]/self.coordsSampling
                 # Calculating the height difference of the particle from the tilt axis
                 xImg = (xTomo * (math.cos(tiltAngleRads))) + (zTomo * (math.sin(tiltAngleRads)))
                 deltaD = xImg * math.sin(tiltAngleRads)

reliontomo/protocols/protocol_reconstruct_subtomo.py

@@ -28,11 +28,12 @@
                                         EnumParam, IntParam, LEVEL_ADVANCED)
 
 from pwem.protocols import ProtReconstruct3D
-from tomo.objects import Tomogram, AverageSubTomogram
+from reliontomo.convert import convert30_tomo
+from tomo.objects import AverageSubTomogram
 from reliontomo import Plugin
-from reliontomo.convert import writeSetOfSubtomograms
 from reliontomo.constants import V30_VALIDATION_MSG
 
+
 class ProtRelionSubTomoReconstruct(ProtReconstruct3D):
     """ This protocol reconstructs a volume using Relion.
 
@@ -93,9 +94,9 @@ def _defineParams(self, form):
     # -------------------------- INSERT steps functions -----------------------
     def _insertAllSteps(self):
         self._createFilenameTemplates()
-        self._insertFunctionStep('convertInputStep')
+        self._insertFunctionStep(self.convertInputStep)
         self._insertReconstructStep()
-        self._insertFunctionStep('createOutputStep')
+        self._insertFunctionStep(self.createOutputStep)
 
     def _getProgram(self, program='relion_reconstruct'):
         """ Get the program name depending on the MPI use or not. """
@@ -136,18 +137,17 @@ def reconstructStep(self, params):
     def _createFilenameTemplates(self):
         """ Centralize how files are called for iterations and references. """
         myDict = {
-            'input_particles': self._getTmpPath(self.inStarName + '.star'),
+            'input_particles': self._getPath(self.inStarName + '.star'),
             'output_volume': self._getExtraPath(self.outTomoName + '.mrc')
             }
         self._updateFilenamesDict(myDict)
 
     def convertInputStep(self):
-        """ Create the input file in STAR format as expected by Relion.
-        """
-        # TODO: If the input particles comes from Relion, just link the file.
+        """ Create the input file in STAR format as expected by Relion."""
         imgSet = self.inputSubtomos.get()
         imgStar = self._getFileName(self.inStarName)
-        writeSetOfSubtomograms(imgSet, imgStar)
+        writer = convert30_tomo.Writer()
+        writer.writeSetOfSubtomograms(imgSet, imgStar)
 
     def createOutputStep(self):
         imgSet = self.inputSubtomos.get()

reliontomo/protocols/protocol_refine_3d_tomo.py

@@ -33,7 +33,7 @@
 from pwem.objects import FSC, Integer
 from pwem.protocols import ProtRefine3D
 from pyworkflow import BETA
-from relion.convert import Table, convert31
+from relion.convert import Table, convert30
 from reliontomo import Plugin
 
 from tomo.objects import AverageSubTomogram
@@ -104,7 +104,7 @@ def createOutputStep(self):
 
         self._defineOutputs(outputVolume=vol)
         self._defineSourceRelation(subtomoSet, vol)
-        self._defineOutputs(outputParticles=outSubtomoSet)
+        self._defineOutputs(outputSetOfSubtomograms=outSubtomoSet)
         self._defineTransformRelation(subtomoSet, outSubtomoSet)
 
         fsc = FSC(objLabel=self.getRunName())
@@ -128,13 +128,13 @@ def _validateNormal(self):
             volumeDim = self.referenceVolume.get().getDim()
 
             if particlesDim is None:
-                errors.append('Can not get dimensions from input particles!!!')
+                errors.append('Can not get dimensions from input particles.')
 
             elif volumeDim is None:
-                errors.append('Can not get dimensions from reference volume!!!')
+                errors.append('Can not get dimensions from reference volume.')
 
             elif particlesDim[0] != volumeDim[0]:
-                errors.append('Volume and particles dimensions must be equal!!!')
+                errors.append('Volume and particles dimensions must be equal.')
 
         return errors
 
@@ -228,7 +228,7 @@ def _fillDataFromIter(self, subtomoClassesSet, iteration):
             item.setAlignmentProj()
         # with open(dataStar) as fid:
         #     self.dataTable.readStar(fid)
-        self.reader = convert31.Reader(alignType=ALIGN_PROJ,
+        self.reader = convert30.Reader(alignType=ALIGN_PROJ,
                                        pixelSize=subtomoClassesSet.getSamplingRate())
 
         mdIter = Table.iterRows(dataStar, key='rlnImageName')