v0.10.3: documentation of new features

CHANGELOG.md

@@ -1,5 +1,54 @@
 #Changes by release
 
+## [0.10.3.dev0] (2023-11-30) - Multifile support for `.star`
+
+* convert multiple `.star` files into a single EMDB-SFF file
+* new class `sfftk.formats.star.RelionMultiStarSegmentation`
+* however, only one mask/particle will be accepted for all, which is handy when displaying the particles with different colours on the same viz.
+
+## [0.10.2.dev1] (2023-11-20) - Remove print warning for OLS
+
+## [0.10.2.dev0] (2023-11-16) - Extra arguments for working with RELION `.star` files
+
+* `--euler-angle-convention` is now user-modifiable
+* `--radians` flag to treat angles as radians instead of degrees
+* adapt to changes in OLS
+* print warning for OLS bug which does not honour `--start` for search
+
+## [0.10.1.dev1] (2023-10-26) - Extension of previous bugfix
+
+## [0.10.1.dev0] (2023-10-26) - Improvements to display of search results
+
+* bugfix: when running `sff notes search '<term>' --as-text`, any multi-line `label`s are concatenated without spaces (FIXED)
+
+## [0.10.0.dev0] (2023-10-24) - Improvements to display of search results
+
+* display search results `--as-text`
+* select search results to save to file using the `--filter-rows` option
+* if concatenating search results to text file then option `--no-header` to only include results
+* new classes to `sfftk.notes.find`: `CSVTable` and `CSVRow`
+* associated unit tests with minor cleanup
+
+## [0.9.0.dev2] (2023-10-03) - Minor corrections and cleanup
+
+## [0.9.0.dev1] (2023-09-27) - Using shape_primitive_list for subtomogram averages
+
+* changed convert argument `--particles` to `--subtomogram-average`
+* required sfftk-rw>=0.8.1 to work
+
+## [0.9.0.dev0] (2023-09-20) - Handling RELION STAR files
+
+* basic working API
+* starreader documentation
+* skeleton implementation of star converter
+* test files
+* main entry point for `starsplit` and `starcrop`
+* display progres bar while reading RELION `.star` file
+* `prep starsplit`: split a .star file by image name; options to handle heterogeneity
+* `prep starcrop`: crop a `.star` file to a certain number of rows
+* unit tests for all the above
+* `--image-name-field` to specify field with image/tomogram
+
 ## [0.8.5] (2023-06-22) - Now `sff view` on STL files displays the bounding box; fixed failing tests
 
 ## [0.8.4] (2023-02-08) - Added `GlobalSimpleNote` to annotate a segmentation

docs/_build/html/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 79fec64e8c91361769441ad6d2c49be6
+config: 758c7c361488de53d9ae381ed531de1a
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/_build/html/_sources/converting.rst.txt

@@ -25,56 +25,78 @@ displays all conversion options.
 
 .. code:: bash
 
-    sff convert
-    usage: sff convert [-h] [-D DETAILS] [-R PRIMARY_DESCRIPTOR] [-v] [-x]
-                       [--json-indent JSON_INDENT] [--json-sort]
-                       [-o OUTPUT | -f FORMAT] [-p CONFIG_PATH] [-b] [-a] [-m]
-                       [--subtype-index SUBTYPE_INDEX] [--image IMAGE]
-                       [from_file [from_file ...]]
-
-    Perform conversions to EMDB-SFF
-
-    positional arguments:
-      from_file             file to convert from
-
-    optional arguments:
-      -h, --help            show this help message and exit
-      -D DETAILS, --details DETAILS
-                            populates <details>...</details> in the XML file
-      -R PRIMARY_DESCRIPTOR, --primary-descriptor PRIMARY_DESCRIPTOR
-                            populates the
-                            <primary_descriptor>...</primary_descriptor> to this
-                            value [valid values: three_d_volume, mesh_list,
-                            shape_primitive_list]
-      -v, --verbose         verbose output
-      -x, --exclude-geometry
-                            do not include the geometry in the conversion;
-                            geometry is included by default [default: False]
-      --json-indent JSON_INDENT
-                            size in spaces of the JSON indent [default: 2]
-      --json-sort           output JSON sorted lexicographically [default: False]
-      -o OUTPUT, --output OUTPUT
-                            file to convert to; the extension (.sff, .hff, .json)
-                            determines the output format [default: None]
-      -f FORMAT, --format FORMAT
-                            output file format; valid options are: sff (XML), hff
-                            (HDF5), json (JSON) [default: sff]
-      -p CONFIG_PATH, --config-path CONFIG_PATH
-                            path to configs file
-      -b, --shipped-configs
-                            use shipped configs only if config path and user
-                            configs fail [default: False]
-      -a, --all-levels      for segments structured hierarchically (e.g. Segger
-                            from UCSF Chimera and Chimera X) convert all segment
-                            leves in the hierarchy [default: False]
-      -m, --multi-file      enables convert to treat multiple files as individual
-                            segments of a single segmentation; only works for the
-                            following filetypes: stl, map, mrc, rec [default:
-                            False]
-      --subtype-index SUBTYPE_INDEX
-                            some file extensions are used by multiple file types
-      --image IMAGE         specify the segmented EMDB MAP/MRC file from which to
-                            determine the correct image-to-physical transform
+	sff convert
+	usage: sff convert [-h] [-D DETAILS] [-R PRIMARY_DESCRIPTOR] [-v] [-x]
+					   [--json-indent JSON_INDENT] [--json-sort]
+					   [-o OUTPUT | -f FORMAT] [-p CONFIG_PATH] [-b] [-a] [-m]
+					   [--subtype-index SUBTYPE_INDEX] [--image IMAGE]
+					   [--label-tree LABEL_TREE]
+					   [--subtomogram-average SUBTOMOGRAM_AVERAGE]
+					   [--image-name-field IMAGE_NAME_FIELD]
+					   [--euler-angle-convention {zyz,zxz,xyx,xzx,yxy,yzy}]
+					   [--radians]
+					   [from_file [from_file ...]]
+
+	Perform conversions to EMDB-SFF
+
+	positional arguments:
+	  from_file             file to convert from
+
+	optional arguments:
+	  -h, --help            show this help message and exit
+	  -D DETAILS, --details DETAILS
+							populates <details>...</details> in the XML file
+	  -R PRIMARY_DESCRIPTOR, --primary-descriptor PRIMARY_DESCRIPTOR
+							populates the
+							<primary_descriptor>...</primary_descriptor> to this
+							value [valid values: three_d_volume, mesh_list,
+							shape_primitive_list]
+	  -v, --verbose         verbose output
+	  -x, --exclude-geometry
+							do not include the geometry in the conversion;
+							geometry is included by default [default: False]
+	  --json-indent JSON_INDENT
+							size in spaces of the JSON indent [default: 2]
+	  --json-sort           output JSON sorted lexicographically [default: False]
+	  -o OUTPUT, --output OUTPUT
+							file to convert to; the extension (.sff, .hff, .json)
+							determines the output format [default: None]
+	  -f FORMAT, --format FORMAT
+							output file format; valid options are: sff (XML), hff
+							(HDF5), json (JSON) [default: sff]
+	  -p CONFIG_PATH, --config-path CONFIG_PATH
+							path to configs file
+	  -b, --shipped-configs
+							use shipped configs only if config path and user
+							configs fail [default: False]
+	  -a, --all-levels      for segments structured hierarchically (e.g. Segger
+							from UCSF Chimera and Chimera X) convert all segment
+							leves in the hierarchy [default: False]
+	  -m, --multi-file      enables convert to treat multiple files as individual
+							segments of a single segmentation; only works for the
+							following filetypes: stl, map, mrc, rec, star
+							[default: False]
+	  --subtype-index SUBTYPE_INDEX
+							some file extensions are used by multiple file types
+	  --image IMAGE         specify the segmented EMDB MAP/MRC file from which to
+							determine the correct image-to-physical transform
+	  --label-tree LABEL_TREE
+							a JSON file produced by running 'sff prep mergemask'
+							which captures: 1) the mask labels (key:
+							'mask_to_label') and 2) the hierarchical relationship
+							between labels (key: 'label_tree')
+	  --subtomogram-average SUBTOMOGRAM_AVERAGE
+							the result of subtomogram averaging or a particle mask
+							for visualisation in CCP4 format (.mrc, .map, .rec)
+	  --image-name-field IMAGE_NAME_FIELD
+							the field in the star file that contains the image
+							name [default: '_rlnImageName']
+	  --euler-angle-convention {zyz,zxz,xyx,xzx,yxy,yzy}
+							the Euler angle convention used in the subtomogram
+							averaging [default: 'zyz' - case insensitive]
+	  --radians             use radians instead of degrees for Euler angles
+							[default: False i.e. use degrees]
+
 
 
 Quick Start
@@ -424,6 +446,50 @@ the user should specify the output file
 
     sff convert -m file1.map file2.map file3.map --output file.sff
 
+Converting Subtomogram Averages
+====================================
+
+Subtomogram averages are the result of subtomogram averaging and are typically stored in CCP4 format
+(``.mrc``, ``.map``, ``.rec``). ``sfftk`` currently only works with RELION subtomogram averages in ``.star`` format.
+Conversion of subtomogram averages is similar to that of other file formats in that
+the user specifies the file to convert and the output file. Optionally, the user may provide the image file from which
+the subtomogram average was derived using the ``--image`` option to accurately determine the image-to-physical transform.
+
+.. code-block:: bash
+
+	sff convert --image file.map file.star
+
+Specifying the Particle File
+----------------------------
+
+It is also possible to specify the particle file from which the subtomogram average was derived using the ``--subtomogram-average`` option. This is useful when the subtomogram average is a particle mask for visualisation. However, given that the particle will be of higher resolution that the original tonmogram, the user may need to downsample the particle for visualisation purposes.
+
+.. code-block:: bash
+
+	sff convert --subtomogram-average particle.mrc --image image.mrc file.star
+
+A tool that can be used to prepare artificial particles is available from https://github.com/emdb-empiar/masks.git.
+
+Multiple Subtomogram Averages
+-----------------------------
+
+As with multifile segmentations, it is possible to convert multiple subtomogram averages into a single EMDB-SFF file. The user should use the ``-m/--multi-file`` option to specify the subtomogram averages. However, only one ``--subtomogram-average`` option should be used to specify the particle file.
+
+.. code-block:: bash
+
+	sff convert -m file1.star file2.star file3.star --output file.sff
+
+Specifying Euler Angle Convention
+---------------------------------
+
+The Euler angle convention used in the subtomogram averaging can be specified using the ``--euler-angle-convention`` option. The default is ``zyz``. The case is not important (``zyz``=``ZYZ``).
+
+
+Specifying Angle Type
+---------------------
+
+The user can specify whether the Euler angles are in degrees or radians using the ``--radians`` option. The default is degrees.
+
 Specifying Configurations To Use
 =================================
 

docs/_build/html/_sources/formats.rst.txt

@@ -49,3 +49,11 @@ Amira HyperSurface format
 .. automodule:: sfftk.formats.surf
     :members: AmiraHyperSurfaceSegmentation, AmiraHyperSurfaceHeader, AmiraHyperSurfaceSegment, AmiraHyperSurfaceAnnotation, AmiraHyperSurfaceMesh
     :show-inheritance:
+
+
+RELION STAR format
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: sfftk.formats.star
+    :members: RelionStarHeader, RelionStarSegment, RelionStarSegmentation, RelionMultiStarSegmentation
+    :show-inheritance:

docs/_build/html/_sources/misc.rst.txt

@@ -647,6 +647,105 @@ Your surface now has fewer polygons with little volume distortion.
 You can play with the parameters in the *Properties* dialogue to modify how the filters work.
 
 
+Cropping STAR Files
+----------------------------
+
+In certain situations, it may be necessary to crop a STAR file to a smaller subset of particles.
+This is particularly useful when
+running tests on a large dataset and only a small subset is required. The ``sff prep starcrop`` utility is used to
+achieve this.
+
+.. code:: bash
+
+	sff prep starcrop
+	usage: sff prep starcrop [-h] [-p CONFIG_PATH] [-b] [-v] [-o OUTPUT]
+							 [--infix INFIX] [--rows ROWS]
+							 [--image-name-field IMAGE_NAME_FIELD]
+							 star_file
+
+	Truncate a composite star file to the specified number of rows (default: 100)
+
+	positional arguments:
+	  star_file             the composite star file
+
+	optional arguments:
+	  -h, --help            show this help message and exit
+	  -p CONFIG_PATH, --config-path CONFIG_PATH
+							path to configs file
+	  -b, --shipped-configs
+							use shipped configs only if config path and user
+							configs fail [default: False]
+	  -v, --verbose         verbose output
+	  -o OUTPUT, --output OUTPUT
+							output file name [default: <infile>_cropped.star]
+	  --infix INFIX         infix to be added to filenames e.g. file.star ->
+							file_<infix>.star [default: 'cropped']
+	  --rows ROWS           the number of rows to keep [default: 100]
+	  --image-name-field IMAGE_NAME_FIELD
+							the field in the star file that contains the image
+							name [default: '_rlnImageName']
+
+
+Splitting STAR Files
+----------------------------
+
+Typically, STAR files may be combined from multiple sources and it may be necessary to split them into their constituent parts.
+This occurs when multiple tomograms are referenced to create a high resolution subtomogram average. The ``sff prep starsplit`` utility is used to achieve this.
+
+.. code-block:: bash
+
+	usage: sff prep starsplit [-h] [-p CONFIG_PATH] [-b] [-v]
+							  [--output-prefix OUTPUT_PREFIX]
+							  [--image-path IMAGE_PATH]
+							  [--image-extension IMAGE_EXTENSION]
+							  [--image-name-prefix IMAGE_NAME_PREFIX]
+							  [--image-name-field IMAGE_NAME_FIELD]
+							  star_file
+
+	Split a composite star file into individual star files distinguished by the
+	<rlnImageName> key
+
+	positional arguments:
+	  star_file             the composite star file
+
+	optional arguments:
+	  -h, --help            show this help message and exit
+	  -p CONFIG_PATH, --config-path CONFIG_PATH
+							path to configs file
+	  -b, --shipped-configs
+							use shipped configs only if config path and user
+							configs fail [default: False]
+	  -v, --verbose         verbose output
+	  --output-prefix OUTPUT_PREFIX
+							a prefix to use for the output files; the output files
+							are named <prefix>_<rlnImageName>.star [default:
+							'<composite-name>_']
+	  --image-path IMAGE_PATH
+							the correct local path to the tomogram files [default:
+							'']
+	  --image-extension IMAGE_EXTENSION
+							the file extension to use for the tomogram files
+							[default: 'mrc']
+	  --image-name-prefix IMAGE_NAME_PREFIX
+							in many star files, the <rlnImageName> values will be
+							a local path; the actual image name (a .mrc file) may
+							contain additional characters that makes it difficult
+							to categorise the tomograms e.g.
+							'path/my_tomogram1_001.mrc',
+							'path/my_tomogram1_002.mrc',
+							'path/my_tomogram2_001.mrc'. In this example, we have
+							two tomograms ('my_tomogram1' and 'my_tomogram2') but
+							the additional characters ('_001', '_002') make it
+							difficult to categorise the tomograms. This option
+							allows you to specify a prefix to remove from the
+							<rlnImageName> values. You can also use a REGEX in
+							quotes e.g. 'my_tomogram\d'. [default: '']
+	  --image-name-field IMAGE_NAME_FIELD
+							the field in the star file that contains the image
+							name [default: '_rlnImageName']
+
+
+
 Setting Configurations
 =======================
 

docs/_build/html/_sources/readers.rst.txt

@@ -46,7 +46,7 @@ STAR reader
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 .. automodule:: sfftk.readers.starreader
-    :members:
+    :members: print_progress, StarReader, RelionStarReader, StarTable, StarTableRow
     :show-inheritance:
 
 

docs/_build/html/_sources/toolkit.rst.txt

@@ -56,6 +56,8 @@ extensions):
 
 -  Segger (.seg)
 
+-  STAR (.star) for subtomogram averages produced using RELION
+
 -  Stereolithography (.stl)
 
 -  Amira HyperSurface (.surf)

docs/_build/html/_static/documentation_options.js

@@ -1,6 +1,6 @@
 var DOCUMENTATION_OPTIONS = {
     URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
-    VERSION: 'v0.9.0.dev0',
+    VERSION: 'v0.10.3.dev0',
     LANGUAGE: 'en-gb',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',