From 4f4af53cb695fbe465fc31ca5737b71e7cb2a813 Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 17 Dec 2021 07:34:50 -0800 Subject: [PATCH 1/3] deprecations update links from readthedocs --- CHANGELOG.rst | 15 ++- README.md | 14 +-- docs/_config.yml | 2 +- docs/_toc.yml | 1 + docs/conf.py | 4 +- docs/metadata.md | 104 +---------------- docs/previous_versions.md | 160 ++++++++++++++++++++++++++ docs/python_api.md | 2 +- docs/tutorial.md | 12 +- docs/vignette_coalescent_diversity.md | 2 +- docs/vignette_continuing.md | 2 +- docs/vignette_parallel_phylo.md | 4 +- pyslim/slim_tree_sequence.py | 62 ++++++---- tests/test_annotation.py | 2 +- tests/test_metadata.py | 3 +- tests/test_tree_sequence.py | 30 +++-- 16 files changed, 259 insertions(+), 160 deletions(-) create mode 100644 docs/previous_versions.md diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8105a987..7465a48b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,13 +4,26 @@ **Breaking changes**: -- +- `pyslim.recapitate` is updated to use new demography features in msprime 1.0, + and differs from `SlimTreeSequence.recapitate()` (now deprecated). The + argument `Ne` is removed; use `ancestral_Ne`. + +- `reference_sequence` is now a tskit attribute, no longer managed by pyslim. + It is no longer mutable on tree sequences (only TableCollections), and + previous calls to `ts.reference_sequence` to get the actual sequence + should be replaced by `ts.reference_sequence.data`. + +- Old-style "legacy" metadata (previously deprecated) has been removed. + See `the documentation `_ + for instructions on migrating your code. + **New features**: - Added `pyslim.population_size( )` to compute an array giving numbers of individuals across a grid of space and time bins. ({user}giliapatterson) + ******************** [0.600] - 2021-02-24 ******************** diff --git a/README.md b/README.md index 5a429abe..6e1a2929 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # pySLiM `pyslim` is a python module to allow reading and writing of SLiM-produced tree sequences -as a thin interface to [tskit](https://tskit.readthedocs.io/en/stable). -Please see [our documentation](https://pyslim.readthedocs.io/en/stable/) for more information. -(That link is to documentation for the [last release](https://pyslim.readthedocs.io/en/stable/) release; -instead, [the latest](https://pyslim.readthedocs.io/en/latest/) documentation +as a thin interface to [tskit](https://tskit.dev/tskit). +Please see [our documentation](https://tskit.dev/pyslim/docs/stable/) for more information. +(That link is to documentation for the last release release; +instead, [the latest](https://tskit.dev/pyslim/docs/latest) documentation may have additional examples, but may also describe features you need to install from github to get.) ## Installation @@ -13,8 +13,4 @@ To install `pyslim`, do ``` pip install pyslim ``` -or read the documentation for how to [install from source](https://pyslim.readthedocs.io/en/stable/development.html#sec-development). - - - Documentation Status - +or read the documentation for how to [install from source](https://tskit.dev/pyslim/stable/development.html#sec-development). diff --git a/docs/_config.yml b/docs/_config.yml index eab0aa8d..675b3dbd 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -42,7 +42,7 @@ sphinx: tskit: ["https://tskit.dev/tskit/docs/stable", null] msprime: ["https://tskit.dev/msprime/docs/stable", null] tutorials: ["https://tskit.dev/tutorials/", null] - stdpopsim: ["https://stdpopsim.readthedocs.io/en/stable", null] + stdpopsim: ["https://popsim-consortium.github.io/stdpopsim-docs/stable", null] myst_enable_extensions: - colon_fence - deflist diff --git a/docs/_toc.yml b/docs/_toc.yml index cb71c078..e772cba1 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -11,6 +11,7 @@ - file: vignette_coalescent_diversity - file: vignette_parallel_phylo - file: metadata + - file: previous_versions - part: pyslim reference chapters: diff --git a/docs/conf.py b/docs/conf.py index 6c37c645..7341218e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -195,6 +195,6 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), - 'tskit': ('https://tskit.readthedocs.io/en/latest', None), - 'msprime': ('https://msprime.readthedocs.io/en/latest', None) + 'tskit': ('https://tskit.dev/tskit/docs/stable', None), + 'msprime': ('https://tskit.dev/tskit/docs/stable', None) } diff --git a/docs/metadata.md b/docs/metadata.md index 92535243..7494e8b3 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -243,12 +243,13 @@ mod_ts.dump("modified_ts.trees") ### Metadata entries -SLiM records additional information in the metadata columns of Population, Individual, Node, and Mutation tables, +SLiM records additional information in the metadata columns of Individual, Node, and Mutation tables, in a binary format using the python ``struct`` module. See {ref}`tskit's metadata documentation ` for details on how this works. Nothing besides this binary information can be stored in the metadata of these tables if the tree sequence is to be used by SLiM, and so when ``pyslim`` annotates an existing tree sequence, anything in those columns is overwritten. +Population metadata is stored as JSON, however, which is more flexible. For more detailed documentation on the contents and format of the metadata, see the SLiM manual. Of particular note is that *nodes* and *populations* may have empty metadata. @@ -256,104 +257,3 @@ SLiM will not use the metadata of nodes that are not associated with alive indiv so this can safely be omitted (and makes recapitation easier). And, populations not used by SLiM will have empty metadata. All remaining metadata are required (besides edges and sites, whose metadata is not used at all). - - -(sec_legacy_metadata)= - -## Legacy metadata - -In previous versions of pyslim, -SLiM-specific metadata was provided as customized objects: -for instance, for a node ``n`` provided by a ``SlimTreeSequence``, -we'd have ``n.metadata`` as a ``NodeMetadata`` object, -with attributes ``n.metadata.slim_id`` and ``n.metadata.is_null`` and ``n.metadata.genome_type``. -However, with tskit 0.3, -the capacity to deal with structured metadata -was implemented in {ref}`tskit itself `, -and so pyslim shifted to using the tskit-native metadata tools. -As a result, parsed metadata is provided as a dictionary instead of an object, -so that now ``n.metadata`` would be a dict, -with entries ``n.metadata["slim_id"]`` and ``n.metadata["is_null"]`` and ``n.metadata["genome_type"]``. -Annotation should be done with tskit methods (e.g., ``packset_metadata``). - -.. note:: - - Until pyslim version 0.600, the old-style metadata was still available, - but this functionality has been removed. - -Here are more detailed notes on how to migrate a script from the legacy -metadata handling. If you run into issues, please ask (open a discussion on github). - -**1.** Use top-level metadata instead of ``slim_provenance``: -previously, information about the model type and the time counter (generation) -in SLiM was provided in the Provenances table, made available through -the ``ts.slim_provenance`` object. This is still available but deprecated, -and should be obtained from the *top-level* metadata object, ``ts.metadata["SLiM"]``. -So, in your scripts ``ts.slim_provenance.model_type`` should be replaced with -``ts.metadata["SLiM"]["model_type"]``, -and (although it's not deprecated), probably ``ts.slim_generation`` should -probably be replaced with -``ts.metadata["SLiM"]["generation"]``. - -**2.** Switch metadata objects to dicts: -if ``md`` is the ``metadata`` property of a population, individual, or node, -this means replacing ``md.X`` with ``md["X"]``. -The ``migration_records`` property of population metadata is similarly -a list of dicts rather than a list of objects, so instead of -``ts.population(1).metadata.migration_records[0].source_subpop`` -we would write -``ts.population(1).metadata["migration_records"][0]["source_subpop"]``. - -Mutations were previously a bit different - if ``mut`` is a mutation -(e.g., ``mut = ts.mutation(0)``) -then ``mut.metadata`` was previously a list of MutationMetadata objects. -Now, ``mut.metadata`` is a dict, with a single entry: -``mut.metadata["mutation_list"]`` is a list of dicts, each containing the information -that was previously in the MutationMetadata objects. -So, for instance, instead of ``mut.metadata[0].selection_coeff`` -we would write ``mut.metadata["mutation_list"][0]["selection_coeff"]``. - -**3.** The ``decode_X`` and ``encode_X`` methods are now deprecated, -as this is handled by tskit itself. -For instance, ``encode_node`` would take a NodeMetadata object -and produce the raw bytes necessary to encode it in a Node table, -and ``decode_node`` would do the inverse operation. -This is now handled by the relevant MetadataSchema object: -for nodes one can obtain this as ``nms = ts.tables.nodes.metadata_schema``, -which has the methods ``nms.validate_and_encode_row`` and ``nms.decode_row``. -Decoding is for the most part not necessary, -since the metadata is automatically decoded, -but ``pyslim.decode_node(raw_md)`` could be replaced by ``nms.decode_row(raw_md)``. -Encoding is necessary to modify tables, -and ``pyslim.encode_node(md)`` can be replaced by ``nms.validate_and_encode_row(md)`` -(where furthermore ``md`` should now be a dict rather than a NodeMetadata object). - -**4.** The ``annotate_X_metadata`` methods are deprecated, -as again tskit has tools to do this. -These methods would set the metadata column of a table - -for instance, if ``metadata`` is a list of NodeMetadata objects, then -``annotate_node_metadata(tables, metadata)`` would modify ``tables.nodes`` in place -to contain the (encoded) metadata in the list ``metadata``. -Now, this could be done as follows (where now ``metadata`` is a list of metadata dicts): - -```{code-cell} -metadata = [ {'slim_id': k, 'is_null': False, 'genome_type': 0} - for k in range(tables.nodes.num_rows) ] -nms = tables.nodes.metadata_schema -tables.nodes.packset_metadata( - [nms.validate_and_encode_row(r) for r in metadata]) -``` - -If speed is an issue, then ``encode_row`` can be substituted for ``validate_and_encode_row``, -but at the risk of missing errors in metadata. - -**5.** the ``extract_X_metadata`` methods are not necessary, -since the metadata in the tables of a TableCollection are automatically decoded. -For instance, ``[ind.metadata["sex"] for ind in tables.individuals]`` will obtain -a list of sexes of the individuals in the IndividualTable. - -:::{warning} - It is our intention to remain backwards-compatible for a time. - However, the legacy code will disappear at some point in the future, - so please migrate over scripts you intend to rely on. -::: diff --git a/docs/previous_versions.md b/docs/previous_versions.md new file mode 100644 index 00000000..bb155a23 --- /dev/null +++ b/docs/previous_versions.md @@ -0,0 +1,160 @@ +--- +jupytext: + text_representation: + extension: .md + format_name: myst + format_version: 0.12 + jupytext_version: 1.9.1 +kernelspec: + display_name: Python 3 + language: python + name: python3 +--- + +```{code-cell} +:tags: [remove-cell] +import pyslim, tskit, msprime + +ts = tskit.load("example_sim.trees") +tables = ts.tables +``` + + +(sec_previous_versions)= + + +# Migrating from previous versions of pyslim + +A number of features that were first introduced in pyslim have been made part of core +tskit functionality. For instance, reference sequence support was provided (although +loosely) inpyslim to support SLiM's nucleotide models, but is now part of a standard +tskit {class}`tskit.TreeSequence`. Similarly, metadata processing in tskit made +code to do this within pyslim obsolete; this "legacy metadata" code has been removed +and instructions for how to migrate your code are {ref}`below `. + +In fact, we are now at the (very good) place where we don't really need +the {class}`pyslim.SlimTreeSequence` class any longer, +and it will soon be deprecated. +So, pyslim is migrating to be purely functional: instead of providing the SlimTreeSequence +class with specialized methods, all methods will be functions of TreeSequences, +that take in a tree sequence and return something +(a modified tree sequence or some summary of it). +Backwards compatibility will be maintained for some time, but we request that you +switch over sooner, as your code will be cleaner and faster. + +To migrate, you should: + + +1. Replace `ts.slim_generation` with `ts.metadata['SLiM']['generation']`, + and `ts.model_type` with `ts.metadata['SLiM']['model_type']`. +2. Replace `ts.reference_sequence` with `ts.reference_sequence.data`. +3. Replace calls to `ts.recapitate(...)` with `pyslim.recapitate(ts, ...)`, + and similarly with other SlimTreeSequence methods. + +If you encounter difficulties, please post an +[issue](https://github.com/tskit-dev/pyslim/issues) +or [discussion](https://github.com/tskit-dev/pyslim/discussions) on github. + + +(sec_legacy_metadata)= + +## Legacy metadata + +In previous versions of pyslim, +SLiM-specific metadata was provided as customized objects: +for instance, for a node ``n`` provided by a ``SlimTreeSequence``, +we'd have ``n.metadata`` as a ``NodeMetadata`` object, +with attributes ``n.metadata.slim_id`` and ``n.metadata.is_null`` and ``n.metadata.genome_type``. +However, with tskit 0.3, +the capacity to deal with structured metadata +was implemented in {ref}`tskit itself `, +and so pyslim shifted to using the tskit-native metadata tools. +As a result, parsed metadata is provided as a dictionary instead of an object, +so that now ``n.metadata`` would be a dict, +with entries ``n.metadata["slim_id"]`` and ``n.metadata["is_null"]`` and ``n.metadata["genome_type"]``. +Annotation should be done with tskit methods (e.g., ``packset_metadata``). + +.. note:: + + Until pyslim version 0.600, the old-style metadata was still available, + but this functionality has been removed. + +Here are more detailed notes on how to migrate a script from the legacy +metadata handling. If you run into issues, please ask (open a discussion on github). + +**1.** Use top-level metadata instead of ``slim_provenance``: +previously, information about the model type and the time counter (generation) +in SLiM was provided in the Provenances table, made available through +the ``ts.slim_provenance`` object. This is still available but deprecated, +and should be obtained from the *top-level* metadata object, ``ts.metadata["SLiM"]``. +So, in your scripts ``ts.slim_provenance.model_type`` should be replaced with +``ts.metadata["SLiM"]["model_type"]``, +and (although it's not deprecated), probably ``ts.slim_generation`` should +probably be replaced with +``ts.metadata["SLiM"]["generation"]``. + +**2.** Switch metadata objects to dicts: +if ``md`` is the ``metadata`` property of a population, individual, or node, +this means replacing ``md.X`` with ``md["X"]``. +The ``migration_records`` property of population metadata is similarly +a list of dicts rather than a list of objects, so instead of +``ts.population(1).metadata.migration_records[0].source_subpop`` +we would write +``ts.population(1).metadata["migration_records"][0]["source_subpop"]``. + +Mutations were previously a bit different - if ``mut`` is a mutation +(e.g., ``mut = ts.mutation(0)``) +then ``mut.metadata`` was previously a list of MutationMetadata objects. +Now, ``mut.metadata`` is a dict, with a single entry: +``mut.metadata["mutation_list"]`` is a list of dicts, each containing the information +that was previously in the MutationMetadata objects. +So, for instance, instead of ``mut.metadata[0].selection_coeff`` +we would write ``mut.metadata["mutation_list"][0]["selection_coeff"]``. + +**3.** The ``decode_X`` and ``encode_X`` methods are now deprecated, +as this is handled by tskit itself. +For instance, ``encode_node`` would take a NodeMetadata object +and produce the raw bytes necessary to encode it in a Node table, +and ``decode_node`` would do the inverse operation. +This is now handled by the relevant MetadataSchema object: +for nodes one can obtain this as ``nms = ts.tables.nodes.metadata_schema``, +which has the methods ``nms.validate_and_encode_row`` and ``nms.decode_row``. +Decoding is for the most part not necessary, +since the metadata is automatically decoded, +but ``pyslim.decode_node(raw_md)`` could be replaced by ``nms.decode_row(raw_md)``. +Encoding is necessary to modify tables, +and ``pyslim.encode_node(md)`` can be replaced by ``nms.validate_and_encode_row(md)`` +(where furthermore ``md`` should now be a dict rather than a NodeMetadata object). + +**4.** The ``annotate_X_metadata`` methods are deprecated, +as again tskit has tools to do this. +These methods would set the metadata column of a table - +for instance, if ``metadata`` is a list of NodeMetadata objects, then +``annotate_node_metadata(tables, metadata)`` would modify ``tables.nodes`` in place +to contain the (encoded) metadata in the list ``metadata``. +Now, this could be done as follows (where now ``metadata`` is a list of metadata dicts): + +```{code-cell} +metadata = [ {'slim_id': k, 'is_null': False, 'genome_type': 0} + for k in range(tables.nodes.num_rows) ] +nms = tables.nodes.metadata_schema +tables.nodes.packset_metadata( + [nms.validate_and_encode_row(r) for r in metadata] +) +``` + +If speed is an issue, then ``encode_row`` can be substituted for ``validate_and_encode_row``, +but at the risk of missing errors in metadata. + +**5.** the ``extract_X_metadata`` methods are not necessary, +since the metadata in the tables of a TableCollection are automatically decoded. +For instance, ``[ind.metadata["sex"] for ind in tables.individuals]`` will obtain +a list of sexes of the individuals in the IndividualTable. + +:::{warning} + It is our intention to remain backwards-compatible for a time. + However, the legacy code will disappear at some point in the future, + so please migrate over scripts you intend to rely on. +::: +======= +>>>>>>> 483184a (deprecation start) diff --git a/docs/python_api.md b/docs/python_api.md index 1e7713a3..123273d4 100644 --- a/docs/python_api.md +++ b/docs/python_api.md @@ -51,7 +51,7 @@ available in pyslim. ## Summarizing tree sequences -Additionally, ``pyslim`` contains the following summary methods: +Additionally, ``pyslim`` contains the following methods: ```{eval-rst} .. autosummary:: diff --git a/docs/tutorial.md b/docs/tutorial.md index 5b4faeae..054cf17d 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -566,9 +566,10 @@ demography.add_migration_rate_change( time=orig_ts.metadata['SLiM']['generation'], rate=0.1, source="p2", dest="p1", ) -rts = pyslim.recapitate(orig_ts, demography=demography, - recombination_rate=1e-8, - random_seed=4 +rts = pyslim.recapitate( + orig_ts, demography=demography, + recombination_rate=1e-8, + random_seed=4 ) ts = pyslim.SlimTreeSequence( msprime.sim_mutations( @@ -685,8 +686,9 @@ and {attr}`.SlimTreeSequence.individual_populations` as follows: ```{code-cell} alive = ts.individuals_alive_at(0) adults = alive[ts.individual_ages[alive] > 2] -pops = [np.where( - ts.individual_populations[adults] == k)[0] for k in [1, 2]] +pops = [ + np.where(ts.individual_populations[adults] == k)[0] for k in [1, 2] +] sample_inds = [np.random.choice(pop, 10, replace=False) for pop in pops] sample_nodes = [] for samp in sample_inds: diff --git a/docs/vignette_coalescent_diversity.md b/docs/vignette_coalescent_diversity.md index 1f654f26..000c9772 100644 --- a/docs/vignette_coalescent_diversity.md +++ b/docs/vignette_coalescent_diversity.md @@ -519,7 +519,7 @@ To do this, we'll compute two standard measures of genetic diversity in windows along the genome: nucleotide diverstiy (also called "Tajima's {math}`\pi`" or "mean density of pairwise differences"), and Tajima's {math}`D` (with no known aliases). -This is easy to do thanks to the (statistics methods in tskit)[https://tskit.readthedocs.io/en/latest/stats.html]. +This is easy to do thanks to the (statistics methods in tskit)[https://tskit.dev/tskit/docs/stable/stats.html]. ```{code-cell} windows = np.linspace(0, mts.sequence_length, 21) diff --git a/docs/vignette_continuing.md b/docs/vignette_continuing.md index 64e7c04c..05c326d5 100644 --- a/docs/vignette_continuing.md +++ b/docs/vignette_continuing.md @@ -78,7 +78,7 @@ We'll add SLiM mutations with "mutation type" 0 so first we check that all the existing mutations are of a different type. ```{code-cell} -rts = ts.recapitate(Ne=1000, recombination_rate=1e-8, random_seed=6) +rts = pyslim.recapitate(ts, ancestral_Ne=1000, recombination_rate=1e-8, random_seed=6) # check type m0 is not used: mut_types = set([md['mutation_type'] diff --git a/docs/vignette_parallel_phylo.md b/docs/vignette_parallel_phylo.md index 794948b6..23d1badc 100644 --- a/docs/vignette_parallel_phylo.md +++ b/docs/vignette_parallel_phylo.md @@ -314,11 +314,9 @@ subsample_nodes = [ np.concatenate([tsu.individual(i).nodes for i in x]) for x in subsample_indivs ] -tsus = pyslim.SlimTreeSequence( - tsu.simplify( +tsus = tsu.simplify( np.concatenate(subsample_nodes), filter_populations=False, - ) ) pop_labels = {v: k for k, v in pop_ids.items()} SVG(tsus.draw_svg( diff --git a/pyslim/slim_tree_sequence.py b/pyslim/slim_tree_sequence.py index af4ab42d..c35a4370 100644 --- a/pyslim/slim_tree_sequence.py +++ b/pyslim/slim_tree_sequence.py @@ -26,6 +26,17 @@ NUCLEOTIDES = ['A', 'C', 'G', 'T'] +def _deprecation_warning(w): + warnings.warn( + "The SlimTreeSequence class is being phased out, " + "as most important functionality is provided by tskit. " + "Please see the " + "`documentation `_. " + f"{w}", + FutureWarning + ) + + def load(path, legacy_metadata=False): ''' Load the SLiM-compatible tree sequence found in the .trees file at ``path``. @@ -36,11 +47,11 @@ def load(path, legacy_metadata=False): raise ValueError( "It looks like you're trying to use pyslim legacy metadata tools, " "which are no longer supported. See `the documentation " - "`_" + "`_" "for how to update your script." ) - ts = SlimTreeSequence.load(path, legacy_metadata=legacy_metadata) + ts = SlimTreeSequence.load(path) return ts @@ -119,11 +130,12 @@ class MetadataDictWrapper(dict): def __getattr__(self, name): if name in self.keys(): raise AttributeError( - f"'dict' object has no attribute '{name}'. " - "It looks like you're trying to use the legacy " - "metadata interface: see " - "`the documentation `_ " - "for how to switch over your script") + f"'dict' object has no attribute '{name}'. " + "It looks like you're trying to use the legacy " + "metadata interface: see the " + "`documentation `_ " + "for how to switch over your script" + ) else: raise AttributeError(f"'dict' object has no attribute '{name}'") @@ -133,10 +145,12 @@ def __getitem__(self, key): except KeyError as e: if isinstance(key, int): msg = e.args[0] - e.args = (f"{msg}: It looks like you're trying to use the legacy " - "metadata interface: see " - "`the documentation `_ " - "for how to switch over your script",) + e.args = ( + f"{msg}: It looks like you're trying to use the legacy " + "metadata interface: see the " + "`documentation `_ " + "for how to switch over your script", + ) raise e @@ -170,7 +184,7 @@ def __init__(self, ts, legacy_metadata=False): raise ValueError( "It looks like you're trying to use pyslim legacy metadata tools, " "which are no longer supported. See `the documentation " - "`_" + "`_" "for how to update your script." ) @@ -186,7 +200,7 @@ def __init__(self, ts, legacy_metadata=False): self.individual_locations = ts.tables.individuals.location self.individual_locations.shape = (int(len(self.individual_locations)/3), 3) self.individual_ages = np.zeros(ts.num_individuals, dtype='int') - if self.model_type != "WF": + if self.metadata['SLiM']['model_type'] != "WF": self.individual_ages = np.fromiter(map(lambda ind: ind.metadata['age'], ts.individuals()), dtype='int64') self.individual_times = np.zeros(ts.num_individuals) @@ -216,16 +230,17 @@ def __setstate__(self, state): @property def slim_generation(self): - # return self.metadata['SLiM']['generation'] + _deprecation_warning("Please access ts.metadata['SLiM']['generation'] instead.") return self._slim_generation @slim_generation.setter def slim_generation(self, value): - # TODO: throw a deprecation warning here after stdpopsim updates + _deprecation_warning("Please access ts.metadata['SLiM']['generation'] instead.") self._slim_generation = value @property def model_type(self): + _deprecation_warning("Please access ts.metadata['SLiM']['model_type'] instead.") return self.metadata['SLiM']['model_type'] @classmethod @@ -254,17 +269,17 @@ def load_tables(cls, tables, **kwargs): def dump(self, path, **kwargs): ''' - Dumps the tree sequence to the path specified. This is mostly just a wrapper for - :func:`tskit.TreeSequence.dump`, but also writes out the reference sequence. + Dumps the tree sequence to the path specified. This is only a wrapper + :func:`tskit.TreeSequence.dump`. :param str path: The file path to write the TreeSequence to. :param kwargs: Additional keyword args to pass to tskit.TreeSequence.dump ''' # temporary until we remove support for setting slim_generation - if self.slim_generation != self.metadata['SLiM']['generation']: + if self._slim_generation != self.metadata['SLiM']['generation']: tables = self.dump_tables() md = tables.metadata - md['SLiM']['generation'] = self.slim_generation + md['SLiM']['generation'] = self._slim_generation tables.metadata = md tables.dump(path, **kwargs) else: @@ -282,10 +297,10 @@ def simplify(self, *args, **kwargs): :rtype SlimTreeSequence: ''' # temporary until we remove support for setting slim_generation - if self.slim_generation != self.metadata['SLiM']['generation']: + if self._slim_generation != self.metadata['SLiM']['generation']: tables = self.dump_tables() md = tables.metadata - md['SLiM']['generation'] = self.slim_generation + md['SLiM']['generation'] = self._slim_generation tables.metadata = md # note we have to go to a tree sequence to get the map_nodes argument sts = tables.tree_sequence().simplify(*args, **kwargs) @@ -407,6 +422,7 @@ def recapitate(self, if "keep_first_generation" in kwargs: raise ValueError("The keep_first_generation argument is deprecated: " "the FIRST_GEN flag is no longer used.") + _deprecation_warning("Please use pyslim.recapitate( ) instead.") if recombination_map is None: recombination_map = msprime.RecombinationMap( @@ -418,10 +434,10 @@ def recapitate(self, population_configurations = [msprime.PopulationConfiguration() for _ in range(self.num_populations)] # temporary until we remove support for setting slim_generation - if self.slim_generation != self.metadata['SLiM']['generation']: + if self._slim_generation != self.metadata['SLiM']['generation']: tables = self.dump_tables() md = tables.metadata - md['SLiM']['generation'] = self.slim_generation + md['SLiM']['generation'] = self._slim_generation tables.metadata = md ts = tables.tree_sequence() else: diff --git a/tests/test_annotation.py b/tests/test_annotation.py index d69736ac..b864501a 100644 --- a/tests/test_annotation.py +++ b/tests/test_annotation.py @@ -448,7 +448,7 @@ def test_load_without_provenance( in_tables.sort() cleared_ts = pyslim.SlimTreeSequence( in_tables.tree_sequence(), - ) + ) out_ts = helper_functions.run_slim_restart(cleared_ts, restart_name, tmp_path) out_tables = out_ts.dump_tables() out_tables.provenances.clear() diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 8bcc8d4b..2650e48f 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -135,7 +135,8 @@ def test_set_tree_sequence_metadata(self, recipe): spatial_dimensionality='xy', spatial_periodicity='y', separate_sexes=False, - nucleotide_based=True) + nucleotide_based=True + ) self.validate_slim_metadata(tables) assert tables.metadata['SLiM']['model_type'] == "WF" assert tables.metadata['SLiM']['generation'] == 99 diff --git a/tests/test_tree_sequence.py b/tests/test_tree_sequence.py index c0344930..e7c6090d 100644 --- a/tests/test_tree_sequence.py +++ b/tests/test_tree_sequence.py @@ -68,15 +68,19 @@ def test_bad_metadata(self): def test_slim_generation(self, recipe, tmp_path): # tests around awkward backwards-compatible patch for setting slim_generation ts = recipe["ts"] - assert ts.slim_generation == ts.metadata['SLiM']['generation'] + with pytest.warns(FutureWarning): + assert ts.slim_generation == ts.metadata['SLiM']['generation'] new_sg = 12345 - ts.slim_generation = new_sg - assert ts.slim_generation == new_sg + with pytest.warns(FutureWarning): + ts.slim_generation = new_sg + with pytest.warns(FutureWarning): + assert ts.slim_generation == new_sg # check persists through dump/load temp_file = tmp_path / "temp.trees" ts.dump(temp_file.name) loaded_ts = pyslim.load(temp_file.name) - assert loaded_ts.slim_generation == new_sg + with pytest.warns(FutureWarning): + assert loaded_ts.slim_generation == new_sg assert loaded_ts.metadata['SLiM']['generation'] == new_sg # check persists through recapitate recap = self.do_recapitate(ts, recombination_rate=1e-8, ancestral_Ne=10) @@ -245,7 +249,8 @@ def test_with_demography(self, recipe): def test_old_recapitate_errors(self, recipe): ts = recipe["ts"] with pytest.raises(ValueError): - _ = ts.recapitate( + with pytest.warns(FutureWarning): + _ = ts.recapitate( recombination_rate=0.0, keep_first_generation=True) @@ -254,14 +259,16 @@ def test_old_recapitation(self, recipe): if ts.num_populations <= 2: # if not we need migration rates recomb_rate = 1.0 / ts.sequence_length - recap = ts.recapitate(recombination_rate=recomb_rate) + with pytest.warns(FutureWarning): + recap = ts.recapitate(recombination_rate=recomb_rate) # there should be no new mutations assert ts.num_mutations == recap.num_mutations assert ts.num_sites == recap.num_sites assert list(ts.tables.sites.position) == list(recap.tables.sites.position) self.check_recap_consistency(ts, recap) - recap = ts.recapitate(recombination_rate=recomb_rate, Ne=1e-6) + with pytest.warns(FutureWarning): + recap = ts.recapitate(recombination_rate=recomb_rate, Ne=1e-6) self.check_recap_consistency(ts, recap) if ts.metadata['SLiM']['generation'] < 200: for t in recap.trees(): @@ -272,7 +279,8 @@ def test_old_recapitation(self, recipe): positions = [0.0, ts.sequence_length], rates = [recomb_rate, 0.0], num_loci=int(ts.sequence_length)) - recap = ts.recapitate(recombination_map=recombination_map, Ne=1e-6) + with pytest.warns(FutureWarning): + recap = ts.recapitate(recombination_map=recombination_map, Ne=1e-6) self.check_recap_consistency(ts, recap) @@ -323,7 +331,11 @@ def test_slim_time(self, recipe): ts = recipe["ts"] # check that slim_times make sense, i.e., that # slim_generation == (time + slim_time + (model_type == "WF" and stage="early")) - offset = (ts.metadata["SLiM"]["model_type"] == "WF") and (ts.metadata["SLiM"]["stage"] == "early") + offset = ( + (ts.metadata["SLiM"]["model_type"] == "WF") + and + (ts.metadata["SLiM"]["stage"] == "early") + ) for mut in ts.mutations(): mut_slim_time = max([u["slim_time"] for u in mut.metadata["mutation_list"]]) assert ts.metadata['SLiM']['generation'] == mut_slim_time + mut.time + offset From 24fe2a132fb050f782863a70647ac339f448cbba Mon Sep 17 00:00:00 2001 From: peter Date: Sun, 19 Dec 2021 04:43:06 -0800 Subject: [PATCH 2/3] switch docs build to github install slim from conda remove windows increase timeout bump conda cache --- .circleci/config.yml | 30 ---------------------------- .github/workflows/tests.yml | 40 +++++++++++++++++++++++++------------ docs/vignette_space.md | 8 +++++--- tests/test_tree_sequence.py | 2 +- 4 files changed, 33 insertions(+), 47 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea65694c..bc6231a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,39 +47,9 @@ jobs: paths: - "/home/circleci/.local" - - run: - name: Build docs - command: | - # don't re-generate these which require tricky prerequisites (inkscape, xmlstarlet) - touch docs/_static/{pedigree0.svg,pedigree1.svg,pedigree2.svg,pedigree_recapitate.svg,pedigree_simplify.svg,pedigree_mutate.svg} - touch docs/_static/{pedigree01.png,pedigree0.png,pedigree1.png,pedigree2.png,pedigree_recapitate.png,pedigree_simplify.png,pedigree_mutate.png} - cd docs && make - - - run: - name: Run Python tests - command: | - PYTHONPATH=python pytest --cov=pyslim --cov-report=xml --cov-branch \ - -n 1 tests - - run: name: Upload Python coverage command: | bash <(curl -s https://codecov.io/bash) -X gcov -n python_tests # Clean up reports so we don't upload them twice. rm -f coverage.xml - - - run: - name: Build Python package - command: | - rm -fR build - python setup.py sdist - python setup.py check - python -m twine check dist/*.tar.gz - python -m venv venv - source venv/bin/activate - pip install --upgrade setuptools pip wheel - python setup.py build_ext - python setup.py egg_info - python setup.py bdist_wheel - pip install dist/*.tar.gz - python3 -c "import pyslim; print(pyslim.__version__)" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 99a618f0..a3c04de1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,13 +31,14 @@ jobs: - name: Cache conda and dependancies id: cache uses: actions/cache@v2 + env: + # Increase this to reset the cache if the key hasn't changed. + CACHE_NUM: 1 with: path: | - c:\Miniconda\envs\anaconda-client-env - /usr/share/miniconda/envs/anaconda-client-env - ~/osx-conda - ~/.profile - key: ${{ runner.os }}-${{ matrix.python}}-conda-v10-${{ hashFiles('requirements/CI/pip-requirements.txt') }}-${{ hashFiles('requirements/CI/conda-requirements.txt') }} + ${{ steps.find-conda.outputs.CONDA }}/envs/${{ env.CONDA_ENV_NAME }} + ~/.bash_profile + key: ${{ runner.os }}-${{ matrix.python}}-conda-v10-${{ hashFiles('requirements/CI/pip-requirements.txt') }}-${{ hashFiles('requirements/CI/conda-requirements.txt') }}-${{ env.CACHE_NUM }} - name: Install Conda uses: conda-incubator/setup-miniconda@v2 @@ -58,7 +59,11 @@ jobs: - name: Install conda deps if: steps.cache.outputs.cache-hit != 'true' shell: bash -l {0} #We need a login shell to get conda - run: conda install --yes --file=requirements/CI/conda-requirements.txt + run: | + if [ "$RUNNER_OS" != "Windows" ]; then + conda install --yes slim + fi + conda install --yes --file=requirements/CI/conda-requirements.txt - name: Install pip deps if: steps.cache.outputs.cache-hit != 'true' @@ -76,13 +81,14 @@ jobs: mkdir -p /usr/local/miniconda/envs sudo cp -r ~/osx-conda /usr/local/miniconda/envs/anaconda-client-env - - name: Build SLiM - run: | - git clone https://github.com/messerlab/SLiM.git - mkdir -p SLiM/Release - cd SLiM/Release - cmake -D CMAKE_BUILD_TYPE=Release .. - make -j 2 +# Installing SLiM from conda now. +# - name: Build SLiM +# run: | +# git clone https://github.com/messerlab/SLiM.git +# mkdir -p SLiM/Release +# cd SLiM/Release +# cmake -D CMAKE_BUILD_TYPE=Release .. +# make -j 2 - name: Run tests run: | @@ -91,5 +97,13 @@ jobs: export PATH=$PATH:$PWD/SLiM/Release pytest -xv -n2 + - name: Build docs + run: | + source ~/.profile + conda activate anaconda-client-env + export PATH=$PATH:$PWD/SLiM/Release + touch docs/_static/{pedigree0.svg,pedigree1.svg,pedigree2.svg,pedigree_recapitate.svg,pedigree_simplify.svg,pedigree_mutate.svg} + touch docs/_static/{pedigree01.png,pedigree0.png,pedigree1.png,pedigree2.png,pedigree_recapitate.png,pedigree_simplify.png,pedigree_mutate.png} + cd docs && make diff --git a/docs/vignette_space.md b/docs/vignette_space.md index 3d202042..d42402e7 100644 --- a/docs/vignette_space.md +++ b/docs/vignette_space.md @@ -6,9 +6,11 @@ jupytext: format_version: 0.12 jupytext_version: 1.9.1 kernelspec: - display_name: Python 3 - language: python - name: python3 + display_name: Python 3 + language: python + name: python3 +execution: + timeout: 90 --- ```{code-cell} diff --git a/tests/test_tree_sequence.py b/tests/test_tree_sequence.py index e7c6090d..bcda8e06 100644 --- a/tests/test_tree_sequence.py +++ b/tests/test_tree_sequence.py @@ -3,9 +3,9 @@ """ import pickle import random +import numpy as np import os -import numpy as np import pytest import tskit import msprime From fb4a94434eee0a6664bfb00763af5bc843714414 Mon Sep 17 00:00:00 2001 From: peter Date: Sun, 19 Dec 2021 15:56:36 -0800 Subject: [PATCH 3/3] update slim example recipes --- .github/workflows/tests.yml | 2 +- tests/extra_recipes/Makefile | 11 ++++++++--- .../Recipe 17.2 - Overlaying neutral mutations.py | 3 ++- ...diversity (analyzing tree heights in Python) II.py | 3 ++- ... sexual nonWF model with a coalescent history I.py | 1 - ...quence recording and nucleotide-based models II.py | 2 +- ...uence recording and nucleotide-based models III.py | 2 +- 7 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a3c04de1..caaacb6a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: uses: actions/cache@v2 env: # Increase this to reset the cache if the key hasn't changed. - CACHE_NUM: 1 + CACHE_NUM: 2 with: path: | ${{ steps.find-conda.outputs.CONDA }}/envs/${{ env.CONDA_ENV_NAME }} diff --git a/tests/extra_recipes/Makefile b/tests/extra_recipes/Makefile index 930afda5..1d4fd95b 100644 --- a/tests/extra_recipes/Makefile +++ b/tests/extra_recipes/Makefile @@ -36,8 +36,13 @@ check : recipe_17.1_overlaid.log recipe_17.3.log recipe_17.4.png recipe_17.5.png clean : -rm -f *.trees *.log *.png +sanitize : + for x in *; do mv -v "$x" "${x/:/-}"; done + for x in *; do mv -v "$x" "${x/(/-}"; done + for x in *; do mv -v "$x" "${x/)/-}"; done + recipe_17.1.trees : Recipe\ 17.1\ -\ A\ minimal\ tree-seq\ model.txt - slim -s 0 "$<" &> $@.log + slim -s 5 "$<" &> $@.log recipe_17.1_overlaid.trees : Recipe\ 17.2\ -\ Overlaying\ neutral\ mutations.py recipe_17.1.trees python3 "$<" @@ -51,10 +56,10 @@ slim_2345_FIXED.trees : Recipe\ 17.3\ -\ Simulation\ conditional\ upon\ fixation recipe_17.3.log : slim_2345_FIXED.trees ./check_trees.py $< > $@ -recipe_17.4.trees : Recipe\ 17.4\ -\ Detecting\ the\ "dip\ in\ diversity"\ (analyzing\ tree\ heights\ in\ Python)\ I.txt +recipe_17.4.trees : Recipe\ 17.4\ -\ Detecting\ the\ dip\ in\ diversity\ (analyzing\ tree\ heights\ in\ Python)\ I.txt slim -s 2345 '$<' &> $@.log -recipe_17.4.png : Recipe\ 17.4\ -\ Detecting\ the\ "dip\ in\ diversity"\ (analyzing\ tree\ heights\ in\ Python)\ II.py recipe_17.4.trees +recipe_17.4.png : Recipe\ 17.4\ -\ Detecting\ the\ dip\ in\ diversity\ (analyzing\ tree\ heights\ in\ Python)\ II.py recipe_17.4.trees python3 '$<' recipe_17.5.trees: Recipe\ 17.5\ -\ Mapping\ admixture\ (analyzing\ ancestry\ in\ Python)\ I.txt diff --git a/tests/extra_recipes/Recipe 17.2 - Overlaying neutral mutations.py b/tests/extra_recipes/Recipe 17.2 - Overlaying neutral mutations.py index a9977a7a..837137f2 100644 --- a/tests/extra_recipes/Recipe 17.2 - Overlaying neutral mutations.py +++ b/tests/extra_recipes/Recipe 17.2 - Overlaying neutral mutations.py @@ -4,7 +4,8 @@ import msprime, pyslim -ts = pyslim.load("./recipe_17.1.trees").simplify() +ts = pyslim.load("./recipe_17.1.trees") +ts = ts.simplify() ## EDIT FOR TESTING asserted = False diff --git a/tests/extra_recipes/Recipe 17.4 - Detecting the dip in diversity (analyzing tree heights in Python) II.py b/tests/extra_recipes/Recipe 17.4 - Detecting the dip in diversity (analyzing tree heights in Python) II.py index 6b8e3a3c..407e6dbc 100644 --- a/tests/extra_recipes/Recipe 17.4 - Detecting the dip in diversity (analyzing tree heights in Python) II.py +++ b/tests/extra_recipes/Recipe 17.4 - Detecting the dip in diversity (analyzing tree heights in Python) II.py @@ -8,7 +8,8 @@ # Run the SLiM model and load the resulting .trees # subprocess.check_output(["slim", "-m", "-s", "0", "./recipe_17.4.slim"]) -ts = pyslim.load("./recipe_17.4.trees").simplify() +ts = pyslim.load("./recipe_17.4.trees") +ts = ts.simplify() # Measure the tree height at each base position height_for_pos = np.zeros(int(ts.sequence_length)) diff --git a/tests/extra_recipes/Recipe 17.9 - Starting a sexual nonWF model with a coalescent history I.py b/tests/extra_recipes/Recipe 17.9 - Starting a sexual nonWF model with a coalescent history I.py index fcc142da..95bc5d3e 100644 --- a/tests/extra_recipes/Recipe 17.9 - Starting a sexual nonWF model with a coalescent history I.py +++ b/tests/extra_recipes/Recipe 17.9 - Starting a sexual nonWF model with a coalescent history I.py @@ -20,7 +20,6 @@ [ims.validate_and_encode_row(md) for md in individual_metadata]) # add selected mutation - mut_ind_id = random.choice(range(tables.individuals.num_rows)) mut_node_id = random.choice(np.where(tables.nodes.individual == mut_ind_id)[0]) mut_node = tables.nodes[mut_node_id] diff --git a/tests/extra_recipes/Recipe 18.13 - Tree-sequence recording and nucleotide-based models II.py b/tests/extra_recipes/Recipe 18.13 - Tree-sequence recording and nucleotide-based models II.py index 357ef75f..c5c19398 100644 --- a/tests/extra_recipes/Recipe 18.13 - Tree-sequence recording and nucleotide-based models II.py +++ b/tests/extra_recipes/Recipe 18.13 - Tree-sequence recording and nucleotide-based models II.py @@ -11,7 +11,7 @@ k = np.argmax([u["slim_time"] for u in mut_list]) derived_nuc = mut_list[k]["nucleotide"] if mut.parent == -1: - acgt = ts.reference_sequence[int(mut.position)] + acgt = ts.reference_sequence.data[int(ts.site(mut.site).position)] parent_nuc = pyslim.NUCLEOTIDES.index(acgt) else: parent_mut = ts.mutation(mut.parent) diff --git a/tests/extra_recipes/Recipe 18.13 - Tree-sequence recording and nucleotide-based models III.py b/tests/extra_recipes/Recipe 18.13 - Tree-sequence recording and nucleotide-based models III.py index 36be8c52..308baf15 100644 --- a/tests/extra_recipes/Recipe 18.13 - Tree-sequence recording and nucleotide-based models III.py +++ b/tests/extra_recipes/Recipe 18.13 - Tree-sequence recording and nucleotide-based models III.py @@ -19,7 +19,7 @@ right_nuc = ts.nucleotide_at(mut.node, pos + 1, time = slim_gen - mut_list[k]["slim_time"] - 1.0) if mut.parent == -1: - acgt = ts.reference_sequence[int(mut.position)] + acgt = ts.reference_sequence.data[int(ts.site(mut.site).position)] parent_nuc = pyslim.NUCLEOTIDES.index(acgt) else: parent_mut = ts.mutation(mut.parent)