other-ot-peyotl-dependencies-snapshot.txt

###############
phylesystem-api
###############
controllers/default.py
======================
from peyotl import convert_nexson_format
from peyotl.utility.str_util import slugify
from peyotl.phylesystem.git_workflows import GitWorkflowError, \
                                             validate_and_convert_nexson
from peyotl.collections import OWNER_ID_PATTERN, \
                               COLLECTION_ID_PATTERN
from peyotl.collections.validation import validate_collection
from peyotl.amendments import AMENDMENT_ID_PATTERN
from peyotl.amendments.validation import validate_amendment
from peyotl.nexson_syntax import get_empty_nexson, \
                                 extract_supporting_file_messages, \
                                 extract_tree, \
                                 PhyloSchema, \
                                 read_as_json, \
                                 BY_ID_HONEY_BADGERFISH
from peyotl.external import import_nexson_from_treebase
+++++++++++++++++++++++++++++++++++++++++++++++++++++++
convert_nexson_format(nexson, dest_format, current_format=current_format)
GitWorkflowError instance has msg str
validate_and_convert_nexson(nexson, repo_nexml2json, allow_invalid=False, max_num_trees_per_study=max_num_trees)
OWNER_ID_PATTERN COLLECTION_ID_PATTERN, AMENDMENT_ID_PATTERN are regex
errors, collection_adaptor = validate_collection(collection_obj)
errors, amendment_adaptor = validate_amendment(amendment_obj)
new_study_nexson = get_empty_nexson(BY_ID_HONEY_BADGERFISH, include_cc0=cc0_agreement)
m_list = extract_supporting_file_messages(study_nexson)

schema = PhyloSchema(schema=kwargs.get('format'), type_ext=type_ext, content=resource, content_id=content_id, repo_nexml2json=repo_nexml2json, **kwargs)
where  kwargs['output_nexml2json'] string like '0.0.0'
       kwargs['bracket_ingroup'] bool
       kwargs from GET optional args
       kwargs.get('subtree_id')
src_schema = PhyloSchema('nexson', version=repo_nexml2json)
blob = read_as_json(fail_file_filename)

new_study_nexson = import_nexson_from_treebase(treebase_id, nexson_syntax_version=BY_ID_HONEY_BADGERFISH)

====================
controllers/merge.py
====================
from peyotl.phylesystem.git_workflows import GitWorkflowError, \
                                             merge_from_master
auth_info = api_utils.authenticate(**kwargs)
gd = phylesystem.create_git_action(resource_id)
try:
    return merge_from_master(gd, resource_id, auth_info, starting_commit_SHA)

======================
controllers/studies.py
======================
from peyotl.api import OTI
++++++++++++++++++++++++++
OTI(oti=api_utils.get_oti_domain(request))

====================
modules/api_utils.py
====================
from peyotl.nexson_syntax import write_as_json
from peyotl.phylesystem import Phylesystem
from peyotl.collections import TreeCollectionStore
from peyotl.amendments import TaxonomicAmendmentStore
from peyotl.utility import read_config as read_peyotl_config NEED to change this to get_raw_default_config_and_read_file_list
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
write_as_json(obj, tmpfn, indent=2, sort_keys=True)
_PHYLESYSTEM = Phylesystem(repos_par=repo_parent,
                               git_ssh=git_ssh,
                               pkey=pkey,
                               git_action_class=GitData,
                               mirror_info={'push':{'parent_dir': push_mirror,
                                                    'remote_map': {'GitHubRemote': git_hub_remote,},
                                                   }
                                           },
                               new_study_prefix=STR_FROM_CONFIG})
_TREE_COLLECTION_STORE = TreeCollectionStore(repos_par=repo_parent,
                                                 git_ssh=git_ssh,
                                                 pkey=pkey,
                                                 git_action_class=GitData, #TODO?
                                                 mirror_info=mirror_info)
_TAXONOMIC_AMENDMENT_STORE = TaxonomicAmendmentStore(repos_par=repo_parent,
                                                 git_ssh=git_ssh,
                                                 pkey=pkey,
                                                 git_action_class=GitData, #TODO?
                                                 mirror_info=mirror_info)
peyotl_config, cfg_filename = read_peyotl_config()
====================
modules/gitdata.py
====================
from peyotl.phylesystem.git_actions import PhylesystemGitAction
class GitData(PhylesystemGitAction):
    def __init__(self, repo, **kwargs):
        PhylesystemGitAction.__init__(self, repo, **kwargs)
    def delete_remote_branch(self, remote, branch, env={}):

======================================
ws-tests/test_sutdy_get_multiformat.py
ws-tests/test_sutdy_put_multiformat.py
======================================
from peyotl import convert_nexson_format
blob =  convert_nexson_format(badger, '1.0.0', current_format='0.0.0')
n = convert_nexson_format(n, '1.2')


################
opentree/curator
################
from peyotl.manip import merge_otus_and_trees, iter_trees
from peyotl.nexson_syntax import can_convert_nexson_forms, \
                                     get_ot_study_info_from_nexml, \
                                     add_resource_meta, \
                                     convert_nexson_format, \
                                     BADGER_FISH_NEXSON_VERSION
    from peyotl.manip import count_num_trees
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
o = merge_otus_and_trees(nexson)
tree_ids = [tree_id for (tree_group_id, tree_id, imported_tree) in iter_trees(nex)]

can_convert_nexson_forms('nexml', NEXSON_VERSION_STR)
get_ot_study_info_from_nexml(NEXML_FILEPATH, nexson_syntax_version=NEXSON_VERSION_STR)
add_resource_meta(n, "ot:dataDeposit", dd, NEXSON_VERSION)
num_trees = count_num_trees(nex, NEXSON_VERSION_STR)


#######
otindex
#######
=================================
dev_scripts/add_update_studies.py
=================================
from peyotl.nexson_proxy import NexsonProxy
+++++++++++++++++++++++++++++++++++++++++++
studyobj = NexsonProxy(nexson=nexson_blob)
nexml = studyobj._nexml_el
study_id = nexml['^ot:studyId']
nexml['treesById']

=======
various
=======
from peyotl.api.phylesystem_api import PhylesystemAPI
from peyotl import gen_otu_dict, iter_node
from peyotl.manip import iter_trees
import peyotl.ott as ott
++++++++++++++++++++++++++++++++++++++++++
PhylesystemAPI(get_from='local') and PhylesystemAPI()
for node_id, node in iter_node(tree)
for study_id, n in phy.iter_study_objs():
    otu_dict = gen_otu_dict(n)
for trees_group_id, tree_id, tree in iter_trees(studyobj):
from peyotl.nexson_syntax import get_nexml_el
nexml = get_nexml_el(studyobj)
taxonomy = ott.OTT(ott_loc)
taxonomy.ott_id_to_names
ott_parents = taxonomy.ott_id2par_ott_id

###########
propinquity
###########
$(PEYOTL_ROOT)/scripts/ott/suppress_by_flag.py \
      --ott-dir=$(OTT_DIR) \
      --output=$(PROPINQUITY_OUT_DIR)/cleaned_ott/cleaned_ott_with_hiddenbarren.tre \
      --log=$(PROPINQUITY_OUT_DIR)/cleaned_ott/cleaned_ott_1.json \
      --flags="$(shell cat $(PROPINQUITY_OUT_DIR)/cleaned_ott/cleaning_flags.txt)" \
      --root="$(shell cat $(PROPINQUITY_OUT_DIR)/cleaned_ott/root_ott_id.txt)"
$(PEYOTL_ROOT)/scripts/nexson/prune_to_clean_mapped.py \
    --ott-dir=$(OTT_DIR) \
    --input-files-list=$(PROPINQUITY_OUT_DIR)/cleaned_phylo/needs_updating.txt \
    --out-dir=$(PROPINQUITY_OUT_DIR)/cleaned_phylo \
    --ott-prune-flags="$(shell cat $(PROPINQUITY_OUT_DIR)/cleaned_ott/cleaning_flags.txt)" \
    --ott-prune-nonflagged-json=$(PROPINQUITY_OUT_DIR)/cleaned_ott/cleaned_ott_pruned_nonflagged.json \
    --root="$(shell cat $(PROPINQUITY_OUT_DIR)/cleaned_ott/root_ott_id.txt)"
"${PEYOTL_ROOT}"/scripts/nexson/prune_to_clean_mapped.py \
  --ott-dir="${OTT_DIR}" \
  --out-dir=cleaned_phylo \
  --ott-prune-flags=major_rank_conflict,major_rank_conflict_direct,major_rank_conflict_inherited,environmental,viral,nootu,barren,not_otu,extinct_inherited,extinct_direct,hidden,tattered \
  phylo_snapshot/${study_tree}.json && \
$(PEYOTL_ROOT)/scripts/collection_export.py \
    --export=studyID_treeID \
    $(PROPINQUITY_OUT_DIR)/phylo_snapshot/concrete_rank_collection.json \
    >$(PROPINQUITY_OUT_DIR)/phylo_input/study_tree_pairs.txt
$(PEYOTL_ROOT)/scripts/phylesystem/export_studies_from_collection.py \
    --phylesystem-par=$(PHYLESYSTEM_ROOT)/shards \
    --output-dir=$(PROPINQUITY_OUT_DIR)/phylo_snapshot \
    $(PROPINQUITY_OUT_DIR)/phylo_input/rank_collection.json \
    -v 2>&1 | tee $(PROPINQUITY_OUT_DIR)/phylo_snapshot/stdouterr.txt
"${PEYOTL_ROOT}/scripts/nexson/propinquity_newick_to_nexson.py" \
    "--ids=${tree_id}" \
    "${filename}" > "$PROPINQUITY_OUT_DIR/phylo_snapshot/${stem}.json"
$PEYOTL_ROOT/scripts/concatenate_collections.py $COLLECTIONS --output="${outp}"
===============================
from peyotl import read_as_json, write_as_json
from peyotl.ott import OTT

from peyotl.nexson_syntax import quote_newick_name
from peyotl.utility import propinquity_fn_to_study_tree
peyotl.__version__
+++++++++++++++++++++++++++++++

ott = OTT(ott_dir=args.ott_dir)
# load up the OTT dictionary...
d = ott.ott_id_to_names

##########
germinator
##########
from peyotl.sugar import taxomachine as TNRS
PEYOTL_CONFIG_FILE
PEYOTL_LOG_FILE_PATH
PEYOTL_LOGGING_LEVEL
from peyotl.phylesystem.phylesystem_umbrella import Phylesystem
from peyotl.nexson_syntax import extract_tree_nexson


##
Phylesystem-api
###############
amendments and collections
==========
docstore = api_utils.get_taxonomic_amendment_store(request)

for id, props in docstore.iter_doc_objs():
    latest_commit = docstore.get_version_history_for_doc_id(id)[0]
    latest_commit.get('author_name'),
    latest_commit.get('relative_date'),
    latest_commit.get('date'),
    latest_commit.get('date_ISO_8601'),
    latest_commit.get('id')  # this is the commit hash

docstore.get_amendment_ids()

docstore.get_configuration_dict()


amendment_obj = __extract_json_from_http_call(request, data_field_name='json', **kwargs)

__extract_and_validate_amendment
    errors, amendment_adaptor = validate_amendment(amendment_obj)
        return amendment_obj, errors, amendment_adaptor


AMENDMENT_ID_PATTERN.match(amendment_id):

amendment_obj, amendment_errors, amendment_adapter = __extract_and_validate_amendment(request,
                                                                                              kwargs)

docstore = api_utils.get_taxonomic_amendment_store(request)
r = docstore.return_doc(amendment_id, commit_sha=parent_sha, return_WIP_map=True)
amendment_json, head_sha, wip_map = r
version_history = docstore.get_version_history_for_doc_id(amendment_id)
external_url = amendments.get_public_url(amendment_id)


r = docstore.update_existing_amendment(amendment_id,
                                           amendment_obj,
                                           auth_info,
                                           parent_sha,
                                           merged_sha,
                                           commit_msg=commit_msg)
mn = r.get('merge_needed')
if (mn is not None) and (not mn):
    __deferred_push_to_gh_call(request, amendment_id, doc_type='amendment', **kwargs)
return r


r = docstore.add_new_amendment(amendment_obj,
                                           auth_info,
                                           commit_msg=commit_msg)
new_amendment_id, commit_return = r
commit_return['error'] != 0:
__deferred_push_to_gh_call(request, new_amendment_id, doc_type='amendment', **kwargs)

x = docstore.delete_amendment(amendment_id,
                                          auth_info,
                                          parent_sha,
                                          commit_msg=commit_msg)
if x.get('error') == 0:



url = collection_obj.get('url', None)
collection_id = url.split('/collection/')[1]
assert collection_id.split('/')[0] == owner_id



phylesystem = api_utils.get_phylesystem(request)
shard_name, path_frag = phylesystem.get_repo_and_path_fragment(study_id)

repo_parent, repo_remote, git_ssh, pkey, git_hub_remote, max_filesize, max_num_trees = api_utils.read_phylesystem_config(request)
repo_nexml2json = phylesystem.repo_nexml2json

def __validate_output_nexml2json(kwargs, resource, type_ext, content_id=None, subID, or (subID, subID)):
    kwargs['output_nexml2json'] = 'x.x.x'
    kwargs['bracket_ingroup'] = bool
    schema = PhyloSchema(schema=kwargs.get('format'),
                         type_ext=type_ext,
                         content=resource,
                         content_id=content_id,
                         repo_nexml2json=repo_nexml2json,
                         **kwargs)
    schema.can_convert_from(resource):

def __finish_write_verb(phylesystem, git_data, nexson, resource_id, auth_info, adaptor, annotation, parent_sha, commit_msg='', master_file_blob_included=None):
    a = phylesystem.annotate_and_write(git_data,
                                       nexson,
                                       resource_id,
                                       auth_info,
                                       adaptor,
                                       annotation,
                                       parent_sha,
                                       commit_msg,
                                       master_file_blob_included)
    a['error'] != 0:

valid_subresources = ('tree', 'meta', 'otus', 'otu', 'otumap')
returning_full_study = False
returning_tree = False
content_id = None
version_history = None
comment_html = None
type_ext = None or phylo-suffix
out_schema = __validate_output_nexml2json(kwargs,
                                          return_type,
                                          type_ext,
                                          content_id=content_id)
parent_sha = kwargs.get('starting_commit_SHA')
phylesystem = api_utils.get_phylesystem(request)
r = phylesystem.return_study(resource_id, commit_sha=parent_sha, return_WIP_map=True)
study_nexson, head_sha, wip_map = r
blob_sha = phylesystem.get_blob_sha_for_study_id(resource_id, head_sha)
phylesystem.add_validation_annotation(study_nexson, blob_sha)
version_history = phylesystem.get_version_history_for_study_id(resource_id)


bundle = __extract_and_validate_nexson(request,
                                               repo_nexml2json,
                                               kwargs)
        new_study_nexson = bundle[0]

r = phylesystem.ingest_new_study(new_study_nexson,
                                         repo_nexml2json,
                                         auth_info,
                                         new_study_id)
new_resource_id, commit_return = r

def __extract_and_validate_nexson(request, repo_nexml2json, kwargs):
    nexson = __extract_nexson_from_http_call(request, **kwargs)
    bundle = validate_and_convert_nexson(nexson, repo_nexml2json, allow_invalid=False, max_num_trees_per_study=max_num_trees)
    nexson, annotation, validation_log, nexson_adaptor = bundle
    return nexson, annotation, nexson_adaptor


auth_info = api_utils.authenticate(**kwargs)
bundle = __extract_and_validate_nexson(request, repo_nexml2json, kwargs)
nexson, annotation, nexson_adaptor = bundle

gd = phylesystem.create_git_action(resource_id)
blob = __finish_write_verb(phylesystem, gd, nexson=nexson, resource_id=resource_id, auth_info=auth_info, adaptor=nexson_adaptor, annotation=annotation, parent_sha=parent_sha, commit_msg=commit_msg, master_file_blob_included=master_file_blob_included)
mn = blob.get('merge_needed')


x = phylesystem.delete_study(resource_id, auth_info, parent_sha, commit_msg=commit_msg)
if x.get('error') == 0:




merge_from_master

docstore = api_utils.get_taxonomic_amendment_store(request)
docstore.push_doc_to_remote('GitHubRemote', resource_id)
ga = docstore.create_git_action(resource_id)
master_sha = ga.get_master_sha()