code/04_qiime2_part2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing marker-gene data in QIIME2, part2\n",
    "\n",
    "**Environment:** qiime2-2020.11"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How to use this notebook:\n",
    "1. Activate qiime2 conda environment\n",
    "   ```\n",
    "   conda activate qiime2-2020.11 \n",
    "   ```\n",
    "    \n",
    "2. Launch Jupyter notebook\n",
    "   ```\n",
    "   jupyter notebook\n",
    "   ```  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Hide excessive warnings (optional):\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/md0/nutrition_group Dropbox/projects/yanxian/AqFl1_microbiota\n"
     ]
    }
   ],
   "source": [
    "## change working directory to the project root directory\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Analyzing sequences at ASV level "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Import and visualize the filtered feature table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mImported data/intermediate/filtering/table-filtered.biom as BIOMV100Format to data/intermediate/qiime2/asv/table-filtered.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/table-filtered.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/taxa-bar-plots-filtered.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Import table\n",
    "!qiime tools import \\\n",
    "  --input-path data/intermediate/filtering/table-filtered.biom \\\n",
    "  --type 'FeatureTable[Frequency]' \\\n",
    "  --input-format BIOMV100Format \\\n",
    "  --output-path data/intermediate/qiime2/asv/table-filtered.qza\n",
    "\n",
    "# Feature table summary \n",
    "!qiime feature-table summarize \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/asv/table-filtered.qzv \n",
    "\n",
    "# Filtered taxonomic barplot \n",
    "!qiime taxa barplot \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered.qza \\\n",
    "  --i-taxonomy data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/asv/taxa-bar-plots-filtered.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Phylogeny "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Filter representative sequences based on the filtered feature table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureData[Sequence] to: data/intermediate/qiime2/asv/rep-seqs-filtered.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/rep-seqs-filtered.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime feature-table filter-seqs \\\n",
    "  --i-data data/intermediate/qiime2/asv/rep-seqs.qza \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered.qza \\\n",
    "  --p-no-exclude-ids \\\n",
    "  --o-filtered-data data/intermediate/qiime2/asv/rep-seqs-filtered.qza\n",
    "\n",
    "!qiime feature-table tabulate-seqs \\\n",
    "  --i-data data/intermediate/qiime2/asv/rep-seqs-filtered.qza \\\n",
    "  --o-visualization data/intermediate/qiime2/asv/rep-seqs-filtered.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Reference-based fragment insertion with SEPP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Phylogeny[Rooted] to: data/intermediate/qiime2/asv/insertion-tree.qza\u001b[0m\n",
      "\u001b[32mSaved Placements to: data/intermediate/qiime2/asv/tree-placements.qza\u001b[0m\n",
      "CPU times: user 1min 50s, sys: 18 s, total: 2min 8s\n",
      "Wall time: 1h 25min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!qiime fragment-insertion sepp \\\n",
    "  --i-representative-sequences data/intermediate/qiime2/asv/rep-seqs-filtered.qza \\\n",
    "  --i-reference-database data/reference/sepp-refs-silva-128.qza \\\n",
    "  --o-tree data/intermediate/qiime2/asv/insertion-tree.qza \\\n",
    "  --o-placements data/intermediate/qiime2/asv/tree-placements.qza \\\n",
    "  --p-threads 16 \\\n",
    "  --p-debug"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter uninserted representative sequences from the feature table  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/asv/table-filtered-sepp-inserted.qza\u001b[0m\r\n",
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/asv/table-filtered-sepp-uninserted.qza\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime fragment-insertion filter-features \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered.qza \\\n",
    "  --i-tree data/intermediate/qiime2/asv/insertion-tree.qza \\\n",
    "  --o-filtered-table data/intermediate/qiime2/asv/table-filtered-sepp-inserted.qza \\\n",
    "  --o-removed-table data/intermediate/qiime2/asv/table-filtered-sepp-uninserted.qza \\\n",
    "  --verbose"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature table summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/table-filtered-sepp-inserted.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/table-filtered-sepp-uninserted.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime feature-table summarize \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered-sepp-inserted.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/asv/table-filtered-sepp-inserted.qzv \n",
    "\n",
    "!qiime feature-table summarize \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered-sepp-uninserted.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/asv/table-filtered-sepp-uninserted.qzv "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Quality control: taxonomic composition of mock samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import expected taxonomic composition of mock samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mImported data/reference/mock-expected.biom as BIOMV210Format to data/reference/mock-expected.qza\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!biom convert \\\n",
    "  -i data/reference/mock_expected.tsv \\\n",
    "  -o data/reference/mock-expected.biom \\\n",
    "  --table-type=\"OTU table\" \\\n",
    "  --to-hdf5\n",
    "\n",
    "!qiime tools import \\\n",
    "  --input-path data/reference/mock-expected.biom \\\n",
    "  --type 'FeatureTable[RelativeFrequency]' \\\n",
    "  --input-format BIOMV210Format \\\n",
    "  --output-path data/reference/mock-expected.qza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the observed taxonomic composition of mock samples "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/asv/quality-control/mock-observed.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/quality-control/mock-observed.qzv\u001b[0m\n",
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/asv/quality-control/mock-observed-l7.qza\u001b[0m\n",
      "\u001b[32mSaved FeatureTable[RelativeFrequency] to: data/intermediate/qiime2/asv/quality-control/mock-observed-l7-rel.qza\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Subset mock samples\n",
    "!qiime feature-table filter-samples \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --p-where \"SampleType='Mock'\" \\\n",
    "  --p-no-exclude-ids \\\n",
    "  --o-filtered-table data/intermediate/qiime2/asv/quality-control/mock-observed.qza\n",
    "\n",
    "# Inspect the taxonomic composition of mock samples at ASV level\n",
    "!qiime taxa barplot \\\n",
    "  --i-table data/intermediate/qiime2/asv/quality-control/mock-observed.qza \\\n",
    "  --i-taxonomy data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/asv/quality-control/mock-observed.qzv\n",
    "\n",
    "# Agglomerate taxa at species level\n",
    "!qiime taxa collapse \\\n",
    "  --i-table data/intermediate/qiime2/asv/quality-control/mock-observed.qza \\\n",
    "  --i-taxonomy data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --p-level 7 \\\n",
    "  --o-collapsed-table data/intermediate/qiime2/asv/quality-control/mock-observed-l7.qza\n",
    "\n",
    "# Convert sequence counts into relative abundances\n",
    "!qiime feature-table relative-frequency \\\n",
    "  --i-table data/intermediate/qiime2/asv/quality-control/mock-observed-l7.qza \\\n",
    "  --o-relative-frequency-table data/intermediate/qiime2/asv/quality-control/mock-observed-l7-rel.qza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare observed and expected taxonomic composition of mock samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/quality-control/mock-comparison.qzv\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime quality-control evaluate-composition \\\n",
    "  --i-expected-features data/reference/mock-expected.qza \\\n",
    "  --i-observed-features data/intermediate/qiime2/asv/quality-control/mock-observed-l7-rel.qza \\\n",
    "  --o-visualization data/intermediate/qiime2/asv/quality-control/mock-comparison.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alpha and beta diversity analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exclude mock and negative control samples from the feature table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/asv/table-filtered-sepp-inserted-no-control.qza\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime feature-table filter-samples \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered-sepp-inserted.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --p-where \"SampleType IN ('Mock', 'Extraction-blank', 'PCR-blank')\" \\\n",
    "  --p-exclude-ids \\\n",
    "  --o-filtered-table data/intermediate/qiime2/asv/table-filtered-sepp-inserted-no-control.qza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rarefaction analysis "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/alpha-rarefaction.qzv\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime diversity alpha-rarefaction \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered-sepp-inserted-no-control.qza \\\n",
    "  --i-phylogeny data/intermediate/qiime2/asv/insertion-tree.qza \\\n",
    "  --p-max-depth 10532 \\\n",
    "  --p-steps 10 \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/asv/alpha-rarefaction.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate core metric results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/asv/core-metrics-results/rarefied_table.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/asv/core-metrics-results/faith_pd_vector.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/asv/core-metrics-results/observed_features_vector.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/asv/core-metrics-results/shannon_vector.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/asv/core-metrics-results/evenness_vector.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/asv/core-metrics-results/unweighted_unifrac_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/asv/core-metrics-results/weighted_unifrac_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/asv/core-metrics-results/jaccard_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/asv/core-metrics-results/bray_curtis_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/asv/core-metrics-results/unweighted_unifrac_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/asv/core-metrics-results/weighted_unifrac_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/asv/core-metrics-results/jaccard_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/asv/core-metrics-results/bray_curtis_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/core-metrics-results/unweighted_unifrac_emperor.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/core-metrics-results/weighted_unifrac_emperor.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/core-metrics-results/jaccard_emperor.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/core-metrics-results/bray_curtis_emperor.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime diversity core-metrics-phylogenetic \\\n",
    "  --i-phylogeny data/intermediate/qiime2/asv/insertion-tree.qza \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered-sepp-inserted-no-control.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --p-sampling-depth 10532 \\\n",
    "  --output-dir data/intermediate/qiime2/asv/core-metrics-results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare beta-diversity using robust Aitchison PCA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved PCoAResults % Properties('biplot') to: data/intermediate/qiime2/asv/robust-Aitchison-pca/biplot.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/asv/robust-Aitchison-pca/distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/asv/robust-Aitchison-pca/biplot.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime deicode rpca \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered-sepp-inserted-no-control.qza \\\n",
    "  --p-min-feature-count 10 \\\n",
    "  --p-min-sample-count 1000 \\\n",
    "  --output-dir data/intermediate/qiime2/asv/robust-Aitchison-pca\n",
    "\n",
    "!qiime emperor biplot \\\n",
    "  --i-biplot data/intermediate/qiime2/asv/robust-Aitchison-pca/biplot.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --m-feature-metadata-file data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --o-visualization data/intermediate/qiime2/asv/robust-Aitchison-pca/biplot.qzv \\\n",
    "  --p-number-of-features 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyzing sequences at OTU level: 99% OTU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cluster sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### De novo OTU picking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/99otu/table-filtered-99otu.qza\u001b[0m\n",
      "\u001b[32mSaved FeatureData[Sequence] to: data/intermediate/qiime2/99otu/rep-seqs-filtered-99otu.qza\u001b[0m\n",
      "CPU times: user 114 ms, sys: 35.1 ms, total: 150 ms\n",
      "Wall time: 6.66 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!qiime vsearch cluster-features-de-novo \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered.qza \\\n",
    "  --i-sequences data/intermediate/qiime2/asv/rep-seqs-filtered.qza \\\n",
    "  --p-perc-identity 0.99 \\\n",
    "  --p-threads 16 \\\n",
    "  --o-clustered-table data/intermediate/qiime2/99otu/table-filtered-99otu.qza \\\n",
    "  --o-clustered-sequences data/intermediate/qiime2/99otu/rep-seqs-filtered-99otu.qza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize the clustered feature table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/table-filtered-99otu.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/taxa-bar-plots-filtered-99otu.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Feature table summary \n",
    "!qiime feature-table summarize \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/99otu/table-filtered-99otu.qzv \n",
    "\n",
    "# Taxonomic barplot \n",
    "!qiime taxa barplot \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu.qza \\\n",
    "  --i-taxonomy data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/99otu/taxa-bar-plots-filtered-99otu.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Phylogeny"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reference-based fragment insertion with SEPP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Phylogeny[Rooted] to: data/intermediate/qiime2/99otu/insertion-tree-99otu.qza\u001b[0m\n",
      "\u001b[32mSaved Placements to: data/intermediate/qiime2/99otu/tree-placements-99otu.qza\u001b[0m\n",
      "CPU times: user 1min 16s, sys: 12.9 s, total: 1min 29s\n",
      "Wall time: 1h 1min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!qiime fragment-insertion sepp \\\n",
    "  --i-representative-sequences data/intermediate/qiime2/99otu/rep-seqs-filtered-99otu.qza \\\n",
    "  --i-reference-database data/reference/sepp-refs-silva-128.qza \\\n",
    "  --o-tree data/intermediate/qiime2/99otu/insertion-tree-99otu.qza \\\n",
    "  --o-placements data/intermediate/qiime2/99otu/tree-placements-99otu.qza \\\n",
    "  --p-threads 16 \\\n",
    "  --p-debug"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter uninserted sequences from the feature table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted.qza\u001b[0m\r\n",
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-uninserted.qza\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime fragment-insertion filter-features \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu.qza \\\n",
    "  --i-tree data/intermediate/qiime2/99otu/insertion-tree-99otu.qza \\\n",
    "  --o-filtered-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted.qza \\\n",
    "  --o-removed-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-uninserted.qza \\\n",
    "  --verbose"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature table summary "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-uninserted.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime feature-table summarize \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted.qzv \n",
    "\n",
    "!qiime feature-table summarize \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-uninserted.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-uninserted.qzv "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Quality control: taxonomic composition of mock samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the observed taxonomic composition of mock samples "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/99otu/quality-control/mock-observed.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/quality-control/mock-observed.qzv\u001b[0m\n",
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/99otu/quality-control/mock-observed-l7.qza\u001b[0m\n",
      "\u001b[32mSaved FeatureTable[RelativeFrequency] to: data/intermediate/qiime2/99otu/quality-control/mock-observed-l7-rel.qza\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Subset mock samples\n",
    "!qiime feature-table filter-samples \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --p-where \"SampleType='Mock'\" \\\n",
    "  --p-no-exclude-ids \\\n",
    "  --o-filtered-table data/intermediate/qiime2/99otu/quality-control/mock-observed.qza\n",
    "\n",
    "# Inspect the taxonomic composition of mock samples at OTU level\n",
    "!qiime taxa barplot \\\n",
    "  --i-table data/intermediate/qiime2/99otu/quality-control/mock-observed.qza \\\n",
    "  --i-taxonomy data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/99otu/quality-control/mock-observed.qzv\n",
    "\n",
    "# Agglomerate taxa at species level\n",
    "!qiime taxa collapse \\\n",
    "  --i-table data/intermediate/qiime2/99otu/quality-control/mock-observed.qza \\\n",
    "  --i-taxonomy data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --p-level 7 \\\n",
    "  --o-collapsed-table data/intermediate/qiime2/99otu/quality-control/mock-observed-l7.qza\n",
    "\n",
    "# Convert sequence counts into relative abundances\n",
    "!qiime feature-table relative-frequency \\\n",
    "  --i-table data/intermediate/qiime2/99otu/quality-control/mock-observed-l7.qza \\\n",
    "  --o-relative-frequency-table data/intermediate/qiime2/99otu/quality-control/mock-observed-l7-rel.qza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare observed and expected taxonomic composition of mock samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/quality-control/mock-comparison.qzv\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime quality-control evaluate-composition \\\n",
    "  --i-expected-features data/reference/mock-expected.qza \\\n",
    "  --i-observed-features data/intermediate/qiime2/99otu/quality-control/mock-observed-l7-rel.qza \\\n",
    "  --o-visualization data/intermediate/qiime2/99otu/quality-control/mock-comparison.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alpha and beta diversity analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exclude mock and negative control samples from the feature table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted-no-control.qza\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime feature-table filter-samples \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --p-where \"SampleType IN ('Mock', 'Extraction-blank', 'PCR-blank')\" \\\n",
    "  --p-exclude-ids \\\n",
    "  --o-filtered-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted-no-control.qza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rarefaction analysis "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/alpha-rarefaction-99otu.qzv\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime diversity alpha-rarefaction \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted-no-control.qza \\\n",
    "  --i-phylogeny data/intermediate/qiime2/99otu/insertion-tree-99otu.qza \\\n",
    "  --p-max-depth 10532 \\\n",
    "  --p-steps 10 \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/99otu/alpha-rarefaction-99otu.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate core metric results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/99otu/core-metrics-results/rarefied_table.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/99otu/core-metrics-results/faith_pd_vector.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/99otu/core-metrics-results/observed_features_vector.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/99otu/core-metrics-results/shannon_vector.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/99otu/core-metrics-results/evenness_vector.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/99otu/core-metrics-results/unweighted_unifrac_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/99otu/core-metrics-results/weighted_unifrac_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/99otu/core-metrics-results/jaccard_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/99otu/core-metrics-results/bray_curtis_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/99otu/core-metrics-results/unweighted_unifrac_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/99otu/core-metrics-results/weighted_unifrac_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/99otu/core-metrics-results/jaccard_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/99otu/core-metrics-results/bray_curtis_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/core-metrics-results/unweighted_unifrac_emperor.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/core-metrics-results/weighted_unifrac_emperor.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/core-metrics-results/jaccard_emperor.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/core-metrics-results/bray_curtis_emperor.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime diversity core-metrics-phylogenetic \\\n",
    "  --i-phylogeny data/intermediate/qiime2/99otu/insertion-tree-99otu.qza \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted-no-control.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --p-sampling-depth 10532 \\\n",
    "  --output-dir data/intermediate/qiime2/99otu/core-metrics-results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare beta-diversity using robust Aitchison PCA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved PCoAResults % Properties('biplot') to: data/intermediate/qiime2/99otu/robust-Aitchison-pca/biplot.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/99otu/robust-Aitchison-pca/distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/99otu/robust-Aitchison-pca/biplot.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime deicode rpca \\\n",
    "  --i-table data/intermediate/qiime2/99otu/table-filtered-99otu-sepp-inserted-no-control.qza \\\n",
    "  --p-min-feature-count 10 \\\n",
    "  --p-min-sample-count 1000 \\\n",
    "  --output-dir data/intermediate/qiime2/99otu/robust-Aitchison-pca\n",
    "\n",
    "!qiime emperor biplot \\\n",
    "  --i-biplot data/intermediate/qiime2/99otu/robust-Aitchison-pca/biplot.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --m-feature-metadata-file data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --o-visualization data/intermediate/qiime2/99otu/robust-Aitchison-pca/biplot.qzv \\\n",
    "  --p-number-of-features 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyzing sequences at OTU level: 97% OTU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cluster sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### De novo OTU picking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/97otu/table-filtered-97otu.qza\u001b[0m\n",
      "\u001b[32mSaved FeatureData[Sequence] to: data/intermediate/qiime2/97otu/rep-seqs-filtered-97otu.qza\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime vsearch cluster-features-de-novo \\\n",
    "  --i-table data/intermediate/qiime2/asv/table-filtered.qza \\\n",
    "  --i-sequences data/intermediate/qiime2/asv/rep-seqs-filtered.qza \\\n",
    "  --p-perc-identity 0.97 \\\n",
    "  --p-threads 16 \\\n",
    "  --o-clustered-table data/intermediate/qiime2/97otu/table-filtered-97otu.qza \\\n",
    "  --o-clustered-sequences data/intermediate/qiime2/97otu/rep-seqs-filtered-97otu.qza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize the clustered feature table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/table-filtered-97otu.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/taxa-bar-plots-filtered-97otu.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Feature table summary \n",
    "!qiime feature-table summarize \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/97otu/table-filtered-97otu.qzv \n",
    "\n",
    "# Taxonomic barplot \n",
    "!qiime taxa barplot \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu.qza \\\n",
    "  --i-taxonomy data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/97otu/taxa-bar-plots-filtered-97otu.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Phylogeny"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reference-based fragment insertion with SEPP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Phylogeny[Rooted] to: data/intermediate/qiime2/97otu/insertion-tree-97otu.qza\u001b[0m\n",
      "\u001b[32mSaved Placements to: data/intermediate/qiime2/97otu/tree-placements-97otu.qza\u001b[0m\n",
      "CPU times: user 1min 7s, sys: 11.6 s, total: 1min 18s\n",
      "Wall time: 54min 30s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!qiime fragment-insertion sepp \\\n",
    "  --i-representative-sequences data/intermediate/qiime2/97otu/rep-seqs-filtered-97otu.qza \\\n",
    "  --i-reference-database data/reference/sepp-refs-silva-128.qza \\\n",
    "  --o-tree data/intermediate/qiime2/97otu/insertion-tree-97otu.qza \\\n",
    "  --o-placements data/intermediate/qiime2/97otu/tree-placements-97otu.qza \\\n",
    "  --p-threads 16 \\\n",
    "  --p-debug"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter uninserted sequences from the feature table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted.qza\u001b[0m\n",
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-uninserted.qza\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime fragment-insertion filter-features \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu.qza \\\n",
    "  --i-tree data/intermediate/qiime2/97otu/insertion-tree-97otu.qza \\\n",
    "  --o-filtered-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted.qza \\\n",
    "  --o-removed-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-uninserted.qza \\\n",
    "  --verbose"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature table summary "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-uninserted.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime feature-table summarize \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted.qzv \n",
    "\n",
    "!qiime feature-table summarize \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-uninserted.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-uninserted.qzv "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Quality control: taxonomic composition of mock samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the observed taxonomic composition of mock samples "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/97otu/quality-control/mock-observed.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/quality-control/mock-observed.qzv\u001b[0m\n",
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/97otu/quality-control/mock-observed-l7.qza\u001b[0m\n",
      "\u001b[32mSaved FeatureTable[RelativeFrequency] to: data/intermediate/qiime2/97otu/quality-control/mock-observed-l7-rel.qza\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Subset mock samples\n",
    "!qiime feature-table filter-samples \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --p-where \"SampleType='Mock'\" \\\n",
    "  --p-no-exclude-ids \\\n",
    "  --o-filtered-table data/intermediate/qiime2/97otu/quality-control/mock-observed.qza\n",
    "\n",
    "# Inspect the taxonomic composition of mock samples at OTU level\n",
    "!qiime taxa barplot \\\n",
    "  --i-table data/intermediate/qiime2/97otu/quality-control/mock-observed.qza \\\n",
    "  --i-taxonomy data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/97otu/quality-control/mock-observed.qzv\n",
    "\n",
    "# Agglomerate taxa at species level\n",
    "!qiime taxa collapse \\\n",
    "  --i-table data/intermediate/qiime2/97otu/quality-control/mock-observed.qza \\\n",
    "  --i-taxonomy data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --p-level 7 \\\n",
    "  --o-collapsed-table data/intermediate/qiime2/97otu/quality-control/mock-observed-l7.qza\n",
    "\n",
    "# Convert sequence counts into relative abundances\n",
    "!qiime feature-table relative-frequency \\\n",
    "  --i-table data/intermediate/qiime2/97otu/quality-control/mock-observed-l7.qza \\\n",
    "  --o-relative-frequency-table data/intermediate/qiime2/97otu/quality-control/mock-observed-l7-rel.qza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare observed and expected taxonomic composition of mock samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/quality-control/mock-comparison.qzv\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime quality-control evaluate-composition \\\n",
    "  --i-expected-features data/reference/mock-expected.qza \\\n",
    "  --i-observed-features data/intermediate/qiime2/97otu/quality-control/mock-observed-l7-rel.qza \\\n",
    "  --o-visualization data/intermediate/qiime2/97otu/quality-control/mock-comparison.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alpha and beta diversity analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exclude mock and negative control samples from the feature table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted-no-control.qza\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime feature-table filter-samples \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --p-where \"SampleType IN ('Mock', 'Extraction-blank', 'PCR-blank')\" \\\n",
    "  --p-exclude-ids \\\n",
    "  --o-filtered-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted-no-control.qza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rarefaction analysis "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/alpha-rarefaction-97otu.qzv\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!qiime diversity alpha-rarefaction \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted-no-control.qza \\\n",
    "  --i-phylogeny data/intermediate/qiime2/97otu/insertion-tree-97otu.qza \\\n",
    "  --p-max-depth 10532 \\\n",
    "  --p-steps 10 \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --o-visualization data/intermediate/qiime2/97otu/alpha-rarefaction-97otu.qzv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate core metric results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved FeatureTable[Frequency] to: data/intermediate/qiime2/97otu/core-metrics-results/rarefied_table.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/97otu/core-metrics-results/faith_pd_vector.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/97otu/core-metrics-results/observed_features_vector.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/97otu/core-metrics-results/shannon_vector.qza\u001b[0m\n",
      "\u001b[32mSaved SampleData[AlphaDiversity] to: data/intermediate/qiime2/97otu/core-metrics-results/evenness_vector.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/97otu/core-metrics-results/unweighted_unifrac_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/97otu/core-metrics-results/weighted_unifrac_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/97otu/core-metrics-results/jaccard_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/97otu/core-metrics-results/bray_curtis_distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/97otu/core-metrics-results/unweighted_unifrac_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/97otu/core-metrics-results/weighted_unifrac_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/97otu/core-metrics-results/jaccard_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved PCoAResults to: data/intermediate/qiime2/97otu/core-metrics-results/bray_curtis_pcoa_results.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/core-metrics-results/unweighted_unifrac_emperor.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/core-metrics-results/weighted_unifrac_emperor.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/core-metrics-results/jaccard_emperor.qzv\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/core-metrics-results/bray_curtis_emperor.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime diversity core-metrics-phylogenetic \\\n",
    "  --i-phylogeny data/intermediate/qiime2/97otu/insertion-tree-97otu.qza \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted-no-control.qza \\\n",
    "  --m-metadata-file data/metadata.tsv \\\n",
    "  --p-sampling-depth 10532 \\\n",
    "  --output-dir data/intermediate/qiime2/97otu/core-metrics-results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare beta-diversity using robust Aitchison PCA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mSaved PCoAResults % Properties('biplot') to: data/intermediate/qiime2/97otu/robust-Aitchison-pca/biplot.qza\u001b[0m\n",
      "\u001b[32mSaved DistanceMatrix to: data/intermediate/qiime2/97otu/robust-Aitchison-pca/distance_matrix.qza\u001b[0m\n",
      "\u001b[32mSaved Visualization to: data/intermediate/qiime2/97otu/robust-Aitchison-pca/biplot.qzv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!qiime deicode rpca \\\n",
    "  --i-table data/intermediate/qiime2/97otu/table-filtered-97otu-sepp-inserted-no-control.qza \\\n",
    "  --p-min-feature-count 10 \\\n",
    "  --p-min-sample-count 1000 \\\n",
    "  --output-dir data/intermediate/qiime2/97otu/robust-Aitchison-pca\n",
    "\n",
    "!qiime emperor biplot \\\n",
    "  --i-biplot data/intermediate/qiime2/97otu/robust-Aitchison-pca/biplot.qza \\\n",
    "  --m-sample-metadata-file data/metadata.tsv \\\n",
    "  --m-feature-metadata-file data/intermediate/qiime2/asv/taxonomy-silva132.qza \\\n",
    "  --o-visualization data/intermediate/qiime2/97otu/robust-Aitchison-pca/biplot.qzv \\\n",
    "  --p-number-of-features 8"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}