LDA_steam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import gensim\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.parsing.preprocessing import STOPWORDS\n",
    "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
    "from nltk.stem.porter import *\n",
    "import numpy as np\n",
    "import ssl\n",
    "import nltk\n",
    "import json\n",
    "import os\n",
    "from gensim import corpora, models\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(2018)\n",
    "\n",
    "data = pd.read_csv('abcnews-date-text.csv', error_bad_lines=False);\n",
    "data_text = data[['headline_text']]\n",
    "data_text['index'] = data_text.index\n",
    "documents = data_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/Users/muskansinghal/Desktop/Reddit/Literature.json', '/Users/muskansinghal/Desktop/Reddit/Audience.json', '/Users/muskansinghal/Desktop/Reddit/Rehearsal.json', '/Users/muskansinghal/Desktop/Reddit/Valence Electrons.json', '/Users/muskansinghal/Desktop/Reddit/RNA.json', '/Users/muskansinghal/Desktop/Reddit/Imagination.json', '/Users/muskansinghal/Desktop/Reddit/Earthquakes.json', '/Users/muskansinghal/Desktop/Reddit/Erosion.json', '/Users/muskansinghal/Desktop/Reddit/Hormones.json', '/Users/muskansinghal/Desktop/Reddit/Organs.json', '/Users/muskansinghal/Desktop/Reddit/Line.json', '/Users/muskansinghal/Desktop/Reddit/Particle.json', '/Users/muskansinghal/Desktop/Reddit/Friction.json', '/Users/muskansinghal/Desktop/Reddit/Percentages.json', '/Users/muskansinghal/Desktop/Reddit/Division.json', '/Users/muskansinghal/Desktop/Reddit/Muscle.json', '/Users/muskansinghal/Desktop/Reddit/Population.json', '/Users/muskansinghal/Desktop/Reddit/Music.json', '/Users/muskansinghal/Desktop/Reddit/Numerator.json', '/Users/muskansinghal/Desktop/Reddit/Geometry.json', '/Users/muskansinghal/Desktop/Reddit/Chi-squared Test.json', '/Users/muskansinghal/Desktop/Reddit/Logarithms.json', '/Users/muskansinghal/Desktop/Reddit/Blood.json', '/Users/muskansinghal/Desktop/Reddit/Circumference.json', '/Users/muskansinghal/Desktop/Reddit/Regression.json', '/Users/muskansinghal/Desktop/Reddit/Electrons.json', '/Users/muskansinghal/Desktop/Reddit/Metal.json', '/Users/muskansinghal/Desktop/Reddit/Acids.json', '/Users/muskansinghal/Desktop/Reddit/Staging.json', '/Users/muskansinghal/Desktop/Reddit/Calcium.json', '/Users/muskansinghal/Desktop/Reddit/Space.json', '/Users/muskansinghal/Desktop/Reddit/Pressure.json', '/Users/muskansinghal/Desktop/Reddit/Momentum.json', '/Users/muskansinghal/Desktop/Reddit/Cell.json', '/Users/muskansinghal/Desktop/Reddit/Eukaryotic.json', '/Users/muskansinghal/Desktop/Reddit/Membrane.json', '/Users/muskansinghal/Desktop/Reddit/Motion.json', '/Users/muskansinghal/Desktop/Reddit/Sea.json', '/Users/muskansinghal/Desktop/Reddit/Species.json', '/Users/muskansinghal/Desktop/Reddit/Acceleration.json', '/Users/muskansinghal/Desktop/Reddit/Positive.json', '/Users/muskansinghal/Desktop/Reddit/Equal.json', '/Users/muskansinghal/Desktop/Reddit/Plane.json', '/Users/muskansinghal/Desktop/Reddit/Volume.json', '/Users/muskansinghal/Desktop/Reddit/Frequency.json', '/Users/muskansinghal/Desktop/Reddit/Voltage.json', '/Users/muskansinghal/Desktop/Reddit/P-value.json', '/Users/muskansinghal/Desktop/Reddit/Square.json', '/Users/muskansinghal/Desktop/Reddit/Weight.json', '/Users/muskansinghal/Desktop/Reddit/Exponents.json', '/Users/muskansinghal/Desktop/Reddit/Plants.json', '/Users/muskansinghal/Desktop/Reddit/Phrases.json', '/Users/muskansinghal/Desktop/Reddit/Universe.json', '/Users/muskansinghal/Desktop/Reddit/Histogram.json', '/Users/muskansinghal/Desktop/Reddit/Neutrons.json', '/Users/muskansinghal/Desktop/Reddit/CO2.json', '/Users/muskansinghal/Desktop/Reddit/Dependent Variable.json', '/Users/muskansinghal/Desktop/Reddit/Reproduction.json', '/Users/muskansinghal/Desktop/Reddit/Negative.json', '/Users/muskansinghal/Desktop/Reddit/Exterior Angle.json', '/Users/muskansinghal/Desktop/Reddit/Self-reflection.json', '/Users/muskansinghal/Desktop/Reddit/Script.json', '/Users/muskansinghal/Desktop/Reddit/Essay.json', '/Users/muskansinghal/Desktop/Reddit/Solid.json', '/Users/muskansinghal/Desktop/Reddit/Molar.json', '/Users/muskansinghal/Desktop/Reddit/Solubility.json', '/Users/muskansinghal/Desktop/Reddit/Boiling Point.json', '/Users/muskansinghal/Desktop/Reddit/Vocabulary.json', '/Users/muskansinghal/Desktop/Reddit/Two Sample z-test.json', '/Users/muskansinghal/Desktop/Reddit/Equilateral Triangle.json', '/Users/muskansinghal/Desktop/Reddit/Contemporary Art.json', '/Users/muskansinghal/Desktop/Reddit/Embryo.json', '/Users/muskansinghal/Desktop/Reddit/Oxygen.json', '/Users/muskansinghal/Desktop/Reddit/Literacy.json', '/Users/muskansinghal/Desktop/Reddit/Tissues.json', '/Users/muskansinghal/Desktop/Reddit/Wavelength.json', '/Users/muskansinghal/Desktop/Reddit/Genres.json', '/Users/muskansinghal/Desktop/Reddit/kg.json', '/Users/muskansinghal/Desktop/Reddit/Poetry.json', '/Users/muskansinghal/Desktop/Reddit/Glucose.json', '/Users/muskansinghal/Desktop/Reddit/DNA.json', '/Users/muskansinghal/Desktop/Reddit/Cells.json', '/Users/muskansinghal/Desktop/Reddit/Percentage.json', '/Users/muskansinghal/Desktop/Reddit/Ocean.json', '/Users/muskansinghal/Desktop/Reddit/Contemporary.json', '/Users/muskansinghal/Desktop/Reddit/Nucleus.json', '/Users/muskansinghal/Desktop/Reddit/Chemistry.json', '/Users/muskansinghal/Desktop/Reddit/Perform.json', '/Users/muskansinghal/Desktop/Reddit/Atomic.json', '/Users/muskansinghal/Desktop/Reddit/Sum of Least Squares.json', '/Users/muskansinghal/Desktop/Reddit/Polynomials.json', '/Users/muskansinghal/Desktop/Reddit/Organism.json', '/Users/muskansinghal/Desktop/Reddit/Gas.json', '/Users/muskansinghal/Desktop/Reddit/Polynomial.json', '/Users/muskansinghal/Desktop/Reddit/Fungi.json', '/Users/muskansinghal/Desktop/Reddit/Y-intercept.json', '/Users/muskansinghal/Desktop/Reddit/Equilibrium.json', '/Users/muskansinghal/Desktop/Reddit/Reading.json', '/Users/muskansinghal/Desktop/Reddit/Coordinate.json', '/Users/muskansinghal/Desktop/Reddit/Humans.json', '/Users/muskansinghal/Desktop/Reddit/Fish.json', '/Users/muskansinghal/Desktop/Reddit/Production.json', '/Users/muskansinghal/Desktop/Reddit/Matter.json', '/Users/muskansinghal/Desktop/Reddit/Boxplot.json', '/Users/muskansinghal/Desktop/Reddit/Microscopic.json', '/Users/muskansinghal/Desktop/Reddit/Chromosomes.json', '/Users/muskansinghal/Desktop/Reddit/Amino Acid.json', '/Users/muskansinghal/Desktop/Reddit/Triangular Prism.json', '/Users/muskansinghal/Desktop/Reddit/Marine.json', '/Users/muskansinghal/Desktop/Reddit/Sulfur.json', '/Users/muskansinghal/Desktop/Reddit/Algebra.json', '/Users/muskansinghal/Desktop/Reddit/Organisms.json', '/Users/muskansinghal/Desktop/Reddit/Infinite.json', '/Users/muskansinghal/Desktop/Reddit/Interior Angle.json', '/Users/muskansinghal/Desktop/Reddit/Earth.json', '/Users/muskansinghal/Desktop/Reddit/Parallel.json', '/Users/muskansinghal/Desktop/Reddit/Control Variable.json', '/Users/muskansinghal/Desktop/Reddit/Random Sampling.json', '/Users/muskansinghal/Desktop/Reddit/Limits.json', '/Users/muskansinghal/Desktop/Reddit/Random Distribution.json', '/Users/muskansinghal/Desktop/Reddit/Perspectives.json', '/Users/muskansinghal/Desktop/Reddit/Ions.json', '/Users/muskansinghal/Desktop/Reddit/Fusion.json', '/Users/muskansinghal/Desktop/Reddit/Whole Numbers.json', '/Users/muskansinghal/Desktop/Reddit/Magnetic.json', '/Users/muskansinghal/Desktop/Reddit/Communication.json', '/Users/muskansinghal/Desktop/Reddit/Treatment Variable.json', '/Users/muskansinghal/Desktop/Reddit/Amino acids.json', '/Users/muskansinghal/Desktop/Reddit/Quadratic.json', '/Users/muskansinghal/Desktop/Reddit/Chloride.json', '/Users/muskansinghal/Desktop/Reddit/Oxidation.json', '/Users/muskansinghal/Desktop/Reddit/Bisector.json', '/Users/muskansinghal/Desktop/Reddit/Ethics.json', '/Users/muskansinghal/Desktop/Reddit/Mammals.json', '/Users/muskansinghal/Desktop/Reddit/Carbon.json', '/Users/muskansinghal/Desktop/Reddit/Residual.json', '/Users/muskansinghal/Desktop/Reddit/Rectangle.json', '/Users/muskansinghal/Desktop/Reddit/Speed.json', '/Users/muskansinghal/Desktop/Reddit/Bisect.json', '/Users/muskansinghal/Desktop/Reddit/Genetic.json', '/Users/muskansinghal/Desktop/Reddit/Area.json', '/Users/muskansinghal/Desktop/Reddit/Elements.json', '/Users/muskansinghal/Desktop/Reddit/History.json', '/Users/muskansinghal/Desktop/Reddit/Biotechnology.json', '/Users/muskansinghal/Desktop/Reddit/Illustration.json', '/Users/muskansinghal/Desktop/Reddit/Chemical.json', '/Users/muskansinghal/Desktop/Reddit/Binomial.json', '/Users/muskansinghal/Desktop/Reddit/Circle.json', '/Users/muskansinghal/Desktop/Reddit/Coordinates.json', '/Users/muskansinghal/Desktop/Reddit/Obtuse Angle.json', '/Users/muskansinghal/Desktop/Reddit/Compounds.json', '/Users/muskansinghal/Desktop/Reddit/Variability.json', '/Users/muskansinghal/Desktop/Reddit/Fractions.json', '/Users/muskansinghal/Desktop/Reddit/Waves.json', '/Users/muskansinghal/Desktop/Reddit/Vertebrates.json', '/Users/muskansinghal/Desktop/Reddit/Gene.json', '/Users/muskansinghal/Desktop/Reddit/Figurative Speech.json', '/Users/muskansinghal/Desktop/Reddit/Hypotenuse.json', '/Users/muskansinghal/Desktop/Reddit/Cells genetic.json', '/Users/muskansinghal/Desktop/Reddit/Probabilities.json', '/Users/muskansinghal/Desktop/Reddit/Acute Angle.json', '/Users/muskansinghal/Desktop/Reddit/Vocation.json', '/Users/muskansinghal/Desktop/Reddit/Statistics.json', '/Users/muskansinghal/Desktop/Reddit/Means.json', '/Users/muskansinghal/Desktop/Reddit/Molecular.json', '/Users/muskansinghal/Desktop/Reddit/Nonfiction.json', '/Users/muskansinghal/Desktop/Reddit/Water.json', '/Users/muskansinghal/Desktop/Reddit/95% Confidence Interval.json', '/Users/muskansinghal/Desktop/Reddit/Microbial.json', '/Users/muskansinghal/Desktop/Reddit/Technology.json', '/Users/muskansinghal/Desktop/Reddit/Protein.json', '/Users/muskansinghal/Desktop/Reddit/X-intercept.json', '/Users/muskansinghal/Desktop/Reddit/Segments.json', '/Users/muskansinghal/Desktop/Reddit/Molecules.json', '/Users/muskansinghal/Desktop/Reddit/Hypotheses.json', '/Users/muskansinghal/Desktop/Reddit/Sodium.json', '/Users/muskansinghal/Desktop/Reddit/Character.json', '/Users/muskansinghal/Desktop/Reddit/Rhombus.json', '/Users/muskansinghal/Desktop/Reddit/Subtraction.json', '/Users/muskansinghal/Desktop/Reddit/Minimum.json', '/Users/muskansinghal/Desktop/Reddit/Secant.json', '/Users/muskansinghal/Desktop/Reddit/Graphs.json', '/Users/muskansinghal/Desktop/Reddit/Quadratic Equations.json', '/Users/muskansinghal/Desktop/Reddit/Express.json', '/Users/muskansinghal/Desktop/Reddit/Denominator.json', '/Users/muskansinghal/Desktop/Reddit/Midpoint.json', '/Users/muskansinghal/Desktop/Reddit/Hypothesis.json', '/Users/muskansinghal/Desktop/Reddit/Electron.json', '/Users/muskansinghal/Desktop/Reddit/Mass.json', '/Users/muskansinghal/Desktop/Reddit/Earthquake.json', '/Users/muskansinghal/Desktop/Reddit/Flow.json', '/Users/muskansinghal/Desktop/Reddit/Light.json', '/Users/muskansinghal/Desktop/Reddit/Cultural.json', '/Users/muskansinghal/Desktop/Reddit/Sum.json', '/Users/muskansinghal/Desktop/Reddit/Fantasy.json', '/Users/muskansinghal/Desktop/Reddit/Playwright.json', '/Users/muskansinghal/Desktop/Reddit/Emotional.json', '/Users/muskansinghal/Desktop/Reddit/Electric.json', '/Users/muskansinghal/Desktop/Reddit/Energy.json', '/Users/muskansinghal/Desktop/Reddit/Plant.json', '/Users/muskansinghal/Desktop/Reddit/Bacterial.json', '/Users/muskansinghal/Desktop/Reddit/Cancer.json', '/Users/muskansinghal/Desktop/Reddit/Radius.json', '/Users/muskansinghal/Desktop/Reddit/Ionization.json', '/Users/muskansinghal/Desktop/Reddit/Finite.json', '/Users/muskansinghal/Desktop/Reddit/Particles.json', '/Users/muskansinghal/Desktop/Reddit/Vectors.json', '/Users/muskansinghal/Desktop/Reddit/Author.json', '/Users/muskansinghal/Desktop/Reddit/Kinetic.json', '/Users/muskansinghal/Desktop/Reddit/Integers.json', '/Users/muskansinghal/Desktop/Reddit/Multimedia.json', '/Users/muskansinghal/Desktop/Reddit/Narrative.json', '/Users/muskansinghal/Desktop/Reddit/Poems.json', '/Users/muskansinghal/Desktop/Reddit/Drama.json', '/Users/muskansinghal/Desktop/Reddit/Plasma.json', '/Users/muskansinghal/Desktop/Reddit/Length.json', '/Users/muskansinghal/Desktop/Reddit/Mutations.json', '/Users/muskansinghal/Desktop/Reddit/Perpendicular.json', '/Users/muskansinghal/Desktop/Reddit/Cycles.json', '/Users/muskansinghal/Desktop/Reddit/Triangle.json', '/Users/muskansinghal/Desktop/Reddit/Orbitals.json', '/Users/muskansinghal/Desktop/Reddit/Statistic.json', '/Users/muskansinghal/Desktop/Reddit/Absolute Value.json', '/Users/muskansinghal/Desktop/Reddit/Textual.json', '/Users/muskansinghal/Desktop/Reddit/Ionic.json', '/Users/muskansinghal/Desktop/Reddit/Animals.json', '/Users/muskansinghal/Desktop/Reddit/Electricity.json', '/Users/muskansinghal/Desktop/Reddit/Hydrogen.json', '/Users/muskansinghal/Desktop/Reddit/Independent Variable.json', '/Users/muskansinghal/Desktop/Reddit/Sample Size.json', '/Users/muskansinghal/Desktop/Reddit/Books.json', '/Users/muskansinghal/Desktop/Reddit/Temperature.json', '/Users/muskansinghal/Desktop/Reddit/Culture.json', '/Users/muskansinghal/Desktop/Reddit/Vertex.json', '/Users/muskansinghal/Desktop/Reddit/Goodness-of-fit.json', '/Users/muskansinghal/Desktop/Reddit/Electromagnetic.json', '/Users/muskansinghal/Desktop/Reddit/Writing.json', '/Users/muskansinghal/Desktop/Reddit/Right Angle.json', '/Users/muskansinghal/Desktop/Reddit/Proofs.json', '/Users/muskansinghal/Desktop/Reddit/Isosceles Triangle.json', '/Users/muskansinghal/Desktop/Reddit/Matrix.json', '/Users/muskansinghal/Desktop/Reddit/Wave.json', '/Users/muskansinghal/Desktop/Reddit/Experiment.json', '/Users/muskansinghal/Desktop/Reddit/Trinomial.json', '/Users/muskansinghal/Desktop/Reddit/Magnesium.json', '/Users/muskansinghal/Desktop/Reddit/Interval Proportion.json', '/Users/muskansinghal/Desktop/Reddit/Biology.json', '/Users/muskansinghal/Desktop/Reddit/Force.json', '/Users/muskansinghal/Desktop/Reddit/Linear Function.json', '/Users/muskansinghal/Desktop/Reddit/Vertices.json', '/Users/muskansinghal/Desktop/Reddit/Physics.json', '/Users/muskansinghal/Desktop/Reddit/Pythagorean Theorem.json', '/Users/muskansinghal/Desktop/Reddit/Probability.json', '/Users/muskansinghal/Desktop/Reddit/Distance.json', '/Users/muskansinghal/Desktop/Reddit/Proteins.json', '/Users/muskansinghal/Desktop/Reddit/Global Warming.json', '/Users/muskansinghal/Desktop/Reddit/Lines.json', '/Users/muskansinghal/Desktop/Reddit/Charge.json', '/Users/muskansinghal/Desktop/Reddit/Essays.json', '/Users/muskansinghal/Desktop/Reddit/Square Root.json', '/Users/muskansinghal/Desktop/Reddit/Creative.json', '/Users/muskansinghal/Desktop/Reddit/Point.json', '/Users/muskansinghal/Desktop/Reddit/Gravity.json', '/Users/muskansinghal/Desktop/Reddit/Chlorine.json', '/Users/muskansinghal/Desktop/Reddit/Atoms.json', '/Users/muskansinghal/Desktop/Reddit/Nuclear Energy.json', '/Users/muskansinghal/Desktop/Reddit/Chromosome.json', '/Users/muskansinghal/Desktop/Reddit/Correlation.json', '/Users/muskansinghal/Desktop/Reddit/Words.json', '/Users/muskansinghal/Desktop/Reddit/Aluminum.json', '/Users/muskansinghal/Desktop/Reddit/Torque.json', '/Users/muskansinghal/Desktop/Reddit/Fluid.json', '/Users/muskansinghal/Desktop/Reddit/Life.json', '/Users/muskansinghal/Desktop/Reddit/Mathematics.json', '/Users/muskansinghal/Desktop/Reddit/Average.json', '/Users/muskansinghal/Desktop/Reddit/Biochemical.json', '/Users/muskansinghal/Desktop/Reddit/Nuclear.json', '/Users/muskansinghal/Desktop/Reddit/Sound.json', '/Users/muskansinghal/Desktop/Reddit/Vapor.json', '/Users/muskansinghal/Desktop/Reddit/Quadrilaterals.json', '/Users/muskansinghal/Desktop/Reddit/Thermal Energy.json', '/Users/muskansinghal/Desktop/Reddit/Standard Error.json', '/Users/muskansinghal/Desktop/Reddit/Coefficient.json', '/Users/muskansinghal/Desktop/Reddit/Reciprocal.json', '/Users/muskansinghal/Desktop/Reddit/Liquid.json', '/Users/muskansinghal/Desktop/Reddit/Scatterplot.json', '/Users/muskansinghal/Desktop/Reddit/Radiation.json', '/Users/muskansinghal/Desktop/Reddit/Fiction.json', '/Users/muskansinghal/Desktop/Reddit/Quadrilateral.json', '/Users/muskansinghal/Desktop/Reddit/Arts.json', '/Users/muskansinghal/Desktop/Reddit/Circuit.json', '/Users/muskansinghal/Desktop/Reddit/Protons.json', '/Users/muskansinghal/Desktop/Reddit/Polygons.json', '/Users/muskansinghal/Desktop/Reddit/Multiplication.json', '/Users/muskansinghal/Desktop/Reddit/Radioactive.json', '/Users/muskansinghal/Desktop/Reddit/Ecosystem.json', '/Users/muskansinghal/Desktop/Reddit/Ratio.json', '/Users/muskansinghal/Desktop/Reddit/Hydroxide.json', '/Users/muskansinghal/Desktop/Reddit/Circles.json', '/Users/muskansinghal/Desktop/Reddit/Sides.json', '/Users/muskansinghal/Desktop/Reddit/Copyright.json', '/Users/muskansinghal/Desktop/Reddit/Bacteria.json', '/Users/muskansinghal/Desktop/Reddit/Inequalities.json', '/Users/muskansinghal/Desktop/Reddit/Factors.json', '/Users/muskansinghal/Desktop/Reddit/Enthalpy.json', '/Users/muskansinghal/Desktop/Reddit/Addition.json', '/Users/muskansinghal/Desktop/Reddit/Parallelogram.json', '/Users/muskansinghal/Desktop/Reddit/Calculus.json', '/Users/muskansinghal/Desktop/Reddit/Enzymes.json', '/Users/muskansinghal/Desktop/Reddit/Fuels.json', '/Users/muskansinghal/Desktop/Reddit/Stage.json', '/Users/muskansinghal/Desktop/Reddit/Congruent.json', '/Users/muskansinghal/Desktop/Reddit/Density.json', '/Users/muskansinghal/Desktop/Reddit/Air.json', '/Users/muskansinghal/Desktop/Reddit/Maximum.json', '/Users/muskansinghal/Desktop/Reddit/Visual arts.json', '/Users/muskansinghal/Desktop/Reddit/Climate.json', '/Users/muskansinghal/Desktop/Reddit/Kinetic energy.json', '/Users/muskansinghal/Desktop/Reddit/Quantitative.json', '/Users/muskansinghal/Desktop/Reddit/Photosynthesis.json', '/Users/muskansinghal/Desktop/Reddit/Heat.json', '/Users/muskansinghal/Desktop/Reddit/Two Sample t-test.json', '/Users/muskansinghal/Desktop/Reddit/Immune System.json', '/Users/muskansinghal/Desktop/Reddit/Matrices.json', '/Users/muskansinghal/Desktop/Reddit/Velocity.json', '/Users/muskansinghal/Desktop/Reddit/Fraction.json', '/Users/muskansinghal/Desktop/Reddit/Gestures.json', '/Users/muskansinghal/Desktop/Reddit/Nitrogen.json', '/Users/muskansinghal/Desktop/Reddit/Converse.json', '/Users/muskansinghal/Desktop/Reddit/Slope.json', '/Users/muskansinghal/Desktop/Reddit/Science.json', '/Users/muskansinghal/Desktop/Reddit/Trigonometry.json', '/Users/muskansinghal/Desktop/Reddit/Nerves.json', '/Users/muskansinghal/Desktop/Reddit/Rational Integers.json', '/Users/muskansinghal/Desktop/Reddit/Mean.json', '/Users/muskansinghal/Desktop/Reddit/Genome.json', '/Users/muskansinghal/Desktop/Reddit/Tangent.json', '/Users/muskansinghal/Desktop/Reddit/Proof.json', '/Users/muskansinghal/Desktop/Reddit/Endpoints.json', '/Users/muskansinghal/Desktop/Reddit/Evolution.json', '/Users/muskansinghal/Desktop/Reddit/Standard Deviation.json', '/Users/muskansinghal/Desktop/Reddit/Stars.json', '/Users/muskansinghal/Desktop/Reddit/Photon.json', '/Users/muskansinghal/Desktop/Reddit/Polygon.json']\n"
     ]
    }
   ],
   "source": [
    "path_to_json = '/Users/muskansinghal/Desktop/Reddit/'\n",
    "\n",
    "json_files = []\n",
    "\n",
    "for file in os.listdir(path_to_json):\n",
    "    if file.endswith('.json'):\n",
    "        json_files.append(path_to_json + file)\n",
    "        \n",
    "print(json_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-136e141a9957>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfile\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mjson_files\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mjson_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjson_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/json/__init__.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(fp, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    291\u001b[0m     \u001b[0mkwarg\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0motherwise\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mJSONDecoder\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mused\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m     \"\"\"\n\u001b[0;32m--> 293\u001b[0;31m     return loads(fp.read(),\n\u001b[0m\u001b[1;32m    294\u001b[0m         \u001b[0mcls\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobject_hook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobject_hook\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m         \u001b[0mparse_float\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparse_float\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparse_int\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparse_int\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/codecs.py\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, input, final)\u001b[0m\n\u001b[1;32m    317\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m         \u001b[0;31m# decode input (taking the buffer into account)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "final_data = {}\n",
    "count = 0\n",
    "for file in json_files:\n",
    "    with open(file) as json_file:\n",
    "        data = json.load(json_file)\n",
    "        data = dict(data)\n",
    "        for i in data:\n",
    "            final_data[count]=i\n",
    "            count+=1\n",
    "            for j in data[i]:\n",
    "                final_data[count]=j\n",
    "                count+=1\n",
    "                \n",
    "documents=pd.DataFrame(final_data.items(),columns=['index','headline_text'])\n",
    "documents['headline_text']=documents['headline_text'].astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# documents=pd.DataFrame(final_data.items(),columns=['index','headline_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    _create_unverified_https_context = ssl._create_unverified_context\n",
    "except AttributeError:\n",
    "    pass\n",
    "else:\n",
    "    ssl._create_default_https_context = _create_unverified_https_context\n",
    "\n",
    "# Wordnet is an large, freely and publicly available lexical database for the English language\n",
    "# aiming to establish structured semantic relationships between words.\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lemmatization considers the context and converts the word to its meaningful base form\n",
    "# based on the context it’s used, identify the ‘part-of-speech’ (POS) tag for the word in that\n",
    "# specific context and extract the appropriate lemma.\n",
    "# j -> Adjective, v -> Verb, n -> Noun. r -> Adverb\n",
    "\n",
    "stemmer = SnowballStemmer('english')\n",
    "\n",
    "def lemmatize_stemming(text):\n",
    "    return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos = 'v'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gensim.utils.simple_preprocess - Convert a document into a list of tokens.\n",
    "# This lowercases, tokenizes, de-accents (optional)\n",
    "def preprocess(text):\n",
    "    result = []\n",
    "\n",
    "    for token in gensim.utils.simple_preprocess(text):\n",
    "\n",
    "        # all stopwords are removed and words that have fewer than 3 characters are removed\n",
    "        if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 3:\n",
    "            result.append(lemmatize_stemming(token))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "#doc_sample = \"The striped bats are hanging on their feet for best\"\n",
    "\n",
    "#print('original document: ')\n",
    "#words = []\n",
    "print(doc_sample)\n",
    "for word in doc_sample.split(' '):\n",
    "    words.append(word)\n",
    "print(words)\n",
    "print('\\n\\n tokenized and lemmatized document: ')\n",
    "print(WordNetLemmatizer().lemmatize('went', pos='v'))\n",
    "print(preprocess(doc_sample))\n",
    "'''\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "#print(lemmatizer.lemmatize('bats'))\n",
    "#print(nltk.pos_tag(nltk.word_tokenize(doc_sample)))\n",
    "\n",
    "\n",
    "\n",
    "processed_docs = documents['headline_text'].map(preprocess)\n",
    "processed_docs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary = gensim.corpora.Dictionary(processed_docs)\n",
    "dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dictionary from ‘processed_docs’ containing the number of times a word appears in the training set.\n",
    "count = 0\n",
    "for key, value in dictionary.iteritems():\n",
    "    print(key, value)\n",
    "    count += 1\n",
    "    if count > 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter out tokens that appear in\n",
    "#less than 15 documents (absolute number) or\n",
    "#more than 0.5 documents (fraction of total corpus size, not absolute number).\n",
    "#after the above two steps, keep only the first 100000 most frequent tokens.\n",
    "dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For each document we create a dictionary reporting how many\n",
    "#words and how many times those words appear. \n",
    "bow_corpus = [dictionary.doc2bow(doc) for doc in processed_docs]\n",
    "bow_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Preview Bag Of Words for our sample preprocessed document\n",
    "bow_doc_431 = bow_corpus[431]\n",
    "for i in range(len(bow_doc_431)):\n",
    "    print(\"Word {} (\\\"{}\\\") appears {} time.\".format(bow_doc_431[i][0], \n",
    "                                               dictionary[bow_doc_4310[i][0]], \n",
    "bow_doc_431[i][1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create tf-idf model object using models\n",
    "#then apply transformation to the entire corpus\n",
    "tfidf = models.TfidfModel(bow_corpus)\n",
    "corpus_tfidf = tfidf[bow_corpus]\n",
    "for doc in corpus_tfidf:\n",
    "    pprint(doc)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in corpus_tfidf:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=10, id2word=dictionary, passes=2, workers=2)\n",
    "\n",
    "# For each topic, we will explore the words occuring in that topic and its relative weight.\n",
    "for idx, topic in lda_model.print_topics(-1):\n",
    "    print('Topic: {} \\nWords: {}'.format(idx, topic))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_docs[4310]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# running LDA using bag-of-words\n",
    "print(documents.loc[431])\n",
    "for index, score in sorted(lda_model[bow_corpus[431]], key=lambda tup: -1*tup[1]):\n",
    "    print(\"\\nScore: {}\\t \\nTopic: {}\".format(score, lda_model.print_topic(index, 10)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# running LDA using tf-idf \n",
    "lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=10, id2word=dictionary, passes=2, workers=4)\n",
    "\n",
    "for idx, topic in lda_model_tfidf.print_topics(-1):\n",
    "    print('Topic: {} Word: {}'.format(idx, topic))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unseen_document = 'The Pythagorean equation relates the sides of a right triangle in a simple way'\n",
    "bow_vector = dictionary.doc2bow(preprocess(unseen_document))\n",
    "for index, score in sorted(lda_model[bow_vector], key=lambda tup: -1*tup[1]):\n",
    "    print(\"Score: {}\\t Topic: {}\".format(score, lda_model.print_topic(index, 5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model.print_topics(20,num_words=15)[:10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model_tfidf.show_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}