Spark MLLib.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1e6caef4",
   "metadata": {},
   "source": [
    "# Spark MLLib\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5e5d75bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import pakages \n",
    "from pyspark import SparkContext\n",
    "from pyspark.mllib.stat import Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1967ada2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the existing SparkContext\n",
    "sc=SparkContext.getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cbe5f4c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#data rdd of vectors \n",
    "dataMatrix = sc.parallelize([[1,2,3],[4,5,6],[7,8,9],[10,11,12]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b27bf9f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5.5 6.5 7.5]\n",
      "[15. 15. 15.]\n",
      "[4. 4. 4.]\n"
     ]
    }
   ],
   "source": [
    "# Compute summarystatistics\n",
    "summary=Statistics.colStats(dataMatrix)\n",
    "print(summary.mean())\n",
    "print(summary.variance())\n",
    "print(summary.numNonzeros())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "727b03de",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f8e597a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcd02a01",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9709664",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cff77589",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fe126fc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a4e7343",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35000cf5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85f354da",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4715953",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99ac15f4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52892b67",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50f6ce87",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "006c2ae2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75d30d7a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25f192b0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90dc1682",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccfe63f3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}