lp-pipeline-for-supervised-er-demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from graphdatascience import GraphDataScience\n",
    "pd.options.display.width = 0\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', 100)\n",
    "pd.set_option('max_colwidth', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "RANDOM_SEED = 7474"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "HOST = 'neo4j://localhost'\n",
    "USERNAME = 'neo4j'\n",
    "PASSWORD = 'password'\n",
    "\n",
    "gds = GraphDataScience(HOST, auth=(USERNAME, PASSWORD), aura_ds=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "nodeProjection                                                                                                                                                                                                                                         {'User': {'label': 'User', 'properties': {}}, 'Website': {'label': 'Website', 'properties': {}}}\nrelationshipProjection    {'SAME_AS': {'orientation': 'UNDIRECTED', 'aggregation': 'DEFAULT', 'type': 'SAME_AS', 'properties': {}}, 'VISITED': {'orientation': 'UNDIRECTED', 'aggregation': 'DEFAULT', 'type': 'VISITED', 'properties': {}}, 'CHILD_OF': {'orientation': 'UNDIRECTED', 'aggregation': 'DEFAULT', 'type': 'CHILD_OF', 'properties': {}}}\ngraphName                                                                                                                                                                                                                                                                                                                                          proj\nnodeCount                                                                                                                                                                                                                                                                                                                                       2433861\nrelationshipCount                                                                                                                                                                                                                                                                                                                              10353608\nprojectMillis                                                                                                                                                                                                                                                                                                                                      2625\nName: 0, dtype: object"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g, _  = gds.graph.project('proj',['User', 'Website'], {\n",
    "    'SAME_AS': {'orientation': 'UNDIRECTED'},\n",
    "    'CHILD_OF': {'orientation': 'UNDIRECTED'},\n",
    "    'VISITED': {'orientation': 'UNDIRECTED'}\n",
    "})\n",
    "_"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "nodePropertiesWritten                                                                                                                                                                                                                                                                                                                                                                                         2433861\nmutateMillis                                                                                                                                                                                                                                                                                                                                                                                                        1\nnodeCount                                                                                                                                                                                                                                                                                                                                                                                                     2433861\npreProcessingMillis                                                                                                                                                                                                                                                                                                                                                                                                 4\ncomputeMillis                                                                                                                                                                                                                                                                                                                                                                                                    8972\nconfiguration            {'nodeSelfInfluence': 0, 'relationshipWeightProperty': None, 'propertyRatio': 0.0, 'concurrency': 4, 'randomSeed': 7474, 'normalizationStrength': 0.0, 'iterationWeights': [0.0, 1.0, 0.7, 0.5, 0.8], 'embeddingDimension': 128, 'nodeLabels': ['*'], 'sudo': False, 'relationshipTypes': ['CHILD_OF', 'VISITED'], 'featureProperties': [], 'mutateProperty': 'embedding', 'username': None}\nName: 0, dtype: object"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.fastRP.mutate(g,\n",
    "                mutateProperty='embedding',\n",
    "                relationshipTypes=['CHILD_OF' , 'VISITED'],\n",
    "                iterationWeights=[0.0, 1.0, 0.7, 0.5, 0.8],\n",
    "                embeddingDimension=128,\n",
    "                randomSeed=RANDOM_SEED)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "name                                                                                                            pipe\nnodePropertySteps                                                                                                 []\nfeatureSteps                                                                                                      []\nsplitConfig          {'negativeSamplingRatio': 1.0, 'testFraction': 0.1, 'validationFolds': 3, 'trainFraction': 0.1}\nparameterSpace                                                        {'RandomForest': [], 'LogisticRegression': []}\nName: 0, dtype: object"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe, _ = gds.beta.pipeline.linkPrediction.create(\"pipe\")\n",
    "_"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "name                                                                                                            pipe\nnodePropertySteps                                                                                                 []\nfeatureSteps                                           [{'name': 'L2', 'config': {'nodeProperties': ['embedding']}}]\nsplitConfig          {'negativeSamplingRatio': 1.0, 'testFraction': 0.1, 'validationFolds': 3, 'trainFraction': 0.1}\nparameterSpace                                                        {'RandomForest': [], 'LogisticRegression': []}\nName: 0, dtype: object"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.addFeature('l2', nodeProperties=['embedding'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "name                                                                                                                                           pipe\nnodePropertySteps                                                                                                                                []\nfeatureSteps         [{'name': 'L2', 'config': {'nodeProperties': ['embedding']}}, {'name': 'COSINE', 'config': {'nodeProperties': ['embedding']}}]\nsplitConfig                                         {'negativeSamplingRatio': 1.0, 'testFraction': 0.1, 'validationFolds': 3, 'trainFraction': 0.1}\nparameterSpace                                                                                       {'RandomForest': [], 'LogisticRegression': []}\nName: 0, dtype: object"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.addFeature('cosine', nodeProperties=['embedding'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "name                                                                                                                                           pipe\nnodePropertySteps                                                                                                                                []\nfeatureSteps         [{'name': 'L2', 'config': {'nodeProperties': ['embedding']}}, {'name': 'COSINE', 'config': {'nodeProperties': ['embedding']}}]\nsplitConfig                                         {'negativeSamplingRatio': 2.0, 'testFraction': 0.2, 'validationFolds': 5, 'trainFraction': 0.4}\nparameterSpace                                                                                       {'RandomForest': [], 'LogisticRegression': []}\nName: 0, dtype: object"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.configureSplit(testFraction=0.2, trainFraction=0.4, validationFolds=5, negativeSamplingRatio=2.0)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "name                                                                                                                                                                                                                                                                                                                                                      pipe\nnodePropertySteps                                                                                                                                                                                                                                                                                                                                           []\nfeatureSteps                                                                                                                                                                                                                    [{'name': 'L2', 'config': {'nodeProperties': ['embedding']}}, {'name': 'COSINE', 'config': {'nodeProperties': ['embedding']}}]\nsplitConfig                                                                                                                                                                                                                                                    {'negativeSamplingRatio': 2.0, 'testFraction': 0.2, 'validationFolds': 5, 'trainFraction': 0.4}\nparameterSpace       {'RandomForest': [], 'LogisticRegression': [{'maxEpochs': 2000, 'minEpochs': 1, 'penalty': 0.0, 'patience': 3, 'methodName': 'LogisticRegression', 'batchSize': 100, 'tolerance': 1e-05}, {'maxEpochs': 1000, 'minEpochs': 1, 'penalty': 0.01, 'patience': 3, 'methodName': 'LogisticRegression', 'batchSize': 100, 'tolerance': 1e-05}]}\nName: 0, dtype: object"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.addLogisticRegression(penalty=0.0, patience=3, maxEpochs=2000, tolerance=0.00001)\n",
    "pipe.addLogisticRegression(penalty=0.01, patience=3, maxEpochs=1000, tolerance=0.00001)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "trained_pipe_model, res = pipe.train(g,\n",
    "    modelName='entity-resolution-model',\n",
    "    randomSeed=RANDOM_SEED,\n",
    "    nodeLabels=['User'],\n",
    "    relationshipTypes=['SAME_AS'],\n",
    "    negativeClassWeight=1.0/2.0)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9332965423634675\n"
     ]
    }
   ],
   "source": [
    "print(trained_pipe_model.metrics()[\"AUCPR\"][\"test\"])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "data": {
      "text/plain": "    node1  node2  probability\n0   33845  33846     0.838466\n1   33806  33807     0.838466\n2    1563  15803     0.838214\n3   15803  21642     0.837809\n4    1563  21642     0.837587\n5   10644  13318     0.837585\n6   12812  15803     0.837573\n7    4592  18438     0.837123\n8   12812  13319     0.836847\n9   13319  15803     0.836830\n10   1563  12812     0.836678\n11   1563  13319     0.836440\n12   4592   8552     0.836396\n13  12812  21642     0.836359\n14   8855  18438     0.836330\n15   2744  23427     0.836184\n16   8552  32234     0.836008\n17  13319  21642     0.835998\n18   4592   8855     0.835950\n19  18438  32234     0.835548",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>node1</th>\n      <th>node2</th>\n      <th>probability</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>33845</td>\n      <td>33846</td>\n      <td>0.838466</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>33806</td>\n      <td>33807</td>\n      <td>0.838466</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1563</td>\n      <td>15803</td>\n      <td>0.838214</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>15803</td>\n      <td>21642</td>\n      <td>0.837809</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1563</td>\n      <td>21642</td>\n      <td>0.837587</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>10644</td>\n      <td>13318</td>\n      <td>0.837585</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>12812</td>\n      <td>15803</td>\n      <td>0.837573</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>4592</td>\n      <td>18438</td>\n      <td>0.837123</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>12812</td>\n      <td>13319</td>\n      <td>0.836847</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>13319</td>\n      <td>15803</td>\n      <td>0.836830</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>1563</td>\n      <td>12812</td>\n      <td>0.836678</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>1563</td>\n      <td>13319</td>\n      <td>0.836440</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>4592</td>\n      <td>8552</td>\n      <td>0.836396</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>12812</td>\n      <td>21642</td>\n      <td>0.836359</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>8855</td>\n      <td>18438</td>\n      <td>0.836330</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>2744</td>\n      <td>23427</td>\n      <td>0.836184</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>8552</td>\n      <td>32234</td>\n      <td>0.836008</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>13319</td>\n      <td>21642</td>\n      <td>0.835998</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>4592</td>\n      <td>8855</td>\n      <td>0.835950</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>18438</td>\n      <td>32234</td>\n      <td>0.835548</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trained_pipe_model.predict_stream(g, sampleRate=1.0, topN=20, nodeLabels=['User'], relationshipTypes=['SAME_AS'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}