Staging to master to add github metrics

scenarios/sentence_similarity/gensen_aml_deep_dive.ipynb

@@ -127,7 +127,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 1,
    "metadata": {
     "scrolled": true
    },
@@ -887,8 +887,6 @@
     }
    ],
    "source": [
-    "import shutil\n",
-    "\n",
     "gensen_folder = os.path.join(project_folder,'utils_nlp/gensen/')\n",
     "shutil.copy('gensen_train.py', gensen_folder)\n",
     "shutil.copy('gensen_config.json', gensen_folder)"
@@ -1030,7 +1028,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [

tests/ci/repo_metrics_pipeline.yml

@@ -0,0 +1,26 @@
+
+jobs:
+- job: Repometrics
+  pool:
+    vmImage: 'ubuntu-16.04'
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.6'
+      architecture: 'x64'
+
+  - script: |
+      cp tools/repo_metrics/config_template.py tools/repo_metrics/config.py
+      sed -i ''s/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/$(github_token)/g'' tools/repo_metrics/config.py
+      sed -i ''s/XXXXXXXXXXXXXXXXXXXXXXXXX/$(cosmosdb_connectionstring)/g'' tools/repo_metrics/config.py
+    displayName: Configure CosmosDB Connection
+
+  - script: | 
+      python -m pip install python-dateutil>=2.80 pymongo>=3.8.0 gitpython>2.1.11 requests>=2.21.0
+      python tools/repo_metrics/track_metrics.py --github_repo "https://github.com/microsoft/nlp" --save_to_database
+    displayName: Python script to record stats
+
+
+
+

tools/repo_metrics/README.md

@@ -1,6 +1,6 @@
 # Repository Metrics
 
-[![Build status](https://msdata.visualstudio.com/AlgorithmsAndDataScience/_apis/build/status/Recommenders/Recommenders%20repo%20stats)](https://msdata.visualstudio.com/AlgorithmsAndDataScience/_build/latest?definitionId=5206)
+[![Build Status](https://dev.azure.com/best-practices/nlp/_apis/build/status/repo_metrics?branchName=master)](https://dev.azure.com/best-practices/nlp/_build/latest?definitionId=36&branchName=master)
 
 We developed a script that allows us to track the metrics of the Recommenders repo. Some of the metrics we can track are listed here:
 
@@ -10,17 +10,27 @@ We developed a script that allows us to track the metrics of the Recommenders re
 * Number of views
 * Number of lines of code
 
-To see the full list of metrics, see [git_stats.py](scripts/repo_metrics/git_stats.py)
+To see the full list of metrics, see [git_stats.py](git_stats.py)
 
 The first step is to set up the credentials, copy the configuration file and fill up the credentials of GitHub and CosmosDB:
 
-    cp scripts/repo_metrics/config_template.py scripts/repo_metrics/config.py
+    cp tools/repo_metrics/config_template.py tools/repo_metrics/config.py
 
 To track the current state of the repository and save it to CosmosDB:
 
     python scripts/repo_metrics/track_metrics.py --github_repo "https://github.com/Microsoft/Recommenders" --save_to_database
 
 To track an event related to this repository and save it to CosmosDB:
 
-    python scripts/repo_metrics/track_metrics.py --event "Today we did our first blog of the project" --event_date 2018-12-01 --save_to_database
+    python tools/repo_metrics/track_metrics.py --event "Today we did our first blog of the project" --event_date 2018-12-01 --save_to_database
+
+
+### Setting up Azure CosmosDB
+
+The API that we is used to track the GitHub metrics is the [Mongo API](https://docs.microsoft.com/en-us/azure/cosmos-db/mongodb-introduction).
+
+The database name and collections name are defined in the [config file](config_template.py). There are two main collections, defined as `COLLECTION_GITHUB_STATS` and `COLLECTION_EVENTS` to store the information defined on the previous section. 
+
+**IMPORTANT NOTE**: If the database and the collections are created directly through the portal, a common partition key should be defined. We recommend to use `date` as partition key.
+
 

tools/repo_metrics/config_template.py

@@ -7,7 +7,7 @@
 
 # CosmosDB Mongo API
 CONNECTION_STRING = "mongodb://XXXXXXXXXXXXXXXXXXXXXXXXX.documents.azure.com:10255/?ssl=true&replicaSet=globaldb"
-DATABASE = "reco_stats"
+DATABASE = "nlp_stats"
 COLLECTION_GITHUB_STATS = "github_stats"
 COLLECTION_EVENTS = "events"
 

tools/repo_metrics/track_metrics.py

@@ -5,7 +5,6 @@
 import os
 
 # Need to append a full path instead of relative path.
-# This seems to be an issue from Azure DevOps command line task.
 # NOTE this does not affect running directly in the shell.
 sys.path.append(os.getcwd())
 import argparse
@@ -14,9 +13,8 @@
 from datetime import datetime
 from dateutil.parser import isoparse
 from pymongo import MongoClient
-from datetime import datetime
-from scripts.repo_metrics.git_stats import Github
-from scripts.repo_metrics.config import (
+from tools.repo_metrics.git_stats import Github
+from tools.repo_metrics.config import (
     GITHUB_TOKEN,
     CONNECTION_STRING,
     DATABASE,
@@ -32,6 +30,7 @@
 
 def parse_args():
     """Argument parser.
+
     Returns:
         obj: Parser.
     """
@@ -61,8 +60,10 @@ def parse_args():
 
 def connect(uri="mongodb://localhost"):
     """Mongo connector.
+
     Args:
         uri (str): Connection string.
+
     Returns:
         obj: Mongo client.
     """
@@ -78,9 +79,11 @@ def connect(uri="mongodb://localhost"):
 
 def event_as_dict(event, date):
     """Encodes an string event input as a dictionary with the date.
+
     Args:
         event (str): Details of a event.
         date (datetime): Date of the event.
+
     Returns:
         dict: Dictionary with the event and the date.
     """
@@ -89,8 +92,10 @@ def event_as_dict(event, date):
 
 def github_stats_as_dict(github):
     """Encodes Github statistics as a dictionary with the date.
+
     Args:
         obj: Github object.
+
     Returns:
         dict: Dictionary with Github details and the date.
     """
@@ -125,6 +130,7 @@ def github_stats_as_dict(github):
 
 def tracker(args):
     """Main function to track metrics.
+
     Args:
         args (obj): Parsed arguments.
     """