Skip to content
Permalink
Browse files

retrieves history of a package

  • Loading branch information...
sdpython committed Mar 24, 2018
1 parent 805ad22 commit 37e2d81469175500b048b8301c944acb162c5f66
@@ -0,0 +1,106 @@
"""
@brief test log(time=42s)
"""

import sys
import os
import unittest
import datetime

if "temp_" in os.path.abspath(__file__):
raise ImportError(
"this file should not be imported in that location: " +
os.path.abspath(__file__))

try:
import src
except ImportError:
path = os.path.normpath(
os.path.abspath(
os.path.join(
os.path.split(__file__)[0],
"..",
"..")))
if path not in sys.path:
sys.path.append(path)
import src

from src.pyquickhelper.loghelper import fLOG
from src.pyquickhelper.pycode import ExtTestCase
from src.pyquickhelper.loghelper.history_helper import build_history, compile_history


class TestHistoryHelper(ExtTestCase):

issues = [{'body': None, 'closed_at': None, 'number': 115, 'state': 'open', 'title': 'run notebook with starting a kernel',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/115'},
{'body': None, 'closed_at': None, 'number': 114, 'state': 'open', 'title': 'automatically builds history with release and issues',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/114'},
{'body': None, 'closed_at': '2018-03-19T20:31:48Z', 'number': 113, 'state': 'closed', 'title': 'propose a fix for a bug introduced by pip 9.0.2',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/113'},
{'body': None, 'closed_at': '2018-03-15T18:20:41Z', 'number': 112, 'state': 'closed',
'title': 'allow to set custom snippets for notebooks',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/112'},
{'body': None, 'closed_at': '2018-03-20T23:24:09Z', 'number': 111, 'state': 'closed',
'title': 'enable manual snippet for notebook, repace add_notebook_menu by toctree in sphinx',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/111'},
{'body': None, 'closed_at': None, 'number': 110, 'state': 'open',
'title': 'sphinx documentation, index is missing in latex final file',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/110'},
{'body': None, 'closed_at': '2018-03-15T00:22:01Z', 'number': 109, 'state': 'closed',
'title': 'run javascript producing svg and convert it into png',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/109'},
{'body': None, 'closed_at': '2018-03-10T14:58:28Z', 'number': 108, 'state': 'closed',
'title': 'add command lab, creates a script to start jupyter lab on notebook',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/108'},
{'body': None, 'closed_at': '2018-03-12T22:19:19Z', 'number': 107, 'state': 'closed', 'title': 'convert svg into png for notebook snippets',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/107'},
{'body': None, 'closed_at': '2018-03-03T14:52:08Z', 'number': 106, 'state': 'closed', 'title': 'replace pdflatex by xelatex to handle utf-8',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/106'},
{'body': None, 'closed_at': None, 'number': 105, 'state': 'open', 'title': 'append bokeh js and css in notebook converted into rst',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/105'},
{'body': None, 'closed_at': '2018-03-01T22:56:16Z', 'number': 104, 'state': 'closed',
'title': 'implement visit, depart for pending_xref and rst translator',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/104'},
{'body': None, 'closed_at': '2018-03-01T22:54:56Z', 'number': 103, 'state': 'closed', 'title': 'fix import issue for Sphinx 1.7.1',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/103'},
{'body': None, 'closed_at': '2018-02-24T00:21:11Z', 'number': 102, 'state': 'closed', 'title': 'fix sphinx command line',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/102'},
{'body': None, 'closed_at': '2018-02-13T23:07:46Z', 'number': 101, 'state': 'closed', 'title': 'migrate to sphinx 1.7',
'url': 'https://api.github.com/repos/sdpython/pyquickhelper/issues/101'},
]

releases = [(datetime.datetime(2018, 3, 22, 0, 57, 1), '1.7.2482', 2087803),
(datetime.datetime(2018, 3, 19, 20, 33, 18), '1.7.2468', 2083943),
(datetime.datetime(2018, 3, 3, 20, 57, 52), '1.7.2448', 2078334), (datetime.datetime(
2018, 2, 23, 22, 53, 8), '1.7.2438', 2077672),
(datetime.datetime(2018, 2, 23, 11, 13, 6), '1.7.2429', 2077611),
(datetime.datetime(2018, 2, 13, 15, 14, 28), '1.6.2413', 2076866),
(datetime.datetime(2018, 2, 4, 15, 49, 41), '1.6.2398', 647370),
(datetime.datetime(2017, 11, 28, 18, 55, 43), '1.5.2275', 521698)]

def test_history(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")

history = build_history('sdpython', 'pyquickhelper',
url="https://api.github.com/repos/sdpython/pyquickhelper/issues/{0}",
issues=TestHistoryHelper.issues,
max_issue=115, fLOG=fLOG, releases=TestHistoryHelper.releases)
nb = 0
for release in history:
self.assertIn('issues', release)
self.assertIn('release', release)
nb += len(release['issues'])
self.assertNotEmpty(history)
self.assertGreater(nb, 1)

output = compile_history(history)
self.assertIn('* - `101`:', output)
self.assertIn('1.7.2482 - 2018-03-22 - 1.99Mb', output)


if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,53 @@
"""
@brief test log(time=42s)
"""

import sys
import os
import unittest
import datetime

if "temp_" in os.path.abspath(__file__):
raise ImportError(
"this file should not be imported in that location: " +
os.path.abspath(__file__))

try:
import src
except ImportError:
path = os.path.normpath(
os.path.abspath(
os.path.join(
os.path.split(__file__)[0],
"..",
"..")))
if path not in sys.path:
sys.path.append(path)
import src

from src.pyquickhelper.loghelper import fLOG
from src.pyquickhelper.loghelper import enumerate_pypi_versions_date


class TestPypiHelper(unittest.TestCase):

def test_clone_repo(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")

iter = enumerate_pypi_versions_date('pyquickhelper')
res = []
for it in iter:
res.append(it)
if len(res) >= 2:
break
self.assertEqual(len(res), 2)
self.assertIsInstance(res[0][0], datetime.datetime)
self.assertGreater(res[0][2], 0)
self.assertIn('.', res[0][1])


if __name__ == "__main__":
unittest.main()
@@ -6,6 +6,7 @@
from .custom_log import CustomLog from .custom_log import CustomLog
from .flog import fLOG, noLOG, fLOGFormat, PQHException, download, unzip, removedirs from .flog import fLOG, noLOG, fLOGFormat, PQHException, download, unzip, removedirs
from .process_helper import reap_children from .process_helper import reap_children
from .pypi_helper import enumerate_pypi_versions_date
from .pyrepo_helper import SourceRepository from .pyrepo_helper import SourceRepository
from .repositories.pygit_helper import clone as git_clone from .repositories.pygit_helper import clone as git_clone
from .run_cmd import run_cmd, decode_outerr, run_script, RunCmdException from .run_cmd import run_cmd, decode_outerr, run_script, RunCmdException
@@ -1,6 +1,6 @@
""" """
@file @file
@brief Calls github API. @brief Calls :epkg:`github` API.
""" """
import requests import requests


@@ -20,7 +20,7 @@ def __init__(self, response, url):


def call_github_api(owner, repo, ask, auth=None, headers=None): def call_github_api(owner, repo, ask, auth=None, headers=None):
""" """
Call `GitHub REST API <https://developer.github.com/v3/>`_. Calls `GitHub REST API <https://developer.github.com/v3/>`_.
@param owner owner of the project @param owner owner of the project
@param auth tuple *(user, password)* @param auth tuple *(user, password)*
@@ -0,0 +1,154 @@
"""
@file
@helper Build history for a module.
"""
from datetime import datetime, timedelta
import requests
import warnings
from jinja2 import Template
from .github_api import call_github_api
from .pypi_helper import enumerate_pypi_versions_date


def enumerate_closed_issues(owner, repo, since=None, issues=None,
url=None, max_issue=None):
"""
Enumerates github issues for a repo and an owner
since a given date.
@param owner repo owner
@param repo repository
@param since not older than that date, if None,
do not go beyond a year
@param issues to bypass @see fn call_github_api
@param url if available, something like
``https://api.github.com/repos/sdpython/pyquickhelper/issues/{0}``
@param max_issue max number of issues
@return iterator on issues ``(number, date, title)``
"""
if since is None:
since = datetime.now() - timedelta(365)
if issues is None and url is not None and max_issue is not None:
issues = [dict(url=url.format(k)) for k in range(max_issue, 0, -1)]
elif issues is None:
issues = call_github_api(owner, repo, 'issues')
if len(issues) == 0:
raise ValueError("No issue found.")
for issue in issues:
if 'title' not in issue:
url = issue['url']
response = requests.get(url)
content = response.json()
if 'API rate limit exceeded' in content.get('message', ''):
warnings.warn('API rate limit exceeded')
break
else:
content = issue
closed = content.get('closed_at', None)
if closed is None:
continue
title = content['title']
closed = datetime.strptime(closed.strip('Z'), "%Y-%m-%dT%H:%M:%S")
number = content['number']
if closed < since:
break
yield number, closed, title


def build_history(owner, repo, name=None, since=None, issues=None, url=None,
max_issue=None, releases=None, fLOG=None):
"""
Returns an history of a module.
@param owner repo owner
@param repo repository
@param name None if ``name == repo``
@param since not older than that date, if None,
do not go beyond a year
@param issues see @see fn call_github_api (unit test)
@param url see @see fn call_github_api (unit test)
@param max_issue see @see fn call_github_api (unit test)
@param releases bypass :epkg:`pypi` (unit test)
@param fLOG logging function
@return iterator on issues ``(number, date, title)``
"""
if since is None:
since = datetime.now() - timedelta(365)
if name is None:
name = repo

kept_issues = []
for issue in enumerate_closed_issues(owner, repo, since, issues=issues,
url=url, max_issue=max_issue):
kept_issues.append(issue)
if fLOG:
fLOG("[build_history] ", name, issue[:2])
if len(kept_issues) == 0:
raise ValueError("No issue found.")

if releases is None:
versions = []
for date, version, size in enumerate_pypi_versions_date(name):
if date < since:
break
if fLOG:
fLOG("[build_history] ", name, version, date)
versions.append((date, version, size))
else:
versions = releases
if len(versions) == 0:
raise ValueError('No release found.')

# merge
dates = [(v[0], "v", v) for v in versions]
dates.extend((i[1], "i", i) for i in kept_issues)
dates.sort(reverse=True)

merged = []
current = None
for d, v, obj in dates:
if v == 'v':
if current is not None:
merged.append(current)
current = dict(release=obj[1], size=obj[2], date=obj[0], issues=[])
elif v == 'i':
if current is not None:
issue = dict(title=obj[2], date=obj[1], number=obj[0])
current['issues'].append(issue)

if current is not None:
merged.append(current)
return merged


_template = """
=======
History
=======
{% for release in releases %}
{{ release['release'] }} - {{ release['date'].strftime("%Y-%m-%d") }} - {{ '%1.2fMb' % (release['size'] * 2**(-20)) }}
{{ '=' * (len(release['release']) + 22) }}
{% for issue in release['issues'] %}
* - `{{issue['number']}}`: {{issue['title']}} ({{issue['date'].strftime("%Y-%m-%d")}}) {% endfor %}
{% endfor %}
"""


def compile_history(releases, template=None):
"""
Compile history and produces a :epkg:`rst` file.
@param releases output of @see fn build_history
@param template :epkg:`jinja2` template (None means default one)
@return output
"""
if template is None:
global _template
template = _template
tmpl = Template(template)
return tmpl.render(releases=releases, len=len)

0 comments on commit 37e2d81

Please sign in to comment.
You can’t perform that action at this time.