Skip to content

Commit

Permalink
ArticleInfo API: cache if the query took 5 seconds
Browse files Browse the repository at this point in the history 
Move API code to ArticleInfo controller

Bug: https://phabricator.wikimedia.org/T175763
  • Loading branch information
@MusikAnimal
MusikAnimal committed Oct 17, 2017
1 parent 7febf54 commit 6739d11
Show file tree
Hide file tree
Showing 6 changed files with 196 additions and 113 deletions.
0 ...Resources/views/api/articleinfo.html.twig → ...Resources/views/articleInfo/api.html.twig
View file
File renamed without changes.
76 changes: 0 additions & 76 deletions src/AppBundle/Controller/ApiController.php
View file
Expand Up @@ -232,80 +232,4 @@ public function nonautomatedEdits(

return $view;
}

/**
* Get basic info on a given article.
* @Rest\Get("/api/articleinfo/{project}/{article}", requirements={"article"=".+"})
* @Rest\Get("/api/page/articleinfo/{project}/{article}", requirements={"article"=".+"})
* @param Request $request The HTTP request.
* @param string $project
* @param string $article
* @return View
*/
public function articleInfo(Request $request, $project, $article)
{
/** @var integer Number of days to query for pageviews */
$pageviewsOffset = 30;

$project = ProjectRepository::getProject($project, $this->container);
if (!$project->exists()) {
return new View(
['error' => "$project is not a valid project"],
Response::HTTP_NOT_FOUND
);
}

$page = new Page($project, $article);
$pageRepo = new PagesRepository();
$pageRepo->setContainer($this->container);
$page->setRepository($pageRepo);

if (!$page->exists()) {
return new View(
['error' => "$article was not found"],
Response::HTTP_NOT_FOUND
);
}

$info = $page->getBasicEditingInfo();
$creationDateTime = DateTime::createFromFormat('YmdHis', $info['created_at']);
$modifiedDateTime = DateTime::createFromFormat('YmdHis', $info['modified_at']);
$secsSinceLastEdit = (new DateTime)->getTimestamp() - $modifiedDateTime->getTimestamp();

$data = [
'revisions' => (int) $info['num_edits'],
'editors' => (int) $info['num_editors'],
'author' => $info['author'],
'author_editcount' => (int) $info['author_editcount'],
'created_at' => $creationDateTime->format('Y-m-d'),
'created_rev_id' => $info['created_rev_id'],
'modified_at' => $modifiedDateTime->format('Y-m-d H:i'),
'secs_since_last_edit' => $secsSinceLastEdit,
'last_edit_id' => (int) $info['modified_rev_id'],
'watchers' => (int) $page->getWatchers(),
'pageviews' => $page->getLastPageviews($pageviewsOffset),
'pageviews_offset' => $pageviewsOffset,
];

$view = View::create()->setStatusCode(Response::HTTP_OK);

if ($request->query->get('format') === 'html') {
$twig = $this->container->get('twig');
$view->setTemplate('api/articleinfo.html.twig');
$view->setTemplateData([
'data' => $data,
'project' => $project,
'page' => $page,
]);
$view->setFormat('html');
} else {
$res = array_merge([
'project' => $project->getDomain(),
'page' => $page->getTitle(),
], $data);
$view->setData($res)->setFormat('json');
}

return $view;
}
}
85 changes: 85 additions & 0 deletions src/AppBundle/Controller/ArticleInfoController.php
View file
Expand Up @@ -10,12 +10,15 @@
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\DependencyInjection\ContainerInterface;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\RedirectResponse;
use Symfony\Component\Process\Process;
use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
use Xtools\ProjectRepository;
use Xtools\Page;
use Xtools\PagesRepository;
use Xtools\ArticleInfo;
use DateTime;

/**
* This controller serves the search form and results for the ArticleInfo tool
Expand Down Expand Up @@ -175,4 +178,86 @@ public function resultAction(Request $request, $article)

return $response;
}

/************************ API endpoints ************************/

/**
* Get basic info on a given article.
* @Route("/api/articleinfo/{project}/{article}", requirements={"article"=".+"})
* @Route("/api/page/articleinfo/{project}/{article}", requirements={"article"=".+"})
* @param Request $request The HTTP request.
* @param string $project
* @param string $article
* @return View
* See ArticleInfoControllerTest::testArticleInfoApi()
* @codeCoverageIgnore
*/
public function articleInfoApiAction(Request $request, $project, $article)
{
/** @var integer Number of days to query for pageviews */
$pageviewsOffset = 30;

$project = ProjectRepository::getProject($project, $this->container);
if (!$project->exists()) {
return new JsonResponse(
['error' => "$project is not a valid project"],
Response::HTTP_NOT_FOUND
);
}

$page = $this->getAndValidatePage($project, $article);
if ($page instanceof RedirectResponse) {
return new JsonResponse(
['error' => "$article was not found"],
Response::HTTP_NOT_FOUND
);
}

$info = $page->getBasicEditingInfo();
$creationDateTime = DateTime::createFromFormat('YmdHis', $info['created_at']);
$modifiedDateTime = DateTime::createFromFormat('YmdHis', $info['modified_at']);
$secsSinceLastEdit = (new DateTime)->getTimestamp() - $modifiedDateTime->getTimestamp();

$data = [
'revisions' => (int) $info['num_edits'],
'editors' => (int) $info['num_editors'],
'author' => $info['author'],
'author_editcount' => (int) $info['author_editcount'],
'created_at' => $creationDateTime->format('Y-m-d'),
'created_rev_id' => $info['created_rev_id'],
'modified_at' => $modifiedDateTime->format('Y-m-d H:i'),
'secs_since_last_edit' => $secsSinceLastEdit,
'last_edit_id' => (int) $info['modified_rev_id'],
'watchers' => (int) $page->getWatchers(),
'pageviews' => $page->getLastPageviews($pageviewsOffset),
'pageviews_offset' => $pageviewsOffset,
];

if ($request->query->get('format') === 'html') {
$response = $this->render('articleInfo/api.html.twig', [
'data' => $data,
'project' => $project,
'page' => $page,
]);

// All /api routes by default respond with a JSON content type.
$response->headers->set('Content-Type', 'text/html');

// This endpoint is hit constantly and user could be browsing the same page over
// and over (popular noticeboard, for instance), so offload brief caching to browser.
$response->setClientTtl(350);

return $response;
}

$body = array_merge([
'project' => $project->getDomain(),
'page' => $page->getTitle(),
], $data);

return new JsonResponse(
$body,
Response::HTTP_OK
);
}
}
29 changes: 25 additions & 4 deletions src/Xtools/PagesRepository.php
View file
Expand Up @@ -203,14 +203,19 @@ public function getNumRevisions(Page $page, User $user = null)
* number of revisions, unique authors, initial author
* and edit count of the initial author.
* This is combined into one query for better performance.
* Caching is intentionally disabled, because using the gadget,
* this will get hit for a different page constantly, where
* the likelihood of cache benefiting us is slim.
* Caching is only applied if it took considerable time to process,
* because using the gadget, this will get hit for a different page
* constantly, where the likelihood of cache benefiting us is slim.
* @param Page $page The page.
* @return string[]
*/
public function getBasicEditingInfo(Page $page)
{
$cacheKey = 'page.basicInfo.'.$page->getId();
if ($this->cache->hasItem($cacheKey)) {
return $this->cache->getItem($cacheKey)->get();
}

$revTable = $this->getTableName($page->getProject()->getDatabaseName(), 'revision');
$userTable = $this->getTableName($page->getProject()->getDatabaseName(), 'user');
$pageTable = $this->getTableName($page->getProject()->getDatabaseName(), 'page');
Expand Down Expand Up @@ -251,7 +256,23 @@ public function getBasicEditingInfo(Page $page)
);";
$params = ['pageid' => $page->getId()];
$conn = $this->getProjectsConnection();
return $conn->executeQuery($sql, $params)->fetch();

// Get current time so we can compare timestamps
// and decide whether or to cache the result.
$time1 = time();
$result = $conn->executeQuery($sql, $params)->fetch();
$time2 = time();

// If it took over 5 seconds, cache the result for 20 minutes.
if ($time2 - $time1 > 5) {
$cacheItem = $this->cache->getItem($cacheKey)
->set($result)
->expiresAfter(new DateInterval('PT20M'));
$this->cache->save($cacheItem);
$this->stopwatch->stop($cacheKey);
}

return $result;
}

/**
Expand Down
33 changes: 0 additions & 33 deletions tests/AppBundle/Controller/ApiControllerTest.php
View file
Expand Up @@ -144,37 +144,4 @@ public function testNonautomatedEdits()
$response = $this->client->getResponse();
$this->assertEquals(403, $response->getStatusCode());
}

/**
* articleinfo endpoint, used for the XTools gadget
*/
public function testArticleInfo()
{
if (!$this->isSingle && $this->container->getParameter('app.is_labs')) {
$crawler = $this->client->request('GET', '/api/page/articleinfo/en.wikipedia.org/Main_Page');

$response = $this->client->getResponse();
$this->assertEquals(200, $response->getStatusCode());

$data = json_decode($response->getContent(), true);

// Some basic tests that should always hold true
$this->assertEquals($data['project'], 'en.wikipedia.org');
$this->assertEquals($data['page'], 'Main Page');
$this->assertTrue($data['revisions'] > 4000);
$this->assertTrue($data['editors'] > 400);
$this->assertEquals($data['author'], 'TwoOneTwo');
$this->assertEquals($data['created_at'], '2002-01-26');
$this->assertEquals($data['created_rev_id'], 139992);

$this->assertEquals(
[
'project', 'page', 'revisions', 'editors', 'author', 'author_editcount',
'created_at', 'created_rev_id', 'modified_at', 'secs_since_last_edit',
'last_edit_id', 'watchers', 'pageviews', 'pageviews_offset',
],
array_keys($data)
);
}
}
}
86 changes: 86 additions & 0 deletions tests/AppBundle/Controller/ArticleInfoControllerTest.php
View file
@@ -0,0 +1,86 @@
<?php
/**
* This file contains only the ArticleInfoControllerTest class.
*/

namespace Tests\AppBundle\Controller;

use Symfony\Bundle\FrameworkBundle\Test\WebTestCase;
use Symfony\Component\DependencyInjection\Container;
use AppBundle\Controller\ArticleInfoController;

/**
* Integration/unit tests for the ArticleInfoController.
* @group integration
*/
class ArticleInfoControllerTest extends WebTestCase
{
/** @var Container The DI container. */
protected $container;

/** @var Client The Symfony client */
protected $client;

/**
* Set up the tests.
*/
public function setUp()
{
$this->client = static::createClient();
$this->container = $this->client->getContainer();
$this->controller = new ArticleInfoController();
$this->controller->setContainer($this->container);
}

/**
* Test that the AdminStats index page displays correctly.
*/
public function testIndex()
{
$crawler = $this->client->request('GET', '/articleinfo/de.wikipedia');
$this->assertEquals(200, $this->client->getResponse()->getStatusCode());

if (!$this->container->getParameter('app.is_labs') || $this->container->getParameter('app.single_wiki')) {
return;
}

// should populate project input field
$this->assertEquals('de.wikipedia.org', $crawler->filter('#project_input')->attr('value'));
}

/**
* Test the method that sets up a AdminStats instance.
*/
public function testArticleInfoApi()
{
// For now...
if (!$this->container->getParameter('app.is_labs') || $this->container->getParameter('app.single_wiki')) {
return;
}

$crawler = $this->client->request('GET', '/api/page/articleinfo/en.wikipedia.org/Main_Page');

$response = $this->client->getResponse();
$this->assertEquals(200, $response->getStatusCode());

$data = json_decode($response->getContent(), true);

// Some basic tests that should always hold true
$this->assertEquals($data['project'], 'en.wikipedia.org');
$this->assertEquals($data['page'], 'Main Page');
$this->assertTrue($data['revisions'] > 4000);
$this->assertTrue($data['editors'] > 400);
$this->assertEquals($data['author'], 'TwoOneTwo');
$this->assertEquals($data['created_at'], '2002-01-26');
$this->assertEquals($data['created_rev_id'], 139992);

$this->assertEquals(
[
'project', 'page', 'revisions', 'editors', 'author', 'author_editcount',
'created_at', 'created_rev_id', 'modified_at', 'secs_since_last_edit',
'last_edit_id', 'watchers', 'pageviews', 'pageviews_offset',
],
array_keys($data)
);
}
}

0 comments on commit 6739d11

Please sign in to comment.