Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
...
Checking mergeability… Don't worry, you can still create the pull request.
  • 11 commits
  • 18 files changed
  • 0 commit comments
  • 1 contributor
View
14 README.markdown
@@ -89,6 +89,20 @@ Note that you will need a functional rabbitmq server − but that's damn easy to
This can take long time. GitHub API is limited to 60 calls per minute,
so the commands needs to wait.
+### Search engine
+
+We use [Solr](http://lucene.apache.org/solr/) and it's PHP client [Solarium](http://solarium-project.org) to search bundles.
+Recommended schema can be found
+[**here**](https://github.com/KnpLabs/KnpBundles/blob/master/src/Knp/Bundle/KnpBundlesBundle/Resources/solr/schema.xml).
+Put it inside the `solr/conf` directory, and run Solr with `$ java -jar start.jar`.
+Bundles will be automatically indexed on next update, or you can force indexing by console command.
+
+If you have Solr up and running, simply do:
+
+ php app/console kb:solr:index --verbose
+
+This will index all bundles.
+
### Generate sitemap
php app/console kb:sitemap:generate --spaceless=1
View
1  app/AppKernel.php
@@ -31,6 +31,7 @@ public function registerBundles()
new Ornicar\GravatarBundle\OrnicarGravatarBundle(),
new JMS\I18nRoutingBundle\JMSI18nRoutingBundle(),
new OldSound\RabbitMqBundle\OldSoundRabbitMqBundle(),
+ new Nelmio\SolariumBundle\NelmioSolariumBundle(),
// register your applications here
new Knp\Bundle\KnpBundlesBundle\KnpBundlesBundle(),
View
4 app/autoload.php
@@ -35,13 +35,15 @@
'PhpAmqpLib' => __DIR__.'/../vendor/videlalvaro/php-amqplib',
'Imagine' => __DIR__.'/../vendor/imagine/lib',
'Igorw' => __DIR__.'/../vendor/bundles',
+ 'Nelmio' => __DIR__.'/../vendor/bundles',
));
$loader->registerPrefixes(array(
'Twig_Extensions_' => __DIR__.'/../vendor/twig-extensions/lib',
'Twig_' => __DIR__.'/../vendor/twig/lib',
'Github_' => __DIR__.'/../vendor/php-github-api/lib',
- 'PHPGit_' => __DIR__.'/../vendor/php-git-repo/lib'
+ 'PHPGit_' => __DIR__.'/../vendor/php-git-repo/lib',
+ 'Solarium_' => __DIR__.'/../vendor/solarium/library',
));
$loader->register();
View
7 app/config/config.yml
@@ -112,3 +112,10 @@ old_sound_rabbit_mq:
exchange_options: {name: 'upload-bundle', type: direct, durable: true }
queue_options: {name: 'upload-bundle'}
callback: knp_bundles.consumer.update_bundle
+
+nelmio_solarium:
+ adapter:
+ class: Solarium_Client_Adapter_Http
+ host: %solarium.host%
+ port: %solarium.port%
+ path: %solarium.path%
View
4 app/config/parameters.yml.dist
@@ -26,3 +26,7 @@ parameters:
old_sound_rabbit_mq.debug: %kernel.debug%
kb_sitemap.base_url: http://knpbundles.com
+
+ solarium.host: 127.0.0.1
+ solarium.port: 8983
+ solarium.path: /solr
View
4 app/config/parameters.yml.test
@@ -25,3 +25,7 @@ parameters:
old_sound_rabbit_mq.vhost: "/"
kb_sitemap.base_url: http://knpbundles.com
+
+ solarium.host: 127.0.0.1
+ solarium.port: 8983
+ solarium.path: /solr
View
10 deps
@@ -172,4 +172,12 @@
[FileServeBundle]
git=http://github.com/igorw/IgorwFileServeBundle.git
- target=bundles/Igorw/FileServeBundle
+ target=bundles/Igorw/FileServeBundle
+
+[solarium]
+ git=http://github.com/basdenooijer/solarium.git
+ version=origin/develop
+
+[NelmioSolariumBundle]
+ git=http://github.com/nelmio/NelmioSolariumBundle.git
+ target=bundles/Nelmio/SolariumBundle
View
4 deps.lock
@@ -42,4 +42,6 @@ RabbitMqBundle f6445361c8abef57eed0f15febe596b35e8a266b
php-amqplib 18d5c146a8e211ffcd22007c3d099f88b91563f9
MonologBundle 289442e385e7c0fe4aa2e8884d11e774a8f9ad99
Imagine 45f0b8e6568b0bca99ae4d4ae6771b69ca6da0dd
-FileServeBundle c11af119eb634d4c16270ebb295dd1038ceb9458
+FileServeBundle c11af119eb634d4c16270ebb295dd1038ceb9458
+solarium 72ffa0568167252a352028ce4b2f360c0bd07848
+NelmioSolarumBundle f1f0c436e727e28acd209c5c9e1176a8ae306ea6
View
80 src/Knp/Bundle/KnpBundlesBundle/Command/KbSolrIndexCommand.php
@@ -0,0 +1,80 @@
+<?php
+
+namespace Knp\Bundle\KnpBundlesBundle\Command;
+
+use Knp\Bundle\KnpBundlesBundle\Entity\Bundle;
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * @author Paweł Jędrzejewski <pjedrzejewski@diweb.pl>
+ * @author Igor Wiedler <igor@wiedler.ch>
+ */
+class KbSolrIndexCommand extends ContainerAwareCommand
+{
+ /**
+ * {@inheritdoc}
+ */
+ protected function configure()
+ {
+ $this
+ ->setName('kb:solr:index')
+ ->setDefinition(array(
+ new InputOption('force', null, InputOption::VALUE_NONE, 'Force a re-indexing of all bundles'),
+ new InputArgument('bundleName', InputArgument::OPTIONAL, 'Bundle name to index'),
+ ))
+ ->setDescription('Indexes bundles in Solr')
+ ;
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function execute(InputInterface $input, OutputInterface $output)
+ {
+ $verbose = $input->getOption('verbose');
+ $force = $input->getOption('force');
+ $bundleName = $input->getArgument('bundleName');
+
+ $doctrine = $this->getContainer()->get('doctrine');
+ $solarium = $this->getContainer()->get('solarium.client');
+ $indexer = $this->getContainer()->get('knp_bundles.indexer.solr');
+
+ if ($bundleName) {
+ list($username, $name) = explode('/', $bundleName);
+ $bundles = array($doctrine->getRepository('Knp\\Bundle\\KnpBundlesBundle\\Entity\\Bundle')->findOneByUsernameAndName($username, $name));
+ } elseif ($force) {
+ $bundles = $doctrine->getRepository('Knp\\Bundle\\KnpBundlesBundle\\Entity\\Bundle')->findAll();
+ } else {
+ $bundles = $doctrine->getRepository('Knp\\Bundle\\KnpBundlesBundle\\Entity\\Bundle')->getStaleBundlesForIndexing();
+ }
+
+ if ($force && !$bundleName) {
+ if ($verbose) {
+ $output->writeln('Deleting existing index.');
+ }
+
+ $update = $solarium->createUpdate();
+ $update->addDeleteQuery('*:*');
+ $update->addCommit();
+
+ $solarium->update($update);
+ }
+
+ foreach ($bundles as $bundle) {
+ if ($verbose) {
+ $output->writeln('Indexing '.$bundle->getFullName().'...');
+ }
+
+ try {
+ $indexer->indexBundle($bundle);
+ } catch (\Exception $e) {
+ $output->writeln('<error>Exception: '.$e->getMessage().', skipping bundle '.$bundle->getFullName().'.</error>');
+ }
+ }
+ $doctrine->getEntityManager()->flush();
+ }
+}
View
22 src/Knp/Bundle/KnpBundlesBundle/Consumer/UpdateBundleConsumer.php
@@ -7,6 +7,7 @@
use Knp\Bundle\KnpBundlesBundle\Travis\Travis;
use Knp\Bundle\KnpBundlesBundle\Entity\Bundle;
use Knp\Bundle\KnpBundlesBundle\Entity\UserManager;
+use Knp\Bundle\KnpBundlesBundle\Indexer\SolrIndexer;
use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
@@ -15,6 +16,7 @@
use Symfony\Component\HttpKernel\Log\LoggerInterface;
use Doctrine\Common\Persistence\ObjectManager;
+
/**
* This class is a consumer which will retrieve a bundle from database
* and update everything that needs to be updated.
@@ -41,17 +43,29 @@ class UpdateBundleConsumer implements ConsumerInterface
private $users;
/**
+ * @var Knp\Bundle\KnpBundlesBundle\Travis\Travis
+ */
+ private $travis;
+
+ /**
+ * @var Knp\Bundle\KnpBundlesBundle\Indexer\SolrIndexer
+ */
+ private $indexer;
+
+ /**
* @param Doctrine\Common\Persistence\ObjectManager $em
* @param Knp\Bundle\KnpBundlesBundle\Entity\UserManager $users
- * @param string $gitRepoDir
- * @param string $gitBin
+ * @param Repo $githubRepoApi
+ * @param Travis $travis
+ * @param SolrIndexer $indexer
*/
- public function __construct(ObjectManager $em, UserManager $users, Repo $githubRepoApi, Travis $travis)
+ public function __construct(ObjectManager $em, UserManager $users, Repo $githubRepoApi, Travis $travis, SolrIndexer $indexer)
{
$this->em = $em;
$this->githubRepoApi = $githubRepoApi;
$this->travis = $travis;
$this->users = $users;
+ $this->indexer = $indexer;
}
/**
@@ -108,6 +122,8 @@ public function execute($msg)
return;
}
+ $this->indexer->indexBundle($bundle);
+
$this->updateContributors($bundle);
$this->updateKeywords($bundle);
$score = $this->em->getRepository('Knp\Bundle\KnpBundlesBundle\Entity\Score')->setScore(new \DateTime(), $bundle, $bundle->getScore());
View
19 src/Knp/Bundle/KnpBundlesBundle/Controller/BundleController.php
@@ -40,15 +40,22 @@ public function searchAction()
return $this->render('KnpBundlesBundle:Bundle:search.html.twig');
}
- $bundles = $this->getRepository('Bundle')->search($query);
+ $solarium = $this->get('solarium.client');
+ $select = $solarium->createSelect();
+ $escapedQuery = $select->getHelper()->escapePhrase($query);
- $format = $this->recognizeRequestFormat();
+ $dismax = $select->getDisMax();
+ $dismax->setQueryFields(array('name', 'description', 'keywords', 'text', 'username', 'fullName'));
+ $select->setQuery($escapedQuery);
- if ('html' == $format && count($bundles) == 1 && strtolower($bundles[0]->getName()) == strtolower($query)) {
- $params = array('username' => $bundles[0]->getUserName(), 'name' => $bundles[0]->getName());
+ $paginator = $this->get('knp_paginator');
+ $bundles = $paginator->paginate(
+ array($solarium, $select),
+ $this->get('request')->query->get('page', 1),
+ 10
+ );
- return $this->redirect($this->generateUrl('bundle_show', $params));
- }
+ $format = $this->recognizeRequestFormat();
return $this->render('KnpBundlesBundle:Bundle:searchResults.'.$format.'.twig', array(
'query' => urldecode($this->get('request')->query->get('q')),
View
27 src/Knp/Bundle/KnpBundlesBundle/Entity/Bundle.php
@@ -250,6 +250,13 @@ public static function loadValidatorMetadata(ClassMetadata $metadata)
*/
protected $symfonyVersion;
+ /**
+ * Last indexing time.
+ *
+ * @ORM\Column(type="datetime", nullable=true)
+ */
+ protected $indexedAt;
+
public function __construct($fullName = null)
{
if ($fullName) {
@@ -1052,4 +1059,24 @@ public function setSymfonyVersion($version)
{
$this->symfonyVersion = $version;
}
+
+ /**
+ * Set indexedAt
+ *
+ * @param \DateTime $indexedAt
+ */
+ public function setIndexedAt(\DateTime $indexedAt)
+ {
+ $this->indexedAt = $indexedAt;
+ }
+
+ /**
+ * Get indexedAt
+ *
+ * @return datetime $indexedAt
+ */
+ public function getIndexedAt()
+ {
+ return $this->indexedAt;
+ }
}
View
77 src/Knp/Bundle/KnpBundlesBundle/Indexer/SolrIndexer.php
@@ -0,0 +1,77 @@
+<?php
+
+namespace Knp\Bundle\KnpBundlesBundle\Indexer;
+
+use Doctrine\Bundle\DoctrineBundle\Registry;
+use Knp\Bundle\KnpBundlesBundle\Entity\Bundle;
+
+/**
+ * Indexes bundles into Solr.
+ *
+ * @author Paweł Jędrzejewski <pjedrzejewski@diweb.pl>
+ */
+class SolrIndexer
+{
+ /**
+ * @var Doctrine\Bundle\DoctrineBundle\Registry
+ */
+ protected $doctrine;
+
+ /**
+ * @var \Solarium_Client
+ */
+ protected $solarium;
+
+ /**
+ * @param Doctrine\Bundle\DoctrineBundle\Registry $doctrine
+ * @param \Solarium_Client $solarium
+ */
+ public function __construct(Registry $doctrine, \Solarium_Client $solarium)
+ {
+ $this->doctrine = $doctrine;
+ $this->solarium = $solarium;
+ }
+
+ /**
+ * Indexes single bundle.
+ *
+ * @param Knp\Bundle\KnpBundlesBundle\Entity\Bundle $bundle
+ */
+ public function indexBundle(Bundle $bundle)
+ {
+ $update = $this->solarium->createUpdate();
+ $document = $update->createDocument();
+ $this->updateDocumentFromBundle($document, $bundle, $update->getHelper());
+ $update->addDocument($document);
+ $update->addCommit();
+ $this->solarium->update($update);
+ $bundle->setIndexedAt(new \DateTime);
+
+ $this->doctrine->getEntityManager()->flush();
+ }
+
+ /**
+ * Populates document with bundle data.
+ *
+ * @param \Solarium_Document_ReadWrite $document
+ * @param Knp\Bundle\KnpBundlesBundle\Entity\Bundle $bundle
+ * @param \Solarium_Query_Helper $helper
+ */
+ private function updateDocumentFromBundle(\Solarium_Document_ReadWrite $document, Bundle $bundle, \Solarium_Query_Helper $helper)
+ {
+ $document->id = $bundle->getId();
+ $document->name = $bundle->getName();
+ $document->username = $bundle->getUsername();
+ $document->fullName = $bundle->getFullName();
+ $document->description = $bundle->getDescription();
+ $document->totalScore = $bundle->getScore();
+ $document->userGravatarHash = $bundle->getUser()->getGravatarHash();
+ $document->lastCommitAt = $helper->formatDate($bundle->getLastCommitAt());
+
+ $keywords = array();
+ foreach ($bundle->getKeywords() as $keyword) {
+ $keywords[] = $keyword->getValue();
+ }
+ $document->keywords = $keywords;
+ }
+}
View
36 src/Knp/Bundle/KnpBundlesBundle/Repository/BundleRepository.php
@@ -7,32 +7,6 @@
class BundleRepository extends EntityRepository
{
- public function search($query)
- {
- $pattern = '%'.str_replace(' ', '%', $query).'%';
-
- $qb = $this->createQueryBuilder('bundle')
- ->leftJoin('bundle.keywords', 'keyword');
-
- $qb->where($qb->expr()->orx(
- $qb->expr()->like('bundle.username', ':username'),
- $qb->expr()->like('bundle.name', ':name'),
- $qb->expr()->like('bundle.description', ':description'),
- $qb->expr()->eq('keyword.value', ':query')
- ));
-
- $qb
- ->orderBy('bundle.score', 'DESC')
- ->setParameters(array(
- 'username' => $pattern,
- 'name' => $pattern,
- 'description' => $pattern,
- 'query' => $query
- ));
-
- return $qb->getQuery()->execute();
- }
-
public function findAllSortedBy($field, $nb = null)
{
$query = $this->queryAllSortedBy($field);
@@ -143,6 +117,16 @@ public function findOneByUsernameAndName($username, $name)
}
}
+ public function getStaleBundlesForIndexing()
+ {
+ return $this->createQueryBuilder('bundle')
+ ->leftJoin('bundle.user', 'user')
+ ->where('bundle.indexedAt IS NULL OR bundle.indexedAt < bundle.updatedAt')
+ ->getQuery()
+ ->getResult()
+ ;
+ }
+
public function updateTrends()
{
// Reset trends
View
8 src/Knp/Bundle/KnpBundlesBundle/Resources/config/services.yml
@@ -17,6 +17,7 @@ parameters:
knp_bundles.imagine.gd.class: Imagine\Gd\Imagine
knp_bundles.imagine.imagick.class: Imagine\Imagick\Imagine
knp_bundles.imagine.gmagick.class: Imagine\Gmagick\Imagine
+ knp_bundles.indexer.solr.class: Knp\Bundle\KnpBundlesBundle\Indexer\SolrIndexer
services:
knp_bundles.updater:
@@ -88,6 +89,12 @@ services:
- @knp_bundles.github_client
- @knp_bundles.output
+ knp_bundles.indexer.solr:
+ class: %knp_bundles.indexer.solr.class%
+ arguments:
+ - @doctrine
+ - @solarium.client
+
knp_bundles.consumer.update_bundle:
class: %knp_bundles.consumer.update_bundle.class%
arguments:
@@ -95,6 +102,7 @@ services:
- @knp_bundles.user.manager
- @knp_bundles.github_repository_api
- @knp_bundles.travis
+ - @knp_bundles.indexer.solr
calls:
- [ setLogger, [@logger]]
View
58 src/Knp/Bundle/KnpBundlesBundle/Resources/solr/schema.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<schema name="knpbundles.com" version="1.4">
+ <types>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true" />
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true" />
+ <fieldtype name="binary" class="solr.BinaryField" />
+
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
+
+ <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0" />
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0" />
+
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true" />
+ <filter class="solr.LowerCaseFilterFactory" />
+ </analyzer>
+ </fieldType>
+
+ <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
+ </types>
+
+ <fields>
+ <field name="id" type="string" indexed="true" stored="true" required="true" />
+ <field name="name" type="text_general" indexed="true" stored="true" />
+ <field name="username" type="text_general" indexed="true" stored="true" />
+ <field name="fullName" type="text_general" indexed="true" stored="true" />
+ <field name="description" type="text_general" indexed="true" stored="true" />
+ <field name="totalScore" type="int" indexed="false" stored="true" />
+ <field name="userGravatarHash" type="string" indexed="false" stored="true" />
+ <field name="lastCommitAt" type="date" indexed="false" stored="true" />
+ <field name="keywords" type="text_general" indexed="true" stored="true" multiValued="true" />
+
+ <field name="text" type="text_general" indexed="true" stored="false" multiValued="true" />
+ </fields>
+
+ <uniqueKey>id</uniqueKey>
+
+ <defaultSearchField>text</defaultSearchField>
+
+ <solrQueryParser defaultOperator="OR" />
+
+ <copyField source="name" dest="text" />
+ <copyField source="username" dest="text" />
+ <copyField source="fullName" dest="text" />
+ <copyField source="description" dest="text" />
+ <copyField source="keywords" dest="text" />
+</schema>
View
25 src/Knp/Bundle/KnpBundlesBundle/Resources/views/Bundle/searchBigList.html.twig
@@ -0,0 +1,25 @@
+<ul class="big-list">
+ {% for bundle in bundles %}
+ <li class="bundle {% if loop.first %}first{% endif %}">
+ <div class="bundlerank">
+ {{ bundle.totalScore }}
+ </div>
+ <div class="generals">
+ <div class="bundle-title">
+ <a href="{{ path('bundle_show', {'username': bundle.username, 'name': bundle.name}) }}" class="name">{{ bundle.name }}</a>
+ </div>
+ <p class="description">{{ bundle.description|default('') }}</p>
+
+ <ul class="details">
+ <li>{% trans %}bundles.list.by{% endtrans %} <a href="{{ path('user_show', {'name': bundle.username }) }}" class="owner">{{ bundle.username }}</a></li>
+ <li>{% trans %}bundles.list.lastCommit{% endtrans %} <span>{{ time_diff(bundle.lastCommitAt) }}</span></li>
+ </ul>
+ </div>
+ <div class="img">
+ {% if bundle.userGravatarHash|default(null) %}
+ <a href="{{ path('bundle_show', {'username': bundle.username, 'name': bundle.name}) }}"><img src="{{ gravatar_hash(bundle.userGravatarHash, 50, none, 'identicon') }}" alt="{{ bundle.name }}" /></a>
+ {% endif %}
+ </div>
+ </li>
+ {% endfor %}
+</ul>
View
3  src/Knp/Bundle/KnpBundlesBundle/Resources/views/Bundle/searchResults.html.twig
@@ -15,7 +15,8 @@
<div class="content-title">
<h1>{{ bundles|length }} Bundle{{ bundles|length > 1 ? 's' : '' }}</h1>
</div>
- {% include 'KnpBundlesBundle:Bundle:bigList.html.twig' with {'bundles': bundles} %}
+ {% include 'KnpBundlesBundle:Bundle:searchBigList.html.twig' with {'bundles': bundles} %}
</section>
+ {{ bundles.render()|raw }}
</div>
{% endblock %}

No commit comments for this range

Something went wrong with that request. Please try again.