Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #191 from borazslo/master 
Merge to prod
  • Loading branch information
@briff
briff committed Jan 11, 2021
2 parents d6217ea + 5043242 commit 3f52235
Show file tree
Hide file tree
Showing 32 changed files with 499 additions and 118 deletions.
26 changes: 16 additions & 10 deletions app/Console/Commands/UpdateTextsCommand.php
View file
Expand Up @@ -74,7 +74,6 @@ public function fire()
if (!$this->option('nohunspell')) {
$this->testHunspell();
}

$abbrev = $this->choice('Melyik fordítást töltsük be?', ['BD', 'KG', 'KNB', 'RUF', 'UF', 'SZIT', 'STL']);
$this->verifyAbbrev($abbrev);

Expand Down Expand Up @@ -146,7 +145,13 @@ public function fire()
$bookNumber = $row[$columns[$abbrev]['gepi']];
$rov = $row[$columns[$abbrev]['rov']];
if (!isset($bookAbbrev2Id[$rov]) AND ($rov != '-' AND $rov != '')) {
$badAbbrevs[] = $rov;
$book = $this->bookRepository->getByAbbrev($rov, $translation);
if ($book) {
$bookNumber2Id[$bookNumber] = $book->id;
} else {
$badAbbrevs[]= $rov;
}

} else if ($rov != '-' AND $rov != '') {
$bookNumber2Id[$bookNumber] = $bookAbbrev2Id[$rov];
}
Expand Down Expand Up @@ -273,7 +278,7 @@ private function stem($verse, $pipes)
}
} else {
$cachedStems = $stems->unique();
Cache::put("hunspell_{$word}", $cachedStems, 525948);
Cache::put("hunspell_{$word}", $cachedStems, 60*24);
$verseroots = $verseroots->merge($stems);
$this->newStems++;
break;
Expand Down Expand Up @@ -334,12 +339,12 @@ private function readLines($verseRowIterator, $cols, $fields, $translation, $boo
$progressBar->setRedrawFrequency(25);
$progressBar->setBarWidth(24);
$progressBar->setFormat("[%bar%] %message%");
$verseRowIterator->rewind();
$verseRowIterator->next();
$verseRowIterator->next();
$rowNumber = 0;
while ($verseRowIterator->valid()) {
$row = $verseRowIterator->current();
foreach ($verseRowIterator as $row) {
if ($rowNumber < 2) {
$rowNumber++;
continue;
}
if (empty($row[$cols[$fields['gepi']]])) {
break;
}
Expand All @@ -364,11 +369,12 @@ private function readLines($verseRowIterator, $cols, $fields, $translation, $boo
if (isset($books_gepi2id[$values['book_number']])) {
$values['book_id'] = $books_gepi2id[$values['book_number']];
} else {
App::abort(500, 'Valami gond van a books id/gepi párossal!');
print_r($values);
print_r($books_gepi2id);
App::abort(500, 'Nincs meg a book number a gepi2id listában');
}
$inserts[$rowNumber] = $values;
}
$verseRowIterator->next();
$rowNumber++;
$progressBar->setMessage("$rowNumber - {$values['gepi']} - új szavak: {$this->newStems}");
$progressBar->advance();
Expand Down
1 change: 1 addition & 0 deletions app/Http/Controllers/Api/ApiController.php
View file
Expand Up @@ -176,6 +176,7 @@ public function getSearch($text)
$params->text = $text;
$results = $this->searchService->getDetailedResults($params);
$refResult = $this->searchRef($text, false);
unset($results['resultsByBookNumber']); // don't use new search results view in the API yet
return $this->formatJsonResponse(["refResult" => $refResult, "fullTextResult" => $results]);
}

Expand Down
34 changes: 28 additions & 6 deletions app/Http/Controllers/Display/TextDisplayController.php
View file
Expand Up @@ -128,20 +128,42 @@ private function bookView($translationAbbrev, CanonicalReference $canonicalRef)
$translatedRef = $this->referenceService->translateReference($canonicalRef, $translation->id);
$book = $this->bookRepository->getByAbbrevForTranslation($translatedRef->bookRefs[0]->bookId, $translation->id);
if ($book) {

$chapters = [];


$verses = $this->verseRepository->getVerses($book->id);
foreach ($verses as $verse) {
$type = $verse->getType();

if ( preg_match('/^heading[0-9]{1}/',$type)) {
$verseContainer = new VerseContainer($book);
$verseContainer->addVerse($verse);
$headings = $this->textService->getHeadings([$verseContainer])[0];

$chapters[$verse['chapter']]['headings'][$verse['numv']] = array (
'gepi' => $verse->gepi,
'level' => key($headings),
'text' => $headings[key($headings)]
);
}
}


$firstVerses = $this->verseRepository->getLeadVerses($book->id);
$groupedVerses = [];
foreach ($firstVerses as $verse) {
$type = $verse->getType();
if ($type == 'text' || $type == 'poemLine') {
$verseContainer = new VerseContainer($book);
$verseContainer->addVerse($verse);
$oldText = "";
if (array_has($groupedVerses, $verse['chapter'])) {
if (array_has($groupedVerses[$verse['chapter']], $verse['numv'])) {
$oldText = $groupedVerses[$verse['chapter']][$verse['numv']];
$oldText = "";
if(isset($chapters[$verse['chapter']]['leadVerses'])) {
if (array_has($chapters[$verse['chapter']]['leadVerses'], $verse['numv'])) {
$oldText = $chapters[$verse['chapter']]['leadVerses'][$verse['numv']];
}
}
$groupedVerses[$verse['chapter']][$verse['numv']] = $oldText . $this->textService->getTeaser([$verseContainer]);
$chapters[$verse['chapter']]['leadVerses'][$verse['numv']] = $oldText . $this->textService->getTeaser([$verseContainer]);
}
}
$allTranslations = $this->translationRepository->getAllOrderedByDenom();
Expand All @@ -150,7 +172,7 @@ private function bookView($translationAbbrev, CanonicalReference $canonicalRef)
'translation' => $translation,
'reference' => $translatedRef,
'book' => $book,
'groupedVerses' => $groupedVerses,
'chapters' => $chapters,
'translations' => $allTranslations,
'translationLinks' => $allTranslations->map(
function ($translation) use ($canonicalRef, $bookTranslations) {
Expand Down
2 changes: 2 additions & 0 deletions app/Http/Controllers/Display/VerseParsers/VerseData.php
View file
Expand Up @@ -22,6 +22,8 @@ class VerseData
public $book;
public $poemLines;

public $elements = [];

function __construct($chapter, $numv)
{
$this->chapter = $chapter;
Expand Down
1 change: 1 addition & 0 deletions app/Http/Controllers/Search/SearchController.php
View file
Expand Up @@ -210,6 +210,7 @@ private function createFullTextSearchParams($form)
}
$searchParams->bookNumbers = $this->extractBookNumbers($form);
$searchParams->synonyms = true;
$searchParams->grouping = $form->grouping;
return $searchParams;
}

Expand Down
2 changes: 1 addition & 1 deletion app/Service/Search/FullTextSearchParams.php
View file
Expand Up @@ -12,7 +12,7 @@ class FullTextSearchParams {
public $translationId;
public $bookNumbers;
public $limit;
public $groupByVerse;
public $grouping;
public $synonyms;

}
1 change: 1 addition & 0 deletions app/Service/Search/FullTextSearchResult.php
View file
Expand Up @@ -10,5 +10,6 @@ class FullTextSearchResult {

public $hitCount;
public $verseIds;
public $verses;

}
155 changes: 151 additions & 4 deletions app/Service/Search/SearchService.php
View file
Expand Up @@ -46,7 +46,7 @@ function getSuggestionsFor($term)
$searchParams = new FullTextSearchParams;
$searchParams->text = $term;
$searchParams->limit = 10;
$searchParams->groupByVerse = true;
$searchParams->grouping = 'verse';
$searchParams->synonyms = true;
$sphinxSearcher = $this->searcherFactory->createSearcherFor($searchParams);
$sphinxResults = $sphinxSearcher->get();
Expand Down Expand Up @@ -102,8 +102,124 @@ public function getDetailedResults($searchParams)

private function handleFullTextResults($sphinxResults, FullTextSearchParams $params)
{

$sortedVerses = $this->verseRepository->getVersesInOrder($sphinxResults->verseIds);
//echo "<pre>".print_r($sphinxResults ,1)."</pre>";


$defaultTranslation = $this->translationRepository->getDefault();

/* beginning of new part */
$results = [];
$translations = [];
foreach($sphinxResults->verses as $id => $verse) {
switch ($params->grouping) {
case 'book':
$limit = 3;
break;
case 'chapter':
$limit = 6;
break;
case 'verse':
$limit = 9;
break;
default:
$limit = 9;
break;
}
$key = substr($verse['attrs']['gepi'],0,$limit);
if(!array_key_exists($key, $results)) {
$results[$key] = ['weights' => [], 'translations' => [$defaultTranslation->abbrev => [] ] ];
}
if(!array_key_exists($verse['attrs']['trans'], $translations)) {
$translations[$verse['attrs']['trans']] = $this->translationRepository->getById($verse['attrs']['trans']);
}
$trans = $translations[$verse['attrs']['trans']];
if(!array_key_exists($trans['abbrev'], $results[$key]['translations']) or $results[$key]['translations'][$trans['abbrev']] == array() ) {
//$book = $this->bookRepository->getByNumberForTranslation($verse['attrs']['book_number'],$verse['attrs']['trans']);
$results[$key]['translations'][$trans['abbrev']] = [
'verseIds' => [],
'verses' => [],
'trans' => $trans,
'book' => $sortedVerses[$id]->book
];
}
//echo "<pre>"; print_r($sortedVerses[$id]); exit;
$results[$key]['weights'][] = $verse['weight'];
$results[$key]['translations'][$trans['abbrev']]['verseIds'][] = $id;
$results[$key]['translations'][$trans['abbrev']]['verses'][] = $sortedVerses[$id];
}

// echo "<pre> ".print_r($results,1)."</pre>";/**/

foreach($results as $key => $result) {
rsort($result['weights']);
//echo "<pre>"; print_R($result['weights']); echo "</pre>";
$results[$key]['weight'] = reset($result['weights']) + sqrt(array_sum($result['weights'])); // log10()

foreach($result['translations'] as $abbrev => $group ) {
if($group == []) unset($results[$key]['translations'][$abbrev]);
else {
$gepis = array_column($group['verses'],'gepi');
array_multisort($gepis, SORT_ASC, $results[$key]['translations'][$abbrev]['verses']);

$currentNumv = false;
$currentChapter = false;
foreach ($group['verses'] as $k => $verse) {
$verseData = [];
$verseData['chapter'] = $verse->chapter;
$verseData['numv'] = $verse->numv;
$verseData['text'] = preg_replace('/<[^>]*>/', ' ', $verse->verse);
if ($verse->headings) { // Ez nem üzemel, mert nem volt getParsedVerse mert nem volt VerseContainer, mert book-onként kall azt csináni.
echo "bizony";
foreach ($verse->headings as $heading) {
$verseData['text'] .= "<small>".$heading . '</small> ';
}
}

if($verse->chapter > $currentChapter ) {
$verseData['chapterStart'] = true;
$currentNumv = $verse->numv;
}
$currentChapter = $verse->chapter;

if($verse->numv > $currentNumv + 1 AND $currentNumv = $verse->numv) {
$verseData['ellipseBefore'] = true;
}
$currentNumv = $verse->numv;

$results[$key]['translations'][$abbrev]['verses'][$k] = $verseData;

}












}
}
}
$weights = array_column($results, 'weight');
array_multisort($weights, SORT_DESC, $results);

//echo "<pre> ".print_r($results,1)."</pre>";/**/

/* end of new part */

$resultsByBookNumber = $results;


/* original */

$verseContainers = $this->groupVersesByBook($sortedVerses, $params->translationId);
//echo "<pre>".print_r($verseContainers,1)."</pre>";
$results = [];
$chapterCount = 0;
$verseCount = 0;
Expand All @@ -120,7 +236,7 @@ private function handleFullTextResults($sphinxResults, FullTextSearchParams $par
$verseData['text'] = '';
if ($verse->headings) {
foreach ($verse->headings as $heading) {
$verseData['text'] .= $heading . ' ';
$verseData['text'] .= "<small>".$heading . '</small> ';
}
}
if ($verse->getText()) {
Expand All @@ -131,7 +247,7 @@ private function handleFullTextResults($sphinxResults, FullTextSearchParams $par
$verseCount++;
}
$chapterCount += count($result['chapters']);
if (!$params->groupByVerse) {
if ($params->grouping == 'chapter') {
foreach ($result['chapters'] as $chapterNumber => $verses) {
usort($verses, function ($verseData1, $verseData2) {
if ($verseData1['numv'] == $verseData2['numv']) {
Expand All @@ -152,9 +268,40 @@ private function handleFullTextResults($sphinxResults, FullTextSearchParams $par
$results[] = $result;
}
}
return ['results' => $results, 'hitCount' => $params->groupByVerse ? $verseCount : $chapterCount];
//echo "<pre>".print_r($results,1)."</pre>";
if($params->grouping == 'verse') $hitCount = $verseCount;
else $hitCount = $chapterCount;
return ['resultsByBookNumber' => $resultsByBookNumber,'results' => $results, 'hitCount' => $hitCount ];
}

private function groupVersesByBookNumber($sortedVerses, $groupByVerse = false)
{
$verseContainers = [];
foreach ($sortedVerses as $verse) {
$book = $verse->book;
if($groupByVerse OR 4==4) $key = $verse->gepi;
else $key = $book->number."_".$verse->chapter;

if (!array_key_exists($key, $verseContainers)) {
$verseContainers[$key] = [];
}
if (!array_key_exists($book->translation_id, $verseContainers[$key])) {
$verseContainers[$key][$book->translation_id] = [
'translation' => $verse->translation,
'book' => $book,
'chapter' => $verse->chapter,
'numv' => $verse->numv,
'verses' => new VerseContainer($book)
];
}

$book->translation_id . '/' . $book->abbrev ;

$verseContainer = $verseContainers[$key][$book->translation_id]['verses'];
$verseContainer->addVerse($verse);
}
return $verseContainers;
}
private function groupVersesByBook($sortedVerses, $translationId)
{
$verseContainers = [];
Expand Down

3 comments on commit 3f52235

@borazslo
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sajnos hiba csúszott a dologba. A részeletes kereső megadta magát.

Addig megtaláltam, hogy mivel a sphinx config-ban jelentősen turkáltam, ezért az /etc/sphinxsearch/sphinx_production.conf -ot is meg kell változtatni. A staging-ét hoztam át a listen portját megváltoztatva. (Ez azért még gond mert a Line18-ból kikerült az AND részlet, ami kezelte, hogy az élesen ne minden fordításban keressen automatikusan.)

Reindex után újraindult sphinx-el a gyorskereső már tökéletesen működik.

De a részletes kereső elszáll ilyesmikkel: production.DEBUG: Book not found in database: Iz 40,31 Ami furcsa. Staging jól üzemel.

Mit hagyhattam ki?

@briff
Copy link
Collaborator Author

@briff briff commented on 3f52235 Jan 13, 2021 via email

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@borazslo
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ennyike: 19a6d64 . Upsz. (Bár azt nem teljesen értem, hogy staging miért ment ha production nem. De azt igen, hogy így jobb.)

Please sign in to comment.