Skip to content

Commit

Permalink
last checkpoint: include cities in the nested set model calculation
Browse files Browse the repository at this point in the history
Previously, the contents of the country files were read and inserted into the database in chunks of 1000.
Then the Nested Set Model is built by reading the hierarchy.txt file and updating the records in the database(slow).

At this point, the Nested Set Model is built first by reading the hierarchy.txt file, and combining it with the contents
of the admin2Codes.txt file which contains the hierarchy for cities(Even though geonames.org has up to ADM5 division,
it cuts-off at ADM2 for cities). After the nestedSet is built, the country files contents is mapped into a Geoname with
the nestedSet properties(_lft, _rgt, depth and parent_id) and inserted into the database
  • Loading branch information
Parables committed Jun 1, 2023
1 parent 7e55142 commit 0d1585b
Show file tree
Hide file tree
Showing 12 changed files with 601 additions and 262 deletions.
88 changes: 45 additions & 43 deletions src/Actions/BuildNestedSetModelAction.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace Parables\Geo\Actions;

use Illuminate\Support\Arr;
use Illuminate\Support\LazyCollection;
use Parables\Geo\Actions\Concerns\HasToastable;

class BuildNestedSetModelAction
Expand All @@ -13,24 +14,30 @@ class BuildNestedSetModelAction

/**
* @param array<int|string,mixed> $hierarchy
* @return array
* @return \Illuminate\Support\LazyCollection
*/
public function execute(array $hierarchy = [], bool $nestChildren = false): array
public function execute(array $hierarchy = [], bool $nestChildren = false): LazyCollection
{
ini_set('memory_limit', -1);

if (empty($hierarchy)) {
return [];
$this->toastable->toast('Hierarchy is empty... Skipping ...', 'error');
return LazyCollection::empty();
}

return $this->buildTree(hierarchy: $hierarchy, nestChildren: $nestChildren);
return LazyCollection::make(function () use ($hierarchy, $nestChildren) {
$index = 1;
yield from $this->buildTree($hierarchy, $index, $nestChildren);
});
}


/**
* @param array<int|string,mixed> $hierarchy
* @return array
* @return \Generator
*/
public function buildTree(array $hierarchy, int &$index = 1, bool $nestChildren = false): array
public function buildTree(array $hierarchy, int &$index = 1, bool $nestChildren = false): \Generator
{
$this->toastable->toast('Building Root Node ...');
$rootId = array_key_first($hierarchy);
$depth = 0;

Expand All @@ -42,71 +49,66 @@ public function buildTree(array $hierarchy, int &$index = 1, bool $nestChildren
'parent_id' => null,
];

$children = $this->buildNodes(
hierarchy: $hierarchy,
parentId: $rootId,
index: $index,
depth: $depth,
nestChildren: $nestChildren
);
$this->toastable->toast('Building Children Nodes ...');
$children = $this->buildNodes($hierarchy, $rootId, $index, $depth, $nestChildren);

$root['_rgt'] = $index++;

$this->toastable->toast('Tree Built successfully...');
if ($nestChildren) {
$root['children'] = $children;
return $root;
yield $rootId => $root;
} else {
yield $rootId => $root;
yield from $children;
}
return [$rootId => $root] + $children;
// return array_merge([$rootId => $root], $children);
//$children[$rootId] = $root;
//return $children;
}

/**
* @param array<int|string,mixed> $hierarchy
* @return array
* @return \Generator
*/
public function buildNodes(array $hierarchy, string|int $parentId, int &$index, int $depth, bool $nestChildren = false): array
public function buildNodes(array $hierarchy, string|int $parentId, int &$index, int $depth, bool $nestChildren = false): \Generator
{
$depth += 1;
$result = [];
$children = [];

foreach ($this->children($hierarchy, $parentId) as $id) {
$node =
[
'id' => $id,
'_lft' => $index++,
'_rgt' => null,
'depth' => $depth,
'parent_id' => $parentId,
];

$children = $this->buildNodes(
hierarchy: $hierarchy,
parentId: $id,
index: $index,
depth: $depth,
nestChildren: $nestChildren
);
$node = [
'id' => $id,
'_lft' => $index++,
'_rgt' => null,
'depth' => $depth,
'parent_id' => $parentId,
];

// $this->toastable->toast('Getting sub nodes for: ' . $id);
$children = $this->buildNodes($hierarchy, $id, $index, $depth, $nestChildren);

$node['_rgt'] = $index++;

if ($nestChildren) {
$node['children'] = $children;
$result[$id] = $node;
yield $id => $node;
} else {
$result = $result + [$id => $node] + $children;
yield $id => $node;
yield from $children;
}
}
return $result;

// $this->toastable->toast('Done.');
}

/**
* @param array<int|string, mixed> $hierarchy
*/
public function children(array $hierarchy, string|int $parentId): array
{
return Arr::wrap($hierarchy[$parentId] ?? []);
// $this->toastable->toast('Getting children for parentId: ' . $parentId);
if (array_key_exists($parentId, $hierarchy)) {
return $hierarchy[$parentId] ?? [];
}
return [];
}

/**
Expand All @@ -115,7 +117,7 @@ public function children(array $hierarchy, string|int $parentId): array
* @param bool $includeLeaves
* @return mixed
*/
function flattenTree(array $tree, array &$result = [], bool $includeLeaves = false)
public function flattenTree(array $tree, array &$result = [], bool $includeLeaves = false)
{
foreach ($tree as $key => $value) {
if (is_array($value) && !empty($value)) {
Expand Down
51 changes: 41 additions & 10 deletions src/Actions/BuildNestedSetModelActionTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace Parables\Geo\Actions;

use Illuminate\Support\Arr;
use Parables\Geo\Actions\Fixtures\Toastable;

it('populates geonames', function () {
Expand Down Expand Up @@ -60,23 +61,53 @@
'Sun Dresses'
]
];
$toastable = new Toastable();

$buildNestedSetModelAction = (new BuildNestedSetModelAction)->toastable(new Toastable);
$buildNestedSetModelAction = (new BuildNestedSetModelAction)->toastable($toastable);

$result = $buildNestedSetModelAction->flattenTree($treeArray);
expect($flatTree)->toBe($result);
// print_r($result);
// $result = $buildNestedSetModelAction->flattenTree($treeArray);
// expect($flatTree)->toBe($result);
//
// $data = $buildNestedSetModelAction->execute(hierarchy: $flatTree, nestChildren: true);
// print_r($data);
//
// $data = $buildNestedSetModelAction->execute(hierarchy: $flatTree, nestChildren: false);
// print_r($data);
//

$data = $buildNestedSetModelAction->execute(hierarchy: $flatTree, nestChildren: true);
$data = $buildNestedSetModelAction->execute();
print_r($data);
print_r(count($data));

$data = $buildNestedSetModelAction->execute(hierarchy: $flatTree, nestChildren: false);
print_r($data);
$cacheFile = storage_path('/geo/countries.json');
$fileNames = array_map(
fn ($fileName) => $fileName . '.txt',
array_keys(Arr::wrap(json_decode(file_get_contents($cacheFile), associative: true)))
);
$contentsOfGeonameFiles = (new ReadFilesAction)->toastable(new Toastable)->execute($fileNames);


$data = $buildNestedSetModelAction->execute();
print_r($data);
print_r(count($data));
$toastable->toast('Getting hierarchy...');
$hierarchy = (new GetHierarchyAction)
->toastable($toastable)
// ->hierarchy();
->execute(contentsOfGeonameFiles: $contentsOfGeonameFiles);
// $this->writeToFile(fileName: storage_path('geo/hierarchy.json'), content: $hierarchy);

$toastable->toast('Building Nested Set Model...');
$nestedSet = (new BuildNestedSetModelAction)
->toastable($toastable)
->execute(hierarchy: $hierarchy, nestChildren: false);
//
// write to cache
$fileName = storage_path('geo/nestedSet.json');
$stream = fopen($fileName, 'w');
fwrite(stream: $stream, data: json_encode($nestedSet->all(), JSON_PRETTY_PRINT));
fclose($stream);

print_r($nestedSet->count());
$toastable->toast("Nested Set Model is complete");


expect('hi')->toBe('hi');
});
145 changes: 145 additions & 0 deletions src/Actions/GenNestedSetModelAction.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
<?php

declare(strict_types=1);

namespace Parables\Geo\Actions;

use Illuminate\Support\Arr;
use Parables\Geo\Actions\Concerns\HasToastable;

class GenNestedSetModelAction
{
use HasToastable;

/**
* @param array<int|string,mixed> $hierarchy
* @return array
*/
public function execute(array $hierarchy = [], bool $nestChildren = false): array
{
ini_set('memory_limit', -1);

if (empty($hierarchy)) {
$this->toastable->toast('Hierarchy is empty... Skipping ...', 'error');
return [];
}

return $this->buildTree(hierarchy: $hierarchy, nestChildren: $nestChildren);
}


/**
* @param array<int|string,mixed> $hierarchy
* @return array
*/
public function buildTree(array $hierarchy, int &$index = 1, bool $nestChildren = false): array
{
$this->toastable->toast('Building Root Node ...');
$rootId = array_key_first($hierarchy);
$depth = 0;

$root = [
'id' => $rootId,
'_lft' => $index++,
'_rgt' => null,
'depth' => $depth,
'parent_id' => null,
];

$this->toastable->toast('Building Children Nodes ...');
$children = $this->buildNodes(
hierarchy: $hierarchy,
parentId: $rootId,
index: $index,
depth: $depth,
nestChildren: $nestChildren
);

$root['_rgt'] = $index++;

if ($nestChildren) {
$root['children'] = $children;
return $root;
}
$this->toastable->toast('Tree Built successfully...');
return [$rootId => $root] + $children;
// return array_merge([$rootId => $root], $children);
//$children[$rootId] = $root;
//return $children;
}

/**
* @param array<int|string,mixed> $hierarchy
* @return array
*/
public function buildNodes(array $hierarchy, string|int $parentId, int &$index, int $depth, bool $nestChildren = false): array
{
$depth += 1;
$result = [];
$children = [];
foreach ($this->children($hierarchy, $parentId) as $id) {
$node =
[
'id' => $id,
'_lft' => $index++,
'_rgt' => null,
'depth' => $depth,
'parent_id' => $parentId,
];

$this->toastable->toast('Getting sub nodes for: ' . $id);
$children = $this->buildNodes(
hierarchy: $hierarchy,
parentId: $id,
index: $index,
depth: $depth,
nestChildren: $nestChildren
);

$node['_rgt'] = $index++;

if ($nestChildren) {
$node['children'] = $children;
$result[$id] = $node;
} else {
$result = $result + [$id => $node] + $children;
}
}
$this->toastable->toast('Done.');
return $result;
}

/**
* @param array<int|string, mixed> $hierarchy
*/
public function children(array $hierarchy, string|int $parentId): array
{
$this->toastable->toast('Getting children for parentId: ' . $parentId);
if (array_key_exists($parentId, $hierarchy)) {
return $hierarchy[$parentId] ?? [];
}
return [];
}

/**
* @param array $tree
* @param array $result
* @param bool $includeLeaves
* @return mixed
*/
function flattenTree(array $tree, array &$result = [], bool $includeLeaves = false)
{
foreach ($tree as $key => $value) {
if (is_array($value) && !empty($value)) {
$result[$key] = array_keys($value);
$this->flattenTree($value, $result, $includeLeaves);
} else {
if ($includeLeaves) {
$result[$key] = [];
}
}
}

return $result;
}
}

0 comments on commit 0d1585b

Please sign in to comment.