Locations refetched

.gitignore

@@ -2,4 +2,5 @@ vendor/
 composer.lock
 /build
 !build/phpcs.xml
-!build/phpmd.xml
+!build/phpmd.xml
+.idea

composer.json

@@ -13,15 +13,15 @@
     "type": "library",
     "require": {
 		"php": ">=7.0",
+        "ext-json": "*",
         "illuminate/support": "5.1.*",
         "guzzlehttp/guzzle": "~6.0",
         "symfony/dom-crawler": "2.7.*",
         "symfony/var-dumper": "2.7.*"
     },    
     "require-dev": {
-		    "phpunit/phpunit": "6.*",
-        "squizlabs/php_codesniffer": "3.*",
-        "phpmd/phpmd": "@stable"
+        "phpunit/phpunit": "6.*",
+        "squizlabs/php_codesniffer": "3.*"
     },
     "autoload": {
         "psr-4": {
@@ -32,11 +32,6 @@
         ]
     },
     "scripts": {
-        "phpcs-src": "./vendor/bin/phpcs --standard=PSR2 --exclude=PSR2.Classes.PropertyDeclaration,Generic.Files.LineLength src/",
-        "phpmd": "./vendor/bin/phpmd src/ text ./build/phpmd.xml",
-        "test-all": [
-            "@phpcs-src",
-            "@phpmd"
-        ]
+        "phpcs-src": "./vendor/bin/phpcs --standard=PSR2 --exclude=PSR2.Classes.PropertyDeclaration,Generic.Files.LineLength src/"
     }
 }

example.php

@@ -53,6 +53,7 @@
  */
 try {
     $prayerTimes = $prayerTimesService->get('turkiye', 'amasya', 'merzifon');
+    //$prayerTimes = $prayerTimesService->get('afganistan', 'afganistan', 'chandler');
 } catch (\Exception $e) {
     dd($e);
 }

get_locations.php

@@ -0,0 +1,26 @@
+<?php
+
+use UmutKirgoz\PrayerTimes\Services\LocationsCrawlerService;
+
+include ('vendor/autoload.php');
+
+error_reporting(E_ALL);
+ini_set('display_errors', 1);
+
+
+$crawlerService = new LocationsCrawlerService();
+
+$crawlerService->storeLocations();
+exit;
+
+//$countries = $crawlerService->getCountries();
+//dd($countries);
+
+//Almanya
+//$cities = $crawlerService->getCities(13);
+//dd($cities);
+
+//Almanya / Bayern
+$towns = $crawlerService->getTowns(13, 851);
+
+dd($towns);

src/Repositories/LocationsRepository.php

@@ -1,6 +1,8 @@
 <?php
 namespace UmutKirgoz\PrayerTimes\Repositories;
 
+use Illuminate\Support\Collection;
+
 /**
  * Manages the locations
  * Class LocationsRepository
@@ -46,7 +48,7 @@ public function get($countrySlug, $citySlug = null, $townSlug = null)
 
     /**
      * Returns all countries
-     * @return static
+     * @return Collection
      */
     public function getCountries()
     {
@@ -79,7 +81,7 @@ public function getCities($country, $citySlug = null)
      * Returns the towns of given cities
      * @param Collection $cities
      * @param string|null $townSlug
-     * @return static
+     * @return Collection
      */
     public function getTowns($cities, $townSlug = null)
     {

src/Services/LocationsCrawlerService.php

@@ -0,0 +1,168 @@
+<?php
+namespace UmutKirgoz\PrayerTimes\Services;
+
+use GuzzleHttp\Client;
+use Stringy\StaticStringy;
+use Symfony\Component\DomCrawler\Crawler;
+
+class LocationsCrawlerService
+{
+    /**
+     * @var Client Client
+     */
+    protected $httpClient;
+
+    protected $locationsDb;
+
+    public function __construct()
+    {
+        $this->locationsDb = dirname(__FILE__)  . '/../Db/locations-new.json';
+
+        $this->httpClient = new Client([
+            'headers'   =>  [
+                'User-Agent'    =>  $this->getRandomUserAgent()
+            ]
+        ]);
+    }
+
+    public function storeLocations()
+    {
+        $data = [];
+        $countries = $this->getCountries();
+
+        foreach ($countries as $country) {
+            $data[] = $country;
+
+
+
+            $cities = $this->getCities($country['id']);
+            if ($country['id'] == 166) {
+                dd($country, $cities);
+            }
+            foreach ($cities as $city) {
+                $hasTown = $city['has_town'];
+                unset($city['has_town']);
+                $data[] = $city;
+                if ($hasTown === false) {
+                    continue;
+                }
+                $towns = $this->getTowns($country['id'], $city['id']);
+                foreach ($towns as $town) {
+                    $data[] = $town;
+                }
+            }
+        }
+        $json = json_encode($data);
+        file_put_contents($this->locationsDb, $json);
+    }
+
+    public function crawlLocations()
+    {
+        $this->countries = $this->getCountries();
+    }
+
+    public function getCountries()
+    {
+        $url = "https://namazvakitleri.diyanet.gov.tr/tr-TR";
+
+        $content = $this->get($url);
+
+        $crawler = new Crawler($content);
+
+        return $crawler->filterXPath('//select[contains(concat(" ",normalize-space(@class)," ")," country-select ")]//option')->each(function (Crawler $node) {
+            $name = $node->text();
+            $slug = StaticStringy::slugify($name);
+            return [
+                'id'    =>  $node->attr('value'),
+                'parent_id' =>  "0",
+                'type' =>  'country',
+                'name'    =>  $name,
+                'slug'    =>  $slug,
+            ];
+        });
+    }
+
+    public function getCities($countryId)
+    {
+        $url = 'https://namazvakitleri.diyanet.gov.tr/tr-TR/home/GetRegList?ChangeType=country&CountryId='.$countryId.'&Culture=tr-TR';
+
+        $content = $this->get($url);
+
+        $data = json_decode($content);
+
+        if (!isset($data->StateList)) {
+            throw new \Exception('Cannot access data');
+        }
+
+        $resultSet = [];
+        foreach ($data->StateList as $state) {
+            $name = $state->SehirAdi;
+            $slug = StaticStringy::slugify($name);
+            $resultSet[] = [
+                'id'    =>  $state->SehirID,
+                'parent_id'    =>  $countryId,
+                'type' =>  'city',
+                'name'    =>  $name,
+                'slug'    =>  $slug,
+                'has_town' =>  (isset($data->HasStateList) && $data->HasStateList === true)
+            ];
+        }
+        return $resultSet;
+    }
+
+    public function getTowns($countryId, $cityId)
+    {
+        $url = "https://namazvakitleri.diyanet.gov.tr/tr-TR/home/GetRegList?ChangeType=state&CountryId=".$countryId
+            ."&Culture=tr-TR&StateId=" . $cityId;
+
+        $content = $this->get($url);
+
+        $data = json_decode($content);
+
+        if (!isset($data->StateRegionList)) {
+            throw new \Exception('Cannot access data');
+        }
+
+        $resultSet = [];
+        foreach ($data->StateRegionList as $state) {
+            $name = $state->IlceAdi;
+            $slug = StaticStringy::slugify($name);
+            $resultSet[] = [
+                'id'    =>  $state->IlceID,
+                'parent_id'    =>  $cityId,
+                'type' =>  'town',
+                'name'    =>  $name,
+                'slug'    =>  $slug
+            ];
+        }
+        return $resultSet;
+    }
+
+    /**
+     * Returns the contents of given URL
+     * @param string $url
+     * @return string
+     */
+    public function get($url)
+    {
+        try {
+            $response = $this->httpClient->get($url);
+            return $response->getBody()->getContents();
+        } catch (\Exception $e) {
+            throw new \Exception('Crawl failed', 500);
+        }
+    }
+
+    private function getRandomUserAgent()
+    {
+        $userAgents = [
+            'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:63.0) Gecko/20100101 Firefox/63.0',
+            'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36',
+            'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Flock/3.5.3.4628 Chrome/7.0.517.450 Safari/534.7',
+            'Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30',
+            'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)',
+        ];
+
+        return $userAgents[array_rand($userAgents)];
+    }
+}