Merge pull request #4 from mousavian/master

Show progress bar during the process
hirak · Jun 27, 2016 · ee5d0a8 · ee5d0a8
2 parents d04d9b0 + 7a8496d
commit ee5d0a8
Show file tree

Hide file tree

Showing 5 changed files with 144 additions and 51 deletions.
diff --git a/check.php b/check.php
@@ -1,4 +1,10 @@
 <?php
+
+use ProgressBar\Manager as ProgressBarManager;
+
+
+require_once __DIR__ . '/vendor/autoload.php';
+
 /**
  * ダウンロードに失敗したファイルを一覧で出す
  */
@@ -14,24 +20,37 @@
 
 $packagejson = json_decode(file_get_contents($cachedir.'packages.json'));
 
-$i = $j = 0;
+
+$j = 0;
+$errors = array();
+$providerCounter = 1;
+$numberOfProviders = count( (array)$packagejson->{'provider-includes'} );
+
 foreach ($packagejson->{'provider-includes'} as $tpl => $provider) {
     $providerjson = str_replace('%hash%', $provider->sha256, $tpl);
     $packages = json_decode(file_get_contents($cachedir.$providerjson));
 
+    $progressBar = new ProgressBarManager(0, count( (array)$packages->providers ));
+    $progressBar->setFormat("      - Package: %current%/%max% [%bar%] %percent%%");
+    echo "   - Check Provider {$providerCounter}/{$numberOfProviders}:\n";
+
     foreach ($packages->providers as $tpl2 => $sha) {
         if (!file_exists($file = $cachedir . "p/$tpl2\$$sha->sha256.json")) {
-            ++$i;
-            echo "$tpl\t$tpl2 file not exists\n";
+            $errors[] = "   - $tpl\t$tpl2 file not exists\n";
         } elseif ($sha->sha256 !== hash_file('sha256', $file)) {
-            ++$i;
             unlink($file);
-            echo "$tpl\t$tpl2\tsha256 not match: {$sha->sha256}\n";
+            $errors[] = "   - $tpl\t$tpl2\tsha256 not match: {$sha->sha256}\n";
         } else {
             ++$j;
         }
+        $progressBar->advance();
     }
+
+    ++$providerCounter;
+}
+
+if (count($errors)) {
+    echo "Errors: \n", implode('', $errors);
 }
-echo $i, ' / ', $i+$j,  PHP_EOL;
 
-exit($i);
+exit(1);
diff --git a/composer.json b/composer.json
@@ -10,6 +10,7 @@
     ],
     "require": {
         "spindle/httpclient": "1.*",
+        "guiguiboy/php-cli-progress-bar": "dev-master",
         "php": ">=5.4",
         "ext-json": "*",
         "ext-hash": "*",

diff --git a/composer.lock b/composer.lock
diff --git a/parallel.php b/parallel.php
@@ -1,6 +1,7 @@
 <?php
 namespace Spindle\HttpClient;
 
+use ProgressBar\Manager as ProgressBarManager;
 use hirak\PackagistCrawler\ExpiredFileManager;
 
 set_time_limit(0);
@@ -82,35 +83,39 @@ function downloadProviders($config, $globals)
 
     $providers = [];
 
+    $numberOfProviders = count( (array)$packages->{'provider-includes'} );
+    $progressBar = new ProgressBarManager(0, $numberOfProviders);
+    $progressBar->setFormat('Downloading Providers: %current%/%max% [%bar%] %percent%%');
+
     foreach ($packages->{'provider-includes'} as $tpl => $version) {
         $fileurl = str_replace('%hash%', $version->sha256, $tpl);
         $cachename = $cachedir . $fileurl;
         $providers[] = $cachename;
 
-        if (file_exists($cachename)) continue;
-
-        $req->setOption('url', $config->packagistUrl . '/' . $fileurl);
-        $res = $req->send();
-
-        error_log($res->getStatusCode(). "\t". $res->getUrl());
-        if (200 !== $res->getStatusCode()) {
-            $globals->retry = true;
-            continue;
-        }
+        if (!file_exists($cachename)){
+            $req->setOption('url', $config->packagistUrl . '/' . $fileurl);
+            $res = $req->send();
 
-        $oldcache = $cachedir . str_replace('%hash%', '*', $tpl);
-        if ($glob = glob($oldcache)) {
-            foreach ($glob as $old) {
-                $globals->expiredManager->add($old, time());
+            if (200 === $res->getStatusCode()) {
+                $oldcache = $cachedir . str_replace('%hash%', '*', $tpl);
+                if ($glob = glob($oldcache)) {
+                    foreach ($glob as $old) {
+                        $globals->expiredManager->add($old, time());
+                    }
+                }
+                if (!file_exists(dirname($cachename))) {
+                    mkdir(dirname($cachename), 0777, true);
+                }
+                file_put_contents($cachename, $res->getBody());
+                if ($config->generateGz) {
+                    file_put_contents($cachename . '.gz', gzencode($res->getBody()));
+                }
+            } else {
+                $globals->retry = true;
             }
         }
-        if (!file_exists(dirname($cachename))) {
-            mkdir(dirname($cachename), 0777, true);
-        }
-        file_put_contents($cachename, $res->getBody());
-        if ($config->generateGz) {
-            file_put_contents($cachename . '.gz', gzencode($res->getBody()));
-        }
+
+        $progressBar->advance();
     }
 
     return $providers;
@@ -123,7 +128,8 @@ function downloadProviders($config, $globals)
 function downloadPackages($config, $globals, $providers)
 {
     $cachedir = $config->cachedir;
-    $i = 0;
+    $i = 1;
+    $numberOfProviders = count($providers);
     $urls = [];
 
     foreach ($providers as $providerjson) {
@@ -133,8 +139,13 @@ function downloadPackages($config, $globals, $providers)
         $list = $list->providers;
         $all = count((array)$list);
 
+        $progressBar = new ProgressBarManager(0, $all);
+        echo "   - Provider {$i}/{$numberOfProviders}:\n";
+        $progressBar->setFormat("      - Package: %current%/%max% [%bar%] %percent%%");
+
         $sum = 0;
         foreach ($list as $packageName => $provider) {
+            $progressBar->advance();
             ++$sum;
             $url = "$config->packagistUrl/p/$packageName\$$provider->sha256.json";
             $cachefile = $cachedir . str_replace("$config->packagistUrl/", '', $url);
@@ -152,7 +163,6 @@ function downloadPackages($config, $globals, $providers)
             do {
                 $requests = $globals->mh->getFinishedResponses(); //block
             } while (0 === count($requests));
-            //error_log("downloaded: $sum / $all");
 
             foreach ($requests as $req) {
                 $res = $req->getResponse();
@@ -181,35 +191,42 @@ function downloadPackages($config, $globals, $providers)
                 }
             }
         }
+
+        ++$i;
     }
 
 
     if (0 === count($globals->mh)) return;
     //残りの端数をダウンロード
     $globals->mh->waitResponse();
+
+    $progressBar = new ProgressBarManager(0, count($globals->mh));
+    $progressBar->setFormat("   - Remianed packages: %current%/%max% [%bar%] %percent%%");
+
     foreach ($globals->mh as $req) {
         $res = $req->getResponse();
 
-        if (200 !== $res->getStatusCode()) {
-            error_log($res->getStatusCode(). "\t". $res->getUrl());
+        if (200 === $res->getStatusCode()) {
+            $cachefile = $cachedir
+                . str_replace("$config->packagistUrl/", '', $res->getUrl());
+            if ($glob = glob("{$cachedir}p/$req->packageName\$*")) {
+                foreach ($glob as $old) {
+                    $globals->expiredManager->add($old, time());
+                }
+            }
+            if (!file_exists(dirname($cachefile))) {
+                mkdir(dirname($cachefile), 0777, true);
+            }
+            file_put_contents($cachefile, $res->getBody());
+            if ($config->generateGz) {
+                file_put_contents($cachefile . '.gz', gzencode($res->getBody()));
+            }
+
+        } else {
             $globals->retry = true;
-            continue;
         }
 
-        $cachefile = $cachedir
-            . str_replace("$config->packagistUrl/", '', $res->getUrl());
-        if ($glob = glob("{$cachedir}p/$req->packageName\$*")) {
-            foreach ($glob as $old) {
-                $globals->expiredManager->add($old, time());
-            }
-        }
-        if (!file_exists(dirname($cachefile))) {
-            mkdir(dirname($cachefile), 0777, true);
-        }
-        file_put_contents($cachefile, $res->getBody());
-        if ($config->generateGz) {
-            file_put_contents($cachefile . '.gz', gzencode($res->getBody()));
-        }
+        $progressBar->advance();
     }
 
 }
@@ -262,12 +279,16 @@ function clearExpiredFiles(ExpiredFileManager $expiredManager)
 {
     $expiredFiles = $expiredManager->getExpiredFileList();
 
+    $progressBar = new ProgressBarManager(0, count($expiredFiles));
+    $progressBar->setFormat("   - Clearing Expired Files: %current%/%max% [%bar%] %percent%%");
+
     foreach ($expiredFiles as $file) {
         if (file_exists($file)) {
             unlink($file) and $expiredManager->delete($file);
         } else {
             $expiredManager->delete($file);
         }
+        $progressBar->advance();
     }
 }
 

diff --git a/src/ExpiredFileManager.php b/src/ExpiredFileManager.php
@@ -109,6 +109,12 @@ function getExpiredFileList($until=null)
         $stmt->bindValue(':expiredAt', $until, PDO::PARAM_INT);
         $stmt->execute();
         $stmt->setFetchMode(PDO::FETCH_COLUMN, 0);
-        return $stmt;
+        $list = array();
+
+        foreach ($stmt as $file){
+            $list[] = $file;
+        }
+
+        return $list;
     }
 }