diff --git a/www/include/lib_flickr_photos_import.php b/www/include/lib_flickr_photos_import.php index af97f5b..6c9d9ad 100644 --- a/www/include/lib_flickr_photos_import.php +++ b/www/include/lib_flickr_photos_import.php @@ -112,9 +112,10 @@ function flickr_photos_import_photo($photo, $more=array()){ # exif data - $more = array( - 'force' => 1, - ); + # why did I do this? (20111206/straup) + # $more = array( + # 'force' => 1, + # ); if ($hasexif = flickr_photos_exif_has_exif($photo, $more)){ @@ -275,69 +276,85 @@ function flickr_photos_import_photo_files(&$photo, $more=array()){ # fetch all the bits using http_multi() if ($count = count($req)){ + _flickr_photos_fetch_multi($req); + } - dumper("fetching {$count} URIs for photo {$photo['id']}"); + } - $multi = array(); + ################################################################# - foreach ($req as $uris){ - list($remote, $local) = $uris; - $multi[] = array('url' => $remote); - } + function _flickr_photos_fetch_multi(&$req, $retries=3){ - $rsp = http_multi($multi); + dumper("fetching {$count} URIs for photo {$photo['id']}"); - for ($i=0; $i < $count; $i++){ + $multi = array(); + $failed = array(); - $_rsp = $rsp[$i]; - $_req = $req[$i]; + foreach ($req as $uris){ + list($remote, $local) = $uris; + $multi[] = array('url' => $remote); + } - if (! $_rsp['ok']){ - # make an error/warning here... - continue; - } + $rsp = http_multi($multi); - list($remote, $local) = $_req; + for ($i=0; $i < $count; $i++){ - # if $source then check to ensure we have something - # worth writing to disk + $_rsp = $rsp[$i]; + $_req = $req[$i]; - if (preg_match("/^json:(\w+):(.*)$/", $local, $m)){ + list($remote, $local) = $_req; - $data = $_rsp['body']; - $source = $m[1]; - $local = $m[2]; + if (! $_rsp['ok']){ + $failed = $_req; - $to_check = array( - 'comments', - ); + $will_retry = ($retries) ? 1 : 0; - if (in_array($source, $to_check)){ + dumper("failed to fetch {$remote}: {$rsp['error']} will retry: {$will_retry}"); + continue; + } - $json = json_decode($data, "as hash"); + # if $source then check to ensure we have something + # worth writing to disk - if (! $json){ - continue; - } - } + if (preg_match("/^json:(\w+):(.*)$/", $local, $m)){ + + $data = $_rsp['body']; + $source = $m[1]; + $local = $m[2]; + + $to_check = array( + 'comments', + ); - if ($source == 'comments'){ + if (in_array($source, $to_check)){ - if (! count($json['comments']['comment'])){ - continue; - } + $json = json_decode($data, "as hash"); + + if (! $json){ + continue; } } - else { - $data = $_rsp['body']; + if ($source == 'comments'){ + + if (! count($json['comments']['comment'])){ + continue; + } } + } - _flickr_photos_import_store($local, $data); - dumper("wrote {$local}"); + else { + $data = $_rsp['body']; } + + _flickr_photos_import_store($local, $data); + dumper("wrote {$local}"); } + if ((count($failed)) && ($retries)){ + $retries = ($retries) ? $retries - 1 : 0; + _flickr_photos_fetch_multi($failed, $retries); + } } ################################################################# @@ -378,11 +395,39 @@ function flickr_photos_import_get_recent($nsid, $more=array()){ $imported = 0; + # TO DO: capture dateupdate for each photo and return that + # if there's a fatal error so that the FlickrBackups database + # can be set with something other than 0 (20111206/straup) + while ((! isset($pages)) || ($pages >= $args['page'])){ - $rsp = flickr_api_call($method, $args); + # because the Flickr API has an annoying habit of + # timing out and this causes an initial import of + # photos to fail and be repeated in-toto over and + # over again (20111206/straup) - if (! $rsp['ok']){ + $tries = 1; + $max_tries = 5; + $ok = 0; + + while ((! $ok) && ($tries < $max_tries)){ + $rsp = flickr_api_call($method, $args); + $ok = $rsp['ok']; + + $tries++; + + if ($ok){ + + $photos = $rsp['rsp']['photos']['photo']; + + if (! is_array($photos)){ + $rsp = not_ok("no photos"); + $ok = 0; + } + } + } + + if (! $ok){ return $rsp; } @@ -390,14 +435,7 @@ function flickr_photos_import_get_recent($nsid, $more=array()){ $pages = $rsp['rsp']['photos']['pages']; } - $photos = $rsp['rsp']['photos']['photo']; - - if (! is_array($photos)){ - return array( - 'ok' => 0, - 'error' => 'no photos', - ); - } + # TO DO: date update stuff (see above) foreach ($photos as $photo){ flickr_photos_import_photo($photo, $more);