Permalink
Browse files

Add rawcontinue=1

  • Loading branch information...
DamianZaremba committed Jun 3, 2015
1 parent 6feb19d commit b3188172dd6162de74e6ef0e2586b5eb1ae0a829
Showing with 15 additions and 15 deletions.
  1. +2 −2 bot/cbng.php
  2. +12 −12 bot/wikibot.classes.php
  3. +1 −1 editstream/botfeed.py
View
@@ -136,7 +136,7 @@ function parseAllFeed( $feed ) {
function genOldFeedData( $id ) {
/* namespace, namespaceid, title, flags, url, revid, old_revid, user, length, comment, timestamp */
ini_set( 'user_agent', 'ClueBot/2.0 (Training EditDB Scraper)' );
$data = unserialize( file_get_contents( 'http://en.wikipedia.org/w/api.php?action=query&prop=revisions&rvprop=timestamp|user|comment&format=php&revids=' . urlencode( $id ) ) );
$data = unserialize( file_get_contents( 'http://en.wikipedia.org/w/api.php?action=query&rawcontinue=1&prop=revisions&rvprop=timestamp|user|comment&format=php&revids=' . urlencode( $id ) ) );
if( isset( $data[ 'query' ][ 'badrevids' ] ) )
return false;
$data = current( $data[ 'query' ][ 'pages' ] );
@@ -162,7 +162,7 @@ function parseFeedData( $feedData, $useOld = false, $pleasedontdie = false ) {
$urls = Array(
//'http://toolserver.org/~cobi/cb' . ( $useOld ? 'old' : '' ) . '.php?user=' . urlencode( $feedData[ 'user' ] ) . '&ns=' . $feedData[ 'namespaceid' ] . '&title=' . urlencode( $feedData[ 'title' ] ) . '&timestamp=' . urlencode( $feedData[ 'timestamp' ] ),
//'http://tools-webproxy/cluebot/cb.php?user=' . urlencode( $feedData[ 'user' ] ) . '&ns=' . $feedData[ 'namespaceid' ] . '&title=' . urlencode( $feedData[ 'title' ] ) . '&timestamp=' . urlencode( $feedData[ 'timestamp' ] ),
'http://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=' . urlencode( ( $feedData[ 'namespaceid' ] == 0 ? '' : $feedData[ 'namespace' ] . ':' ) . $feedData[ 'title' ] ) . '&rvstartid=' . $feedData[ 'revid' ] . '&rvlimit=2&rvprop=timestamp|user|content&format=php'
'http://en.wikipedia.org/w/api.php?action=query&rawcontinue=1&prop=revisions&titles=' . urlencode( ( $feedData[ 'namespaceid' ] == 0 ? '' : $feedData[ 'namespace' ] . ':' ) . $feedData[ 'title' ] ) . '&rvstartid=' . $feedData[ 'revid' ] . '&rvlimit=2&rvprop=timestamp|user|content&format=php'
);
list( $api ) = getUrlsInParallel( $urls );
View
@@ -230,7 +230,7 @@ function gettokens ($title,$flush = false) {
if (isset($this->tokencache[$title]) && (!$flush)) {
return $this->tokencache[$title]['tokens'];
} else {
$x = $this->http->get($this->apiurl.'?action=query&format=php&prop=info&intoken=edit|delete|protect|move|block|unblock|email&titles='.urlencode($title));
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&format=php&prop=info&intoken=edit|delete|protect|move|block|unblock|email&titles='.urlencode($title));
$x = unserialize($x);
foreach ($x['query']['pages'] as $y) {
$tokens = array();
@@ -270,7 +270,7 @@ function recentchanges ($count = 10,$namespace = null,$dir = 'older',$ts = null)
if ($ts !== null) { $append .= '&rcstart='.urlencode($ts); }
$append .= '&rcdir='.urlencode($dir);
if ($namespace !== null) { $append .= '&rcnamespace='.urlencode($namespace); }
$x = $this->http->get($this->apiurl.'?action=query&list=recentchanges&rcprop=user|comment|flags|timestamp|title|ids|sizes&format=php&rclimit='.$count.$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&list=recentchanges&rcprop=user|comment|flags|timestamp|title|ids|sizes&format=php&rclimit='.$count.$append);
$x = unserialize($x);
return $x['query']['recentchanges'];
}
@@ -293,7 +293,7 @@ function search ($search,$limit = 10,$offset = 0,$namespace = 0,$what = 'text',$
if ($what != null) $append .= '&srwhat='.urlencode($what);
if ($redirs == true) $append .= '&srredirects=1';
else $append .= '&srredirects=0';
$x = $this->http->get($this->apiurl.'?action=query&list=search&format=php&srsearch='.urlencode($search).$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&list=search&format=php&srsearch='.urlencode($search).$append);
$x = unserialize($x);
return $x['query']['search'];
}
@@ -318,7 +318,7 @@ function logs ($user = null,$title = null,$limit = 50,$type = null,$start = null
if ($start != null) $append.= '&lestart='.urlencode($start);
if ($end != null) $append.= '&leend='.urlencode($end);
if ($dir != null) $append.= '&ledir='.urlencode($dir);
$x = $this->http->get($this->apiurl.'?action=query&format=php&list=logevents&leprop=ids|title|type|user|timestamp|comment|details'.$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&format=php&list=logevents&leprop=ids|title|type|user|timestamp|comment|details'.$append);
$x = unserialize($x);
return $x['query']['logevents'];
}
@@ -337,7 +337,7 @@ function usercontribs ($user,$count = 50,&$continue = null,$dir = 'older') {
} else {
$append = '';
}
$x = $this->http->get($this->apiurl.'?action=query&format=php&list=usercontribs&ucuser='.urlencode($user).'&uclimit='.urlencode($count).'&ucdir='.urlencode($dir).$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&format=php&list=usercontribs&ucuser='.urlencode($user).'&uclimit='.urlencode($count).'&ucdir='.urlencode($dir).$append);
$x = unserialize($x);
$continue = $x['query-continue']['usercontribs']['ucstart'];
return $x['query']['usercontribs'];
@@ -357,7 +357,7 @@ function usercontribs ($user,$count = 50,&$continue = null,$dir = 'older') {
* @return Associative array of revision data.
**/
function revisions ($page,$count = 1,$dir = 'older',$content = false,$revid = null,$wait = true,$getrbtok = false,$dieonerror = true,$redirects = false) {
$x = $this->http->get($this->apiurl.'?action=query&prop=revisions&titles='.urlencode($page).'&rvlimit='.urlencode($count).'&rvprop=timestamp|ids|user|comment'.(($content)?'|content':'').'&format=php&meta=userinfo&rvdir='.urlencode($dir).(($revid !== null)?'&rvstartid='.urlencode($revid):'').(($getrbtok == true)?'&rvtoken=rollback':'').(($redirects == true)?'&redirects':''));
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&prop=revisions&titles='.urlencode($page).'&rvlimit='.urlencode($count).'&rvprop=timestamp|ids|user|comment'.(($content)?'|content':'').'&format=php&meta=userinfo&rvdir='.urlencode($dir).(($revid !== null)?'&rvstartid='.urlencode($revid):'').(($getrbtok == true)?'&rvtoken=rollback':'').(($redirects == true)?'&redirects':''));
$x = unserialize($x);
if ($revid !== null) {
@@ -410,7 +410,7 @@ function users ($start = null,$limit = 1,$group = null,$requirestart = false,&$c
$append = '';
if ($start != null) $append .= '&aufrom='.urlencode($start);
if ($group != null) $append .= '&augroup='.urlencode($group);
$x = $this->http->get($this->apiurl.'?action=query&list=allusers&format=php&auprop=blockinfo|editcount|registration|groups&aulimit='.urlencode($limit).$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&list=allusers&format=php&auprop=blockinfo|editcount|registration|groups&aulimit='.urlencode($limit).$append);
$x = unserialize($x);
$continue = $x['query-continue']['allusers']['aufrom'];
if (($requirestart == true) and ($x['query']['allusers'][0]['name'] != $start)) return false;
@@ -431,7 +431,7 @@ function categorymembers ($category,$count = 500,&$continue = null) {
$append = '';
}
$category = 'Category:'.str_ireplace('category:','',$category);
$x = $this->http->get($this->apiurl.'?action=query&list=categorymembers&cmtitle='.urlencode($category).'&format=php&cmlimit='.$count.$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&list=categorymembers&cmtitle='.urlencode($category).'&format=php&cmlimit='.$count.$append);
$x = unserialize($x);
$continue = $x['query-continue']['categorymembers']['cmcontinue'];
return $x['query']['categorymembers'];
@@ -452,7 +452,7 @@ function listcategories (&$start = null,$limit = 50,$dir = 'ascending',$prefix =
if ($dir != null) $append .= '&acdir='.urlencode($dir);
if ($prefix != null) $append .= '&acprefix='.urlencode($prefix);
$x = $this->http->get($this->apiurl.'?action=query&list=allcategories&acprop=size&format=php'.$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&list=allcategories&acprop=size&format=php'.$append);
$x = unserialize($x);
$start = $x['query-continue']['allcategories']['acfrom'];
@@ -478,7 +478,7 @@ function backlinks ($page,$count = 500,&$continue = null,$filter = null) {
$append .= '&blfilterredir='.urlencode($filter);
}
$x = $this->http->get($this->apiurl.'?action=query&list=backlinks&bltitle='.urlencode($page).'&format=php&bllimit='.$count.$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&list=backlinks&bltitle='.urlencode($page).'&format=php&bllimit='.$count.$append);
$x = unserialize($x);
$continue = $x['query-continue']['backlinks']['blcontinue'];
return $x['query']['backlinks'];
@@ -497,7 +497,7 @@ function embeddedin ($page,$count = 500,&$continue = null) {
} else {
$append = '';
}
$x = $this->http->get($this->apiurl.'?action=query&list=embeddedin&eititle='.urlencode($page).'&format=php&eilimit='.$count.$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&list=embeddedin&eititle='.urlencode($page).'&format=php&eilimit='.$count.$append);
$x = unserialize($x);
$continue = $x['query-continue']['embeddedin']['eicontinue'];
return $x['query']['embeddedin'];
@@ -516,7 +516,7 @@ function listprefix ($prefix,$namespace = 0,$count = 500,&$continue = null) {
if ($continue != null) {
$append .= '&apfrom='.urlencode($continue);
}
$x = $this->http->get($this->apiurl.'?action=query&list=allpages&apprefix='.urlencode($prefix).'&format=php&aplimit='.$count.$append);
$x = $this->http->get($this->apiurl.'?action=query&rawcontinue=1&list=allpages&apprefix='.urlencode($prefix).'&format=php&aplimit='.$count.$append);
$x = unserialize($x);
$continue = $x['query-continue']['allpages']['apfrom'];
return $x['query']['allpages'];
View
@@ -121,7 +121,7 @@ def run(self):
def gen(self,id,title):
r={}
url="""http://en.wikipedia.org/w/api.php?action=query&format=json&rvlimit=2&prop=revisions&titles=%s&rvprop=timestamp|user|comment|flags|content|ids&rvstartid=%d"""%(quote(title),int(id))
url="""http://en.wikipedia.org/w/api.php?action=query&rawcontinue=1&format=json&rvlimit=2&prop=revisions&titles=%s&rvprop=timestamp|user|comment|flags|content|ids&rvstartid=%d"""%(quote(title),int(id))
#print url
url = urlopen(url).read()

0 comments on commit b318817

Please sign in to comment.