Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Updated code to use data from web/DB rather than upstream.txt previou…

…sly used.
  • Loading branch information...
commit f8c4847110264744c820b1a7ce64dd4006cf4948 1 parent f27eb0a
@nbprashanth authored
Showing with 346 additions and 172 deletions.
  1. +176 −79 New Scripts/Upstream.py
  2. +170 −93 New Scripts/__init__.py
View
255 New Scripts/Upstream.py
@@ -8,6 +8,7 @@
from urllib2 import HTTPError, URLError
import BeautifulSoup
import re
+from difflib import Match
class Upstream(object):
'''
@@ -71,8 +72,24 @@ def getPageVersions(self, links, pkgname):
return versions
def getLatestVersion(self, versions):
-
- return max(versions)
+
+ sums=[]
+
+ for x in versions:
+ k=10**4#len(x.split('.'))
+ sum=0
+
+ try:
+ for m in x.split('.'):
+ sum=sum+int(m)*k/10
+ k=k/10
+ sums.append(sum)
+ except:
+ sums.append(0)
+
+ #print max(sums)
+
+ return versions[sums.index(max(sums))]
class HTTPLS(Upstream):
@@ -83,47 +100,64 @@ def __init__(self, pkgname, url):
def process(self, debugBool):
+ error=False
+ errorMsg=None
+ latestVer=None
+ path=None
+
debug=debugBool
data=Upstream.getPageData(self, self.url)
if data==None:
- print 'Unable to read page source'
- return None
+ error=True
+ errorMsg='Unable to read page source'
+ return (latestVer, path, error, errorMsg)
links=Upstream.getPageLinks(self, data)
if links==None:
- print 'Unable to find links on page'
- return None
+ error=True
+ errorMsg='Unable to find links on page'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Links (' + self.pkgname + '): ',
- print links
-
+ return (latestVer, path, error, errorMsg)
+
versions=Upstream.getPageVersions(self, links, self.pkgname)
if versions==None:
- print 'Unable to extract version string'
- return None
+ error=True
+ errorMsg='Unable to extract version string'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Versions (' + self.pkgname + '): ',
- print versions
+ return (latestVer, path, error, errorMsg)
latestVer=Upstream.getLatestVersion(self, versions)
if latestVer==None:
- print 'Unable to obtain latest version'
- return None
+ error=True
+ errorMsg='Unable to obtain latest version'
+ return (latestVer, path, error, errorMsg)
# else:
# print 'Latest Version (' + self.pkgname + '): ' + str(latestVer)
if debug:
print 'Latest Version (' + self.pkgname + '): ',
- print latestVer
+ return (latestVer, path, error, errorMsg)
+
+ fileName='Not Found'
- return latestVer
+ for link in links:
+ if link.find(latestVer)>=0:
+ fileName=link
+
+ path=self.url.strip()+fileName.strip()
+
+ return (latestVer, path, error, errorMsg)
class DualHTTPLS(Upstream):
@@ -135,6 +169,11 @@ def __init__(self, pkgname, url):
def process(self, debugBool):
+ error=False
+ errorMsg=None
+ latestVer=None
+ path=None
+
debug=debugBool
data1=Upstream.getPageData(self, self.url1)
@@ -144,14 +183,16 @@ def process(self, debugBool):
data=data1+data2
if data1==None and data2==None:
- print 'Unable to read page source'
- return None
+ error=True
+ errorMsg='Unable to read page source'
+ return (latestVer, path, error, errorMsg)
links=Upstream.getPageLinks(self, data)
if links==None:
- print 'Unable to find links on page'
- return None
+ error=True
+ errorMsg='Unable to find links on page'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Links (' + self.pkgname + '): ',
@@ -160,8 +201,9 @@ def process(self, debugBool):
versions=Upstream.getPageVersions(self, links, self.pkgname)
if versions==None:
- print 'Unable to extract version string'
- return None
+ error=True
+ errorMsg='Unable to extract version string'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Versions (' + self.pkgname + '): ',
@@ -170,8 +212,9 @@ def process(self, debugBool):
latestVer=Upstream.getLatestVersion(self, versions)
if latestVer==None:
- print 'Unable to obtain latest version'
- return None
+ error=True
+ errorMsg='Unable to obtain latest version'
+ return (latestVer, path, error, errorMsg)
# else:
# print 'Latest Version (' + self.pkgname + '): ' + str(latestVer)
@@ -179,7 +222,7 @@ def process(self, debugBool):
print 'Latest Version (' + self.pkgname + '): ',
print latestVer
- return latestVer
+ return (latestVer, path, error, errorMsg)
class Launchpad(Upstream):
@@ -190,6 +233,11 @@ def __init__(self, pkgname, url):
def process(self, debugBool):
+ error=False
+ errorMsg=None
+ latestVer=None
+ path=None
+
debug=debugBool
self.url='http://launchpad.net/'+self.url.strip()+'/+download'
@@ -197,14 +245,16 @@ def process(self, debugBool):
data=Upstream.getPageData(self, self.url)
if data==None:
- print 'Unable to read page source'
- return None
+ error=True
+ errorMsg='Unable to read page source'
+ return (latestVer, path, error, errorMsg)
links=Upstream.getPageLinks(self, data)
if links==None:
- print 'Unable to find links on page'
- return None
+ error=True
+ errorMsg='Unable to find links on page'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Links (' + self.pkgname + '): ',
@@ -213,8 +263,9 @@ def process(self, debugBool):
versions=Upstream.getPageVersions(self, links, self.pkgname)
if versions==None:
- print 'Unable to extract version string'
- return None
+ error=True
+ errorMsg='Unable to extract version string'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Versions (' + self.pkgname + '): ',
@@ -223,8 +274,9 @@ def process(self, debugBool):
latestVer=Upstream.getLatestVersion(self, versions)
if latestVer==None:
- print 'Unable to obtain latest version'
- return None
+ error=True
+ errorMsg='Unable to obtain latest version'
+ return (latestVer, path, error, errorMsg)
# else:
# print 'Latest Version (' + self.pkgname + '): ' + str(latestVer)
@@ -232,7 +284,7 @@ def process(self, debugBool):
print 'Latest Version (' + self.pkgname + '): ',
print latestVer
- return latestVer
+ return (latestVer, path, error, errorMsg)
class SVNLS(Upstream):
@@ -243,19 +295,26 @@ def __init__(self, pkgname, url):
def process(self, debugBool):
+ error=False
+ errorMsg=None
+ latestVer=None
+ path=None
+
debug=debugBool
data=Upstream.getPageData(self, self.url)
if data==None:
- print 'Unable to read page source'
- return None
+ error=True
+ errorMsg='Unable to read page source'
+ return (latestVer, path, error, errorMsg)
links=Upstream.getPageLinks(self, data, 'file')
if links==None:
- print 'Unable to find links on page'
- return None
+ error=True
+ errorMsg='Unable to find links on page'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Links (' + self.pkgname + '): ',
@@ -264,8 +323,9 @@ def process(self, debugBool):
versions=Upstream.getPageVersions(self, links, self.pkgname)
if versions==None:
- print 'Unable to extract version string'
- return None
+ error=True
+ errorMsg='Unable to extract version string'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Versions (' + self.pkgname + '): ',
@@ -274,8 +334,9 @@ def process(self, debugBool):
latestVer=Upstream.getLatestVersion(self, versions)
if latestVer==None:
- print 'Unable to obtain latest version'
- return None
+ error=True
+ errorMsg='Unable to obtain latest version'
+ return (latestVer, path, error, errorMsg)
# else:
# print 'Latest Version (' + self.pkgname + '): ' + str(latestVer)
@@ -283,7 +344,7 @@ def process(self, debugBool):
print 'Latest Version (' + self.pkgname + '): ',
print latestVer
- return latestVer
+ return (latestVer, path, error, errorMsg)
class Google(Upstream):
@@ -294,6 +355,11 @@ def __init__(self, pkgname, url):
def process(self, debugBool):
+ error=False
+ errorMsg=None
+ latestVer=None
+ path=None
+
debug=debugBool
if self.url.find('|')>=0:
@@ -308,14 +374,16 @@ def process(self, debugBool):
data=Upstream.getPageData(self, self.url)
if data==None:
- print 'Unable to read page source'
- return None
+ error=True
+ errorMsg='Unable to read page source'
+ return (latestVer, path, error, errorMsg)
links=Upstream.getPageLinks(self, data)
if links==None:
- print 'Unable to find links on page'
- return None
+ error=True
+ errorMsg='Unable to find links on page'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Links (' + self.pkgname + '): ',
@@ -324,8 +392,9 @@ def process(self, debugBool):
versions=Upstream.getPageVersions(self, links, tarball)
if versions==None:
- print 'Unable to extract version string'
- return None
+ error=True
+ errorMsg='Unable to extract version string'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Versions (' + self.pkgname + '): ',
@@ -334,8 +403,9 @@ def process(self, debugBool):
latestVer=Upstream.getLatestVersion(self, versions)
if latestVer==None:
- print 'Unable to obtain latest version'
- return None
+ error=True
+ errorMsg='Unable to obtain latest version'
+ return (latestVer, path, error, errorMsg)
# else:
# print 'Latest Version (' + self.pkgname + '): ' + str(latestVer)
@@ -343,7 +413,7 @@ def process(self, debugBool):
print 'Latest Version (' + self.pkgname + '): ',
print latestVer
- return latestVer
+ return (latestVer, path, error, errorMsg)
import ftplib
import urlparse
@@ -357,6 +427,11 @@ def __init__(self, pkgname, url):
def process(self, debugBool):
+ error=False
+ errorMsg=None
+ latestVer=None
+ path=None
+
debug=debugBool
files = []
@@ -376,14 +451,16 @@ def process(self, debugBool):
raise
if files==None:
- print 'Unable to read page source'
- return None
+ error=True
+ errorMsg='Unable to read page source'
+ return (latestVer, path, error, errorMsg)
versions=Upstream.getPageVersions(self, files, self.pkgname)
if versions==None:
- print 'Unable to extract version string'
- return None
+ error=True
+ errorMsg='Unable to extract version string'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Versions (' + self.pkgname + '): ',
@@ -392,8 +469,9 @@ def process(self, debugBool):
latestVer=Upstream.getLatestVersion(self, versions)
if latestVer==None:
- print 'Unable to obtain latest version'
- return None
+ error=True
+ errorMsg='Unable to obtain latest version'
+ return (latestVer, path, error, errorMsg)
# else:
# print 'Latest Version (' + self.pkgname + '): ' + str(latestVer)
@@ -401,7 +479,7 @@ def process(self, debugBool):
print 'Latest Version (' + self.pkgname + '): ',
print latestVer
- return latestVer
+ return (latestVer, path, error, errorMsg)
class Trac(Upstream):
@@ -412,19 +490,26 @@ def __init__(self, pkgname, url):
def process(self, debugBool):
+ error=False
+ errorMsg=None
+ latestVer=None
+ path=None
+
debug=debugBool
data=Upstream.getPageData(self, self.url)
if data==None:
- print 'Unable to read page source'
- return None
+ error=True
+ errorMsg='Unable to read page source'
+ return (latestVer, path, error, errorMsg)
links=Upstream.getPageLinks(self, data,'a', True)
if links==None:
- print 'Unable to find links on page'
- return None
+ error=True
+ errorMsg='Unable to find links on page'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Links (' + self.pkgname + '): ',
@@ -433,8 +518,9 @@ def process(self, debugBool):
versions=Upstream.getPageVersions(self, links, self.pkgname)
if versions==None:
- print 'Unable to extract version string'
- return None
+ error=True
+ errorMsg='Unable to extract version string'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Versions (' + self.pkgname + '): ',
@@ -443,8 +529,9 @@ def process(self, debugBool):
latestVer=Upstream.getLatestVersion(self, versions)
if latestVer==None:
- print 'Unable to obtain latest version'
- return None
+ error=True
+ errorMsg='Unable to obtain latest version'
+ return (latestVer, path, error, errorMsg)
# else:
# print 'Latest Version (' + self.pkgname + '): ' + str(latestVer)
@@ -452,7 +539,7 @@ def process(self, debugBool):
print 'Latest Version (' + self.pkgname + '): ',
print latestVer
- return latestVer
+ return (latestVer, path, error, errorMsg)
class SubdirHTTPLS(Upstream):
@@ -463,19 +550,26 @@ def __init__(self, pkgname, url):
def process(self, debugBool):
+ error=False
+ errorMsg=None
+ latestVer=None
+ path=None
+
debug=debugBool
data=Upstream.getPageData(self, self.url)
if data==None:
- print 'Unable to read page source'
- return None
+ error=True
+ errorMsg='Unable to read page source'
+ return (latestVer, path, error, errorMsg)
links=Upstream.getPageLinks(self, data)
if links==None:
- print 'Unable to find links on page'
- return None
+ error=True
+ errorMsg='Unable to find links on page'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Links (' + self.pkgname + '): ',
@@ -501,8 +595,9 @@ def process(self, debugBool):
links=Upstream.getPageLinks(self, data)
if links==None:
- print 'Unable to find links on page'
- return None
+ error=True
+ errorMsg='Unable to find links on page'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Links (' + self.pkgname + '): ',
@@ -511,8 +606,9 @@ def process(self, debugBool):
versions=Upstream.getPageVersions(self, links, self.pkgname)
if versions==None:
- print 'Unable to extract version string'
- return None
+ error=True
+ errorMsg='Unable to extract version string'
+ return (latestVer, path, error, errorMsg)
if debug:
print 'Versions (' + self.pkgname + '): ',
@@ -521,8 +617,9 @@ def process(self, debugBool):
latestVer=Upstream.getLatestVersion(self, versions)
if latestVer==None:
- print 'Unable to obtain latest version'
- return None
+ error=True
+ errorMsg='Unable to obtain latest version'
+ return (latestVer, path, error, errorMsg)
# else:
# print 'Latest Version (' + self.pkgname + '): ' + str(latestVer)
@@ -531,6 +628,6 @@ def process(self, debugBool):
print latestVer
if latestVer==None:
- return latestMajorVer
+ return (latestMajorVer, path, error, errorMsg)
else:
- return latestVer
+ return (latestVer, path, error, errorMsg)
View
263 New Scripts/__init__.py
@@ -7,106 +7,183 @@
import Record
import sys
import Upstream
+from Upstream import HTTPLS
+from test.test_binop import isnum
+import urllib2
+from WebParse import WebParse
if __name__ == '__main__':
- filename='upstream.txt'
- debug=False
+# filename='upstream.txt'
+# debug=False
+#
+# try:
+# records=Record.Record(filename).getRecords()
+# if debug:
+# print 'Records : ',
+# print records
+# except IOError as e:
+# if e[0]==2:
+# print 'Cannot find file ' + filename
+# else:
+# print 'Error in reading records from file'
+# sys.exit(1)
+#
+# for record in records:
+#
+# pkgname=record[0]
+# method=record[1]
+# url=record[2]
+
+ import WebParse
+ wp = WebParse.WebParse('http://localhost','3000')
+ records=wp.getRecords()
- try:
- records=Record.Record(filename).getRecords()
- if debug:
- print 'Records : ',
- print records
- except IOError as e:
- if e[0]==2:
- print 'Cannot find file ' + filename
- else:
- print 'Error in reading records from file'
- sys.exit(1)
-
for record in records:
- pkgname=record[0]
- method=record[1]
- url=record[2]
+# if record['processed']=='false':
-# if method=='httpls':
-# print 'Latest Version (' + pkgname + '): ',
-# try:
-# latestVer = Upstream.HTTPLS(pkgname, url).process(False)
-# if latestVer!=None:
-# print latestVer
-# except:
-# print 'Can\'t find version'
-# pass
+ pkgname=record['pkgname']
+ method=record['method']
+ url=record['url']
+ id=record['id']
-# if method=='dualhttpls':
-# print 'Latest Version (' + pkgname + '): ',
-# try:
-# latestVer = Upstream.DualHTTPLS(pkgname, url).process(False)
-# if latestVer!=None:
-# print latestVer
-# except:
-# print 'Can\'t find version'
-# pass
-
-# if method=='lp':
-# print 'Latest Version (' + pkgname + '): ',
-# try:
-# latestVer = Upstream.Launchpad(pkgname, url).process(False)
-# if latestVer!=None:
-# print latestVer
-# except:
-# print 'Can\'t find version'
-# pass
-
-# if method=='svnls':
-# print 'Latest Version (' + pkgname + '): ',
-# try:
-# latestVer = Upstream.SVNLS(pkgname, url).process(False)
-# if latestVer!=None:
-# print latestVer
-# except:
-# print 'Can\'t find version'
-# pass
+ if method=='httpls':
+ print 'Latest Version (' + pkgname + '): ',
+ try:
+ (latestVer, path, error, errorMsg) = Upstream.HTTPLS(pkgname, url).process(False)
+ print latestVer
+ except:
+ print 'Can\'t find version'
+ latestVer=None
+ path=None
+ error=True
+ errorMsg='Unknown Error'
+
+ wp.updateRecord('latest_ver', latestVer, str(id))
+ wp.updateRecord('processed', 'true', str(id))
+ wp.updateRecord('error', str(error).lower(), str(id))
+ wp.updateRecord('errorMessage', errorMsg, str(id))
-# if method=='google':
-# print 'Latest Version (' + pkgname + '): ',
-# try:
-# latestVer = Upstream.Google(pkgname, url).process(False)
-# if latestVer!=None:
-# print latestVer
-# except:
-# print 'Can\'t find version'
-# pass
-
-# if method=='ftpls':
-# print 'Latest Version (' + pkgname + '): ',
-# try:
-# latestVer = Upstream.FTPLS(pkgname, url).process(False)
-# if latestVer!=None:
-# print latestVer
-# except:
-# print 'Can\'t find version'
-# pass
-
-# if method=='trac':
-# print 'Latest Version (' + pkgname + '): ',
-# try:
-# latestVer = Upstream.Trac(pkgname, url).process(False)
-# if latestVer!=None:
-# print latestVer
-# except:
-# print 'Can\'t find version'
-# pass
-
- if method=='subdirhttpls':
- print 'Latest Version (' + pkgname + '): ',
- try:
- latestVer = Upstream.SubdirHTTPLS(pkgname, url).process(False)
- if latestVer!=None:
+ if method=='dualhttpls':
+ print 'Latest Version (' + pkgname + '): ',
+ try:
+ (latestVer, path, error, errorMsg) = Upstream.DualHTTPLS(pkgname, url).process(False)
+ print latestVer
+ except:
+ print 'Can\'t find version'
+ latestVer=None
+ path=None
+ error=True
+ errorMsg='Unknown Error'
+
+ wp.updateRecord('latest_ver', latestVer, str(id))
+ wp.updateRecord('processed', 'true', str(id))
+ wp.updateRecord('error', str(error).lower(), str(id))
+ wp.updateRecord('errorMessage', errorMsg, str(id))
+
+
+
+ if method=='lp':
+ print 'Latest Version (' + pkgname + '): ',
+ try:
+ (latestVer, path, error, errorMsg) = Upstream.Launchpad(pkgname, url).process(False)
+ print latestVer
+ except:
+ print 'Can\'t find version'
+ latestVer=None
+ path=None
+ error=True
+ errorMsg='Unknown Error'
+
+ wp.updateRecord('latest_ver', latestVer, str(id))
+ wp.updateRecord('processed', 'true', str(id))
+ wp.updateRecord('error', str(error).lower(), str(id))
+ wp.updateRecord('errorMessage', errorMsg, str(id))
+
+ if method=='svnls':
+ print 'Latest Version (' + pkgname + '): ',
+ try:
+ (latestVer, path, error, errorMsg) = Upstream.SVNLS(pkgname, url).process(False)
+ print latestVer
+ except:
+ print 'Can\'t find version'
+ latestVer=None
+ path=None
+ error=True
+ errorMsg='Unknown Error'
+
+ wp.updateRecord('latest_ver', latestVer, str(id))
+ wp.updateRecord('processed', 'true', str(id))
+ wp.updateRecord('error', str(error).lower(), str(id))
+ wp.updateRecord('errorMessage', errorMsg, str(id))
+
+
+ if method=='google':
+ print 'Latest Version (' + pkgname + '): ',
+ try:
+ (latestVer, path, error, errorMsg) = Upstream.Google(pkgname, url).process(False)
+ print latestVer
+ except:
+ print 'Can\'t find version'
+ latestVer=None
+ path=None
+ error=True
+ errorMsg='Unknown Error'
+
+ wp.updateRecord('latest_ver', latestVer, str(id))
+ wp.updateRecord('processed', 'true', str(id))
+ wp.updateRecord('error', str(error).lower(), str(id))
+ wp.updateRecord('errorMessage', errorMsg, str(id))
+
+ if method=='ftpls':
+ print 'Latest Version (' + pkgname + '): ',
+ try:
+ (latestVer, path, error, errorMsg) = Upstream.FTPLS(pkgname, url).process(False)
+ print latestVer
+ except:
+ print 'Can\'t find version'
+ latestVer=None
+ path=None
+ error=True
+ errorMsg='Unknown Error'
+
+ wp.updateRecord('latest_ver', latestVer, str(id))
+ wp.updateRecord('processed', 'true', str(id))
+ wp.updateRecord('error', str(error).lower(), str(id))
+ wp.updateRecord('errorMessage', errorMsg, str(id))
+
+ if method=='trac':
+ print 'Latest Version (' + pkgname + '): ',
+ try:
+ (latestVer, path, error, errorMsg) = Upstream.Trac(pkgname, url).process(False)
+ print latestVer
+ except:
+ print 'Can\'t find version'
+ latestVer=None
+ path=None
+ error=True
+ errorMsg='Unknown Error'
+
+ wp.updateRecord('latest_ver', latestVer, str(id))
+ wp.updateRecord('processed', 'true', str(id))
+ wp.updateRecord('error', str(error).lower(), str(id))
+ wp.updateRecord('errorMessage', errorMsg, str(id))
+
+
+ if method=='subdirhttpls':
+ print 'Latest Version (' + pkgname + '): ',
+ try:
+ (latestVer, path, error, errorMsg) = Upstream.SubdirHTTPLS(pkgname, url).process(False)
print latestVer
- except:
- print 'Can\'t find version'
- pass
+ except:
+ print 'Can\'t find version'
+ latestVer=None
+ path=None
+ error=True
+ errorMsg='Unknown Error'
+
+ wp.updateRecord('latest_ver', latestVer, str(id))
+ wp.updateRecord('processed', 'true', str(id))
+ wp.updateRecord('error', str(error).lower(), str(id))
+ wp.updateRecord('errorMessage', errorMsg, str(id))
Please sign in to comment.
Something went wrong with that request. Please try again.