Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

added support for pagination

  • Loading branch information...
commit d995786850bda95c9b502fbbfbde7ff32aa4722a 1 parent b8cfed1
@wendlers authored
View
14 README.md
@@ -233,6 +233,20 @@ By doing so, log messages are printed to "stdout", while the log-file specified
Customize
---------
+__General ScoundCloud API Settings__
+
+The general SoundCloud API settings are done under the *[scapi]* section. Currently the only
+thing configurable here is the number of items (groups, users, tracks) to retrive from SC
+per request. The maximum number one could specify for "maxitems" is 8000 (this is the upper
+limmit of SC):
+
+ [scapi]
+ maxitems=400
+
+Note: if you use a slow internet connection, it is likely that your clients time out when
+maxitems is choosen to high.
+
+
__Favorite Users__
To modify the list of your favorite users to browse, edit the *[favorite-users]* section
View
12 etc/pyscmpd.conf
@@ -21,6 +21,18 @@ level=info
# File to which daemon should log. Uncomment to output on stdout
file=/tmp/pyscmpd.log
+[scapi]
+# Number of items (users/groups/tracks) to retrive on requests.
+# The maximum number is 8000. One should also take into account,
+# that on a slow connection clients may time out when retriving
+# takes too long.
+maxitems=1000
+
+# You could specify a proxy host and porxy port to use for connection
+# tp the sound-cloud API.
+#proxyhost=my.proxy.domain
+#proxyport=3128
+
##
# Favorite users
#
View
8 src/pyscmpd/daemon.py
@@ -45,6 +45,7 @@ class PyScMpd:
mpd = None
mainloop = None
port = 9900
+ maxitems = 200
favoriteUsers = None
favoriteGroups = None
@@ -225,6 +226,11 @@ def readConfig(self, cfgFile, foreground=False):
sys.stdout.write("Logging to file: %s\n" % logFile)
+ if parser.has_section("scapi"):
+
+ if parser.has_option("scapi", "maxitems"):
+ self.maxitems = parser.getint("scapi", "maxitems")
+
if parser.has_section("favorite-users"):
for category, values in parser.items("favorite-users"):
@@ -270,7 +276,7 @@ def run(self):
logging.info("pyscmpd v%s started" % PYSCMPD_VERSION)
self.mpd = scmpd.ScMpdServerDaemon(self.favoriteUsers, self.favoriteGroups,
- self.favoriteFavorites, self.port)
+ self.favoriteFavorites, self.port, self.maxitems)
self.mainloop = gobject.MainLoop()
self.mainloop.run()
View
2  src/pyscmpd/pyscmpdctrl.py
@@ -50,6 +50,8 @@ def prepare():
f.write("\n[logging]\n")
f.write("level=info\n")
f.write("file=%s\n" % DEF_LOG_FILE)
+ f.write("\n[scapi]\n")
+ f.write("maxitems=1000\n")
f.write("\n[favorite-users]\n")
f.write("electroswing : maddecent, barelylegit\n")
f.write("electrosoul: griz\n")
View
6 src/pyscmpd/scmpd.py
@@ -35,12 +35,14 @@ class ScMpdServerDaemon(mpdserver.MpdServerDaemon):
scroot = None
player = None
- def __init__(self, favoriteUsers, favoriteGroups, favoriteFavorites, serverPort = 9900):
+ def __init__(self, favoriteUsers, favoriteGroups, favoriteFavorites, serverPort = 9900, maxitems = 200):
ScMpdServerDaemon.player = gstplayer.GstPlayer()
ScMpdServerDaemon.player.retrivePlaylist()
- ScMpdServerDaemon.scp = provider.ResourceProvider(favoriteUsers, favoriteGroups, favoriteFavorites)
+ ScMpdServerDaemon.scp = provider.ResourceProvider(favoriteUsers, favoriteGroups,
+ favoriteFavorites, maxitems)
+
ScMpdServerDaemon.scroot = ScMpdServerDaemon.scp.getRoot()
mpdserver.MpdServerDaemon.__init__(self, serverPort)
View
162 src/pyscmpd/scprovider.py
@@ -32,9 +32,17 @@ class ResourceProvider:
sc = None
root = None
+ reqLimit = 200
+ reqMaxitems = 1000
- def __init__(self, favoriteUsers, favoriteGroups, favoriteFavorites):
+ def __init__(self, favoriteUsers, favoriteGroups, favoriteFavorites, maxitems):
+
+ if ResourceProvider.reqLimit > maxitems:
+ ResourceProvider.reqLimit = maxitems
+
+ ResourceProvider.reqMaxitems = maxitems
+ logging.info("Retriving a maximum of %d items per scapi request" % ResourceProvider.reqMaxitems)
ResourceProvider.sc = soundcloud.Client(client_id='aa13bebc2d26491f7f8d1e77ae996a64')
self.root = Root(favoriteUsers, favoriteGroups, favoriteFavorites)
@@ -207,17 +215,37 @@ def retriveChildren(self):
try:
- groups = ResourceProvider.sc.get("/groups")
+ lim = ResourceProvider.reqLimit
+ off = 0
+
+ while True:
+
+ logging.info("Requesting page with limit=%d; offset=%d" % (lim, off))
+ groups = ResourceProvider.sc.get("/groups", limit=lim, offset=off)
- for group in groups:
- logging.info("processing group %s" % group.permalink)
- g = Group(resource.ID_OFFSET + group.id, group.permalink, self.category + "/" + self.name)
- g.setMeta({"directory" : self.category + "/" + self.name + "/"
- + group.permalink})
- self.addChild(g)
+ if len(groups) == 0:
+ break
+
+ for group in groups:
+
+ logging.info("processing group %s" % group.permalink)
+
+ g = Group(resource.ID_OFFSET + group.id, group.permalink,
+ self.category + "/" + self.name)
+ g.setMeta({"directory" : self.category + "/" + self.name + "/"
+ + group.permalink})
+ self.addChild(g)
+
+ logging.info("successfully retrieved data for URI %s: id=%d; name=%s" %
+ (group.uri, g.getId(), group.permalink))
+
+ off = off + lim
+
+ if off >= ResourceProvider.reqMaxitems:
+ logging.info("Maximum request offset exceeded limit of %d. Ending this request." %
+ ResourceProvider.reqMaxitems)
+ break
- logging.info("successfully retrieved data for URI %s: id=%d; name=%s" %
- (group.uri, g.getId(), group.permalink))
except Exception as e:
logging.warn("Unable to retrive data for groups: %s" % `e`)
@@ -251,23 +279,41 @@ def retriveChildren(self):
self.children = []
try:
- allUsers = ResourceProvider.sc.get("/users")
- for user in allUsers:
-
- try:
+ lim = ResourceProvider.reqLimit
+ off = 0
- u = User(resource.ID_OFFSET + user.id, user.uri + "/tracks", user.permalink,
- user.username, self.category + "/" + self.name)
- u.setMeta({"directory" : self.category + "/" + self.name + "/" + user.permalink})
+ while True:
- self.addChild(u)
+ logging.info("Requesting page with limit=%d; offset=%d" % (lim, off))
- logging.info("successfully retrieved data for URI %s: id=%d; name=%s" %
- (user.uri, u.getId(), user.permalink))
+ allUsers = ResourceProvider.sc.get("/users", limit=lim, offset=off)
+
+ if len(allUsers) == 0:
+ break
+
+ for user in allUsers:
+
+ try:
+
+ u = User(resource.ID_OFFSET + user.id, user.uri + "/tracks", user.permalink,
+ user.username, self.category + "/" + self.name)
+ u.setMeta({"directory" : self.category + "/" + self.name + "/" + user.permalink})
+
+ self.addChild(u)
+
+ logging.info("successfully retrieved data for URI %s: id=%d; name=%s" %
+ (user.uri, u.getId(), user.permalink))
- except Exception as e:
- logging.warn("Unable to retrive data for user: %s" % `e`)
+ except Exception as e:
+ logging.warn("Unable to retrive data for user: %s" % `e`)
+
+ off = off + lim
+
+ if off >= ResourceProvider.reqMaxitems:
+ logging.info("Maximum request offset exceeded limit of %d. Ending this request." %
+ ResourceProvider.reqMaxitems)
+ break
except Exception as e:
logging.warn("Unable to retrive data for URI users: %s" % `e`)
@@ -316,20 +362,37 @@ def retriveChildren(self):
groupId = self.id
else:
groupId = self.getId() - resource.ID_OFFSET
-
- users = ResourceProvider.sc.get("/groups/%d/users" % groupId)
- for user in users:
+ lim = ResourceProvider.reqLimit
+ off = 0
- u = User(resource.ID_OFFSET + user.id, user.uri + "/tracks", user.permalink, user.username,
- self.category + "/" + self.name)
+ while True:
- u.setMeta({"directory" : self.category + "/" + self.name + "/" + user.permalink})
+ logging.info("Requesting page with limit=%d; offset=%d" % (lim, off))
- children.append(u)
+ users = ResourceProvider.sc.get("/groups/%d/users" % groupId, limit=lim, offset=off)
+
+ if len(users) == 0:
+ break
+
+ for user in users:
+
+ u = User(resource.ID_OFFSET + user.id, user.uri + "/tracks", user.permalink,
+ user.username, self.category + "/" + self.name)
+
+ u.setMeta({"directory" : self.category + "/" + self.name + "/" + user.permalink})
- logging.info("Successfully retrieved user data: id=%d; name=%s" %
- (u.getId(), user.permalink))
+ children.append(u)
+
+ logging.info("Successfully retrieved user data: id=%d; name=%s" %
+ (u.getId(), user.permalink))
+
+ off = off + lim
+
+ if off >= ResourceProvider.reqMaxitems:
+ logging.info("Maximum request offset exceeded limit of %d. Ending this request." %
+ ResourceProvider.reqMaxitems)
+ break
except Exception as e:
logging.warn("Unable to retrive data for group %d: %s" % (groupId, `e`))
@@ -369,19 +432,36 @@ def retriveChildren(self):
try:
- tracks = ResourceProvider.sc.get(self.location)
+ lim = ResourceProvider.reqLimit
+ off = 0
+
+ while True:
+
+ logging.info("Requesting page with limit=%d; offset=%d" % (lim, off))
+
+ tracks = ResourceProvider.sc.get(self.location, limit=lim, offset=off)
+
+ if len(tracks) == 0:
+ break
+
+ for track in tracks:
+ tr = Track(resource.ID_OFFSET + track.id, track.stream_url, track.permalink)
+ tr.setMeta({
+ "file" : self.category + "/" + self.name + "/" + track.permalink,
+ "Artist" : self.artist,
+ "Title" : track.title,
+ "Time" : track.duration})
+
+ children.append(tr)
- for track in tracks:
- tr = Track(resource.ID_OFFSET + track.id, track.stream_url, track.permalink)
- tr.setMeta({
- "file" : self.category + "/" + self.name + "/" + track.permalink,
- "Artist" : self.artist,
- "Title" : track.title,
- "Time" : track.duration})
+ logging.debug("Added track to user [%s]: %s" % (self.getName(), track.title))
- children.append(tr)
+ off = off + lim
- logging.debug("Added track to user [%s]: %s" % (self.getName(), track.title))
+ if off >= ResourceProvider.reqMaxitems:
+ logging.info("Maximum request offset exceeded limit of %d. Ending this request." %
+ ResourceProvider.reqMaxitems)
+ break
except Exception as e:
Please sign in to comment.
Something went wrong with that request. Please try again.