From 14bca87570a95b8d50bb50e95d4974e3188071a3 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Thu, 16 May 2013 18:56:44 -0400 Subject: [PATCH 01/72] advance develop version --- HISTORY.rst | 5 +++++ gmusicapi/_version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1e11b0e7..db4aed68 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,11 @@ History As of 1.0.0, `semantic versioning `__ is used. +1.2.0-dev ++++++++++ +release 2013-XX-XX + + 1.2.0 +++++ released 2013-05-16 diff --git a/gmusicapi/_version.py b/gmusicapi/_version.py index c68196d1..ca5babe0 100644 --- a/gmusicapi/_version.py +++ b/gmusicapi/_version.py @@ -1 +1 @@ -__version__ = "1.2.0" +__version__ = "1.2.0-dev" From 3a2adf306610aa1186e31a39caf0820fe56e523d Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sat, 18 May 2013 01:31:57 -0300 Subject: [PATCH 02/72] add @Stono's sync tool to example projects --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index acc05ae8..f80ffec9 100644 --- a/README.rst +++ b/README.rst @@ -31,6 +31,7 @@ That said, it's actively maintained, and used in a bunch of cool projects: - Kilian Lackhove's `Google Music support `__ for http://www.tomahawk-player.org - Tom Graham's `playlist syncing tool `__ +- Karl Stoney's `sync tool `__ Getting started From 8b9b057f56ccbcfd0503f5a6527c9b6c197251b6 Mon Sep 17 00:00:00 2001 From: Ryan McGuire Date: Tue, 21 May 2013 18:40:12 -0400 Subject: [PATCH 03/72] Adds multi-part streaming support to get_stream_url --- gmusicapi/clients.py | 5 ++++- gmusicapi/protocol/webclient.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index c8865e91..e8a67c80 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -905,7 +905,10 @@ def get_stream_url(self, song_id): res = self._make_call(webclient.GetStreamUrl, song_id) - return res['url'] + try: + return res['url'] + except KeyError: + return res['urls'] def copy_playlist(self, playlist_id, copy_name): """Copies the contents of a playlist to a new playlist. Returns the id of the new playlist. diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 7d73ff3c..9b103a3d 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -503,7 +503,8 @@ class GetStreamUrl(WcCall): _res_schema = { "type": "object", "properties": { - "url": {"type": "string"} + "url": {"type": "string", "required": False}, + "urls": {"type": "array", "required": False} }, "additionalProperties": False } From d483c960a66fdbead375722bab0aed4a62564f45 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Thu, 23 May 2013 13:37:00 -0300 Subject: [PATCH 04/72] note absence in readme --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index f80ffec9..5c1172e2 100644 --- a/README.rst +++ b/README.rst @@ -48,6 +48,8 @@ Status and updates .. image:: https://travis-ci.org/simon-weber/Unofficial-Google-Music-API.png?branch=develop :target: https://travis-ci.org/simon-weber/Unofficial-Google-Music-API +I'll be without an internet connection until June 2nd. + Version 1.2.0 fixes a bug that fixes uploader_id formatting from a mac address. This change may cause another machine to be registered - you can safely remove the old machine (it's the one without the version in the name). From d6880dfadb99aefbdeefb14475956faf88a72c64 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 3 Jun 2013 16:10:06 -0300 Subject: [PATCH 05/72] remove absence note --- README.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.rst b/README.rst index 5c1172e2..f80ffec9 100644 --- a/README.rst +++ b/README.rst @@ -48,8 +48,6 @@ Status and updates .. image:: https://travis-ci.org/simon-weber/Unofficial-Google-Music-API.png?branch=develop :target: https://travis-ci.org/simon-weber/Unofficial-Google-Music-API -I'll be without an internet connection until June 2nd. - Version 1.2.0 fixes a bug that fixes uploader_id formatting from a mac address. This change may cause another machine to be registered - you can safely remove the old machine (it's the one without the version in the name). From c6efdd484c6b25ca0ea825d89e19c7d147a011d2 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 12 Jun 2013 12:27:13 -0400 Subject: [PATCH 06/72] can get AA urls successfully --- docs/source/reference/webclient.rst | 2 +- gmusicapi/clients.py | 24 ++++++++++++++++-------- gmusicapi/protocol/webclient.py | 18 ++++++++++++++---- gmusicapi/test/server_tests.py | 9 +++++++-- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/docs/source/reference/webclient.rst b/docs/source/reference/webclient.rst index c0c9d8a9..17cdc61f 100644 --- a/docs/source/reference/webclient.rst +++ b/docs/source/reference/webclient.rst @@ -22,7 +22,7 @@ Getting songs and playlists Song downloading and streaming ------------------------------ .. automethod:: Webclient.get_song_download_info -.. automethod:: Webclient.get_stream_url +.. automethod:: Webclient.get_stream_urls .. automethod:: Webclient.report_incorrect_match Song manipulation diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index e8a67c80..552ed6d6 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -885,28 +885,36 @@ def get_song_download_info(self, song_id): return (url, info["downloadCounts"][song_id]) - def get_stream_url(self, song_id): - """Returns a url that points to a streamable version of this song. + def get_stream_urls(self, song_id): + """Return a list of urls that point to a streamable version of this song. + + If you just need the audio and are ok with gmusicapi doing the download, + consider using :func:`get_stream_audio` instead. + This abstracts away the differences between different kinds of tracks: + * normal tracks return a single url + * All Access tracks return multiple urls, which must be combined :param song_id: a single song id. - While acquiring the url requires authentication, retreiving the - url contents does not. + While acquiring the urls requires authentication, retreiving the + contents does not. - However, there are limitation as to how the stream url can be used: - * the url expires after about a minute + However, there are limitations on how the stream urls can be used: + * the urls expire after a minute * only one IP can be streaming music at once. Other attempts will get an http 403 with ``X-Rejected-Reason: ANOTHER_STREAM_BEING_PLAYED``. *This is only intended for streaming*. The streamed audio does not contain metadata. - Use :func:`get_song_download_info` to download complete files with metadata. + Use :func:`get_song_download_info` or :func:`Musicmanager.download_song + ` + to download files with metadata. """ res = self._make_call(webclient.GetStreamUrl, song_id) try: - return res['url'] + return [res['url']] except KeyError: return res['urls'] diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 9b103a3d..78837ce9 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -511,12 +511,22 @@ class GetStreamUrl(WcCall): @staticmethod def dynamic_params(song_id): - return { - 'u': 0, # select first user of logged in; probably shouldn't be hardcoded - 'pt': 'e', # unknown - 'songid': song_id, + params = { + 'u': 0, + 'pt': 'e' } + # all access streams use a different param + # https://github.com/simon-weber/Unofficial-Google-Music-API/pull/131#issuecomment-18843993 + # thanks @lukegb! + + if song_id[0] == 'T': + # all access + params['mjck'] = song_id + else: + params['songid'] = song_id + return params + class Search(WcCall): """Fuzzily search for songs, artists and albums. diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index bed92122..117ef79d 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -299,10 +299,15 @@ def assert_download(sid=self.song.sid): assert_download() @song_test - def get_stream_url(self): - url = self.wc.get_stream_url(self.song.sid) + def get_normal_stream_urls(self): + urls = self.wc.get_stream_urls(self.song.sid) + + assert_equal(len(urls), 1) + + url = urls[0] assert_is_not_none(url) + assert_equal(url[:7], 'http://') @song_test def upload_album_art(self): From a7c27f879f53f125506aed7f184ecc73d48415c5 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 12 Jun 2013 14:03:21 -0400 Subject: [PATCH 07/72] add get_stream_audio --- docs/source/reference/webclient.rst | 1 + gmusicapi/clients.py | 33 ++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/source/reference/webclient.rst b/docs/source/reference/webclient.rst index 17cdc61f..dab1ea01 100644 --- a/docs/source/reference/webclient.rst +++ b/docs/source/reference/webclient.rst @@ -22,6 +22,7 @@ Getting songs and playlists Song downloading and streaming ------------------------------ .. automethod:: Webclient.get_song_download_info +.. automethod:: Webclient.get_stream_audio .. automethod:: Webclient.get_stream_urls .. automethod:: Webclient.report_incorrect_match diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index 552ed6d6..2d02fe4f 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -11,6 +11,7 @@ from socket import gethostname import time import urllib +from urlparse import urlparse, parse_qsl from uuid import getnode as getmac import webbrowser @@ -886,7 +887,7 @@ def get_song_download_info(self, song_id): return (url, info["downloadCounts"][song_id]) def get_stream_urls(self, song_id): - """Return a list of urls that point to a streamable version of this song. + """Returns a list of urls that point to a streamable version of this song. If you just need the audio and are ok with gmusicapi doing the download, consider using :func:`get_stream_audio` instead. @@ -918,6 +919,36 @@ def get_stream_urls(self, song_id): except KeyError: return res['urls'] + def get_stream_audio(self, song_id): + """Returns a bytestring containing mp3 audio for this song. + + :param song_id: a single song id + """ + + urls = self.get_stream_urls(song_id) + + if len(urls) == 1: + return self.session._rsession.get(urls[0]).content + + # AA tracks are separated into multiple files + # the url contains the range of each file to be used + + range_pairs = [[int(s) for s in val.split('-')] + for url in urls + for key, val in parse_qsl(urlparse(url)[4]) + if key == 'range'] + + stream_pieces = [] + prev_end = 0 + + for url, (start, end) in zip(urls, range_pairs): + audio = self.session._rsession.get(url).content + stream_pieces.append(audio[prev_end - start:]) + + prev_end = end + 1 + + return ''.join(stream_pieces) + def copy_playlist(self, playlist_id, copy_name): """Copies the contents of a playlist to a new playlist. Returns the id of the new playlist. From 61b87785dec0eee63eff5b4f893e9cdb86710071 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 12 Jun 2013 14:09:43 -0400 Subject: [PATCH 08/72] add aa urls test --- gmusicapi/test/server_tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 117ef79d..52b024dd 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -309,6 +309,13 @@ def get_normal_stream_urls(self): assert_is_not_none(url) assert_equal(url[:7], 'http://') + @song_test + def get_aa_stream_urls(self): + # that dumb little intro track on Conspiracy of One + urls = self.wc.get_stream_urls('Tqqufr34tuqojlvkolsrwdwx7pe') + + assert_true(len(urls) > 1) + @song_test def upload_album_art(self): orig_md = self._assert_get_song(self.song.sid) From a9fba63e11f0939a9787e1f91f09c1fddf540880 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 12 Jun 2013 14:15:28 -0400 Subject: [PATCH 09/72] disable non-AA trialed test account --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5e8083ac..c151e709 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,3 @@ branches: env: matrix: - secure: "UAXrAYzrYRVFWSRldd5o9NrreexCsXdBna/kcKLlAQ8ygRahpqP7qXX8qiNU\nVhz0kdgBxgS84AEE3H/30o9v2IDWqmJCI5OqMfH1o5Pm+CelBt+8FHu3SMjm\nNvNcm/Vmip7WCSx7P2FfOf8HboSH/kVXuF0iPlOdozrTR1wPUq8=" - - secure: "ILJA2Vhh3BRav2xLIMxS7/mE65AF15OA7UmDNdqE22qUoeEBL+ZduBQY8oAz\n+pJwHndP8bdD23Y9PLdC99qm1yZRV1x3yY1JclKzYnuwbeI1zYFHfGXTjtM+\n+rxxwo3ygEzrDnzIqKqP1+xKpKAGNqqLj0Pro8quyLhtKNUImMc=" From 791cbbe340c2777310fad55889ab991086d9f31d Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 12 Jun 2013 15:06:29 -0400 Subject: [PATCH 10/72] support py2.7.5 --- setup.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index dc5e2fea..be033f88 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ import sys #Only 2.6-2.7 are supported. -if not ((2, 6, 0) <= sys.version_info[:3] <= (2, 7, 4)): +if not ((2, 6, 0) <= sys.version_info[:3] <= (2, 7, 5)): sys.stderr.write('gmusicapi does not officially support this Python version.\n') #try to continue anyway @@ -42,16 +42,16 @@ long_description=(open('README.rst').read() + '\n\n' + open('HISTORY.rst').read()), install_requires=[ - 'validictory == 0.9.0', - 'decorator == 3.3.2', - 'mutagen == 1.21', - 'protobuf == 2.4.1', - 'requests == 1.2.0', - 'python-dateutil == 2.1', - 'proboscis==1.2.5.3', - 'oauth2client==1.1', - 'mock==1.0.1', - 'appdirs==1.2.0', + 'validictory == 0.9.0', # validation + 'decorator == 3.3.2', # keep + 'mutagen == 1.21', # MM + 'protobuf == 2.4.1', # MM + 'requests == 1.2.0', # keep + 'python-dateutil == 2.1', # MM + 'proboscis==1.2.5.3', # testing + 'oauth2client==1.1', # MM + 'mock==1.0.1', # testing + 'appdirs==1.2.0', # keep ] + dynamic_requires, classifiers=[ 'Programming Language :: Python', From b8f48cd987608b6cebe418a55f96f86829961a8d Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Thu, 13 Jun 2013 11:45:29 -0400 Subject: [PATCH 11/72] clarify timestamp format [#132] --- gmusicapi/protocol/metadata.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gmusicapi/protocol/metadata.py b/gmusicapi/protocol/metadata.py index 60066222..e115ff1a 100644 --- a/gmusicapi/protocol/metadata.py +++ b/gmusicapi/protocol/metadata.py @@ -135,7 +135,7 @@ def get_schema(self): ('subjectToCuration', 'boolean', 'meaning unknown.'), ('matchedId', 'string', 'meaning unknown; related to scan and match?'), - ('recentTimestamp', 'integer', 'meaning unknown.'), + ('recentTimestamp', 'integer', 'UTC/microsecond timestamp: meaning unknown.'), ) ] + [ Expectation(name, type_str, mutable=False, optional=True, explanation=explain) @@ -150,7 +150,7 @@ def get_schema(self): ('playlistEntryId', 'string', 'identifies position in the context of a playlist.'), ('albumArtUrl', 'string', "if present, the url of an image for this song's album art."), ('artistMatchedId', 'string', 'id of a matching artist in the Play Store?'), - ('albumPlaybackTimestamp', 'integer', 'the last time this album was played?'), + ('albumPlaybackTimestamp', 'integer', 'UTC/microsecond timestamp: the last time this album was played?'), ('origin', 'array', '???'), ('artistImageBaseUrl', 'string', 'like albumArtUrl, but for the artist. May be blank.'), ) @@ -169,7 +169,9 @@ def get_schema(self): optional=False, allowed_values=tuple(range(6)), explanation='0 == no thumb, 1 == down thumb, 5 == up thumb.'), - Expectation('lastPlayed', 'integer', mutable=False, optional=True, volatile=True), + Expectation('lastPlayed', 'integer', mutable=False, optional=True, volatile=True, + explanation='UTC/microsecond timestamp'), + Expectation('playCount', 'integer', mutable=True, optional=False), Expectation('title', 'string', mutable=False, optional=False, From de50406afde15329ad259cee11c6d88408538060 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Thu, 13 Jun 2013 13:54:06 -0300 Subject: [PATCH 12/72] add @thebigmunch's scripts to the README --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f80ffec9..d837d50b 100644 --- a/README.rst +++ b/README.rst @@ -27,9 +27,10 @@ That said, it's actively maintained, and used in a bunch of cool projects: (`screenshot `__) - Ryan McGuire's `GMusicFS `__ - a FUSE filesystem linked to your music -- David Dooling's `sync scripts for Banshee `__ - Kilian Lackhove's `Google Music support `__ for http://www.tomahawk-player.org +- `@thebigmunch `__'s `syncing scripts `__ +- David Dooling's `sync scripts for Banshee `__ - Tom Graham's `playlist syncing tool `__ - Karl Stoney's `sync tool `__ From de663fd6a0e5f650bbd1b9bbf54a4abfdef32f39 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Thu, 13 Jun 2013 15:10:56 -0400 Subject: [PATCH 13/72] clean up top of files --- gmusicapi/__init__.py | 1 - gmusicapi/clients.py | 1 - gmusicapi/exceptions.py | 1 - gmusicapi/gmtools/__init__.py | 27 +-------------------------- gmusicapi/gmtools/tools.py | 1 - gmusicapi/protocol/__init__.py | 1 + gmusicapi/protocol/metadata.py | 1 - gmusicapi/protocol/musicmanager.py | 1 - gmusicapi/protocol/shared.py | 1 - gmusicapi/protocol/webclient.py | 1 - gmusicapi/session.py | 1 - gmusicapi/test/__init__.py | 27 +-------------------------- gmusicapi/test/local_tests.py | 3 +-- gmusicapi/test/run_tests.py | 3 +++ gmusicapi/test/server_tests.py | 2 ++ gmusicapi/test/utils.py | 1 - gmusicapi/utils/__init__.py | 27 +-------------------------- gmusicapi/utils/counter.py | 9 ++++++--- gmusicapi/utils/utils.py | 1 - setup.py | 1 + 20 files changed, 17 insertions(+), 94 deletions(-) diff --git a/gmusicapi/__init__.py b/gmusicapi/__init__.py index 1b6d6881..ed0f63a9 100644 --- a/gmusicapi/__init__.py +++ b/gmusicapi/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- from gmusicapi._version import __version__ diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index 2d02fe4f..49fe91c1 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ diff --git a/gmusicapi/exceptions.py b/gmusicapi/exceptions.py index ab73adcd..13a4a973 100644 --- a/gmusicapi/exceptions.py +++ b/gmusicapi/exceptions.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """Custom exceptions used across the project.""" diff --git a/gmusicapi/gmtools/__init__.py b/gmusicapi/gmtools/__init__.py index 9ee3d617..40a96afc 100644 --- a/gmusicapi/gmtools/__init__.py +++ b/gmusicapi/gmtools/__init__.py @@ -1,26 +1 @@ -#!/usr/bin/env python - -# Copyright (c) 2012, Simon Weber -# All rights reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of the copyright holder nor the -# names of the contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# -*- coding: utf-8 -*- diff --git a/gmusicapi/gmtools/tools.py b/gmusicapi/gmtools/tools.py index 8b3f6d03..fe386fa5 100644 --- a/gmusicapi/gmtools/tools.py +++ b/gmusicapi/gmtools/tools.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """Tools for manipulating client-received Google Music data.""" diff --git a/gmusicapi/protocol/__init__.py b/gmusicapi/protocol/__init__.py index e69de29b..40a96afc 100644 --- a/gmusicapi/protocol/__init__.py +++ b/gmusicapi/protocol/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/gmusicapi/protocol/metadata.py b/gmusicapi/protocol/metadata.py index e115ff1a..deda759b 100644 --- a/gmusicapi/protocol/metadata.py +++ b/gmusicapi/protocol/metadata.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ diff --git a/gmusicapi/protocol/musicmanager.py b/gmusicapi/protocol/musicmanager.py index 57e9b3a5..171a0acf 100644 --- a/gmusicapi/protocol/musicmanager.py +++ b/gmusicapi/protocol/musicmanager.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """Calls made by the Music Manager (related to uploading).""" diff --git a/gmusicapi/protocol/shared.py b/gmusicapi/protocol/shared.py index 6009ade4..598c1cc8 100644 --- a/gmusicapi/protocol/shared.py +++ b/gmusicapi/protocol/shared.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """Definitions shared by multiple clients.""" diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 78837ce9..36921060 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """Calls made by the web client.""" diff --git a/gmusicapi/session.py b/gmusicapi/session.py index 2b88f5bf..4a4e33ad 100644 --- a/gmusicapi/session.py +++ b/gmusicapi/session.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ diff --git a/gmusicapi/test/__init__.py b/gmusicapi/test/__init__.py index 9ee3d617..40a96afc 100644 --- a/gmusicapi/test/__init__.py +++ b/gmusicapi/test/__init__.py @@ -1,26 +1 @@ -#!/usr/bin/env python - -# Copyright (c) 2012, Simon Weber -# All rights reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of the copyright holder nor the -# names of the contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# -*- coding: utf-8 -*- diff --git a/gmusicapi/test/local_tests.py b/gmusicapi/test/local_tests.py index 3716a49d..81734d2a 100644 --- a/gmusicapi/test/local_tests.py +++ b/gmusicapi/test/local_tests.py @@ -1,11 +1,10 @@ -from collections import namedtuple -#!/usr/bin/env python # -*- coding: utf-8 -*- """ Tests that don't hit the Google Music servers. """ +from collections import namedtuple import time from mock import MagicMock as Mock diff --git a/gmusicapi/test/run_tests.py b/gmusicapi/test/run_tests.py index b2ed17f5..b2e9f6d6 100644 --- a/gmusicapi/test/run_tests.py +++ b/gmusicapi/test/run_tests.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + from functools import partial, update_wrapper from getpass import getpass import logging diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 52b024dd..61ff8585 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + """ These tests all run against an actual Google Music account. diff --git a/gmusicapi/test/utils.py b/gmusicapi/test/utils.py index a8b021ee..582fdb97 100644 --- a/gmusicapi/test/utils.py +++ b/gmusicapi/test/utils.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """Utilities used in testing.""" diff --git a/gmusicapi/utils/__init__.py b/gmusicapi/utils/__init__.py index 9ee3d617..40a96afc 100644 --- a/gmusicapi/utils/__init__.py +++ b/gmusicapi/utils/__init__.py @@ -1,26 +1 @@ -#!/usr/bin/env python - -# Copyright (c) 2012, Simon Weber -# All rights reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of the copyright holder nor the -# names of the contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# -*- coding: utf-8 -*- diff --git a/gmusicapi/utils/counter.py b/gmusicapi/utils/counter.py index 5fbd5c8a..0c8f66cf 100644 --- a/gmusicapi/utils/counter.py +++ b/gmusicapi/utils/counter.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + from operator import itemgetter from heapq import nlargest from itertools import repeat, ifilter @@ -22,7 +25,7 @@ def __init__(self, iterable=None, **kwds): >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping >>> c = Counter(a=4, b=2) # a new counter from keyword args - ''' + ''' self.update(iterable, **kwds) def __missing__(self, key): @@ -35,7 +38,7 @@ def most_common(self, n=None): >>> Counter('abracadabra').most_common(3) [('a', 5), ('r', 2), ('b', 2)] - ''' + ''' if n is None: return sorted(self.iteritems(), key=itemgetter(1), reverse=True) return nlargest(n, self.iteritems(), key=itemgetter(1)) @@ -74,7 +77,7 @@ def update(self, iterable=None, **kwds): >>> c['h'] # four 'h' in which, witch, and watch 4 - ''' + ''' if iterable is not None: if hasattr(iterable, 'iteritems'): if self: diff --git a/gmusicapi/utils/utils.py b/gmusicapi/utils/utils.py index 44e47bae..8f725dee 100644 --- a/gmusicapi/utils/utils.py +++ b/gmusicapi/utils/utils.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """Utility functions used across api code.""" diff --git a/setup.py b/setup.py index dc5e2fea..96bc1834 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- import re from setuptools import setup, find_packages From 07c892b52fc6cec5ed559f30dd22ccfb30d21cdc Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Thu, 13 Jun 2013 17:19:29 -0400 Subject: [PATCH 14/72] enforce id params [close #103] --- gmusicapi/clients.py | 16 +++-- gmusicapi/utils/utils.py | 144 +++++++++++++++++++++++++-------------- 2 files changed, 104 insertions(+), 56 deletions(-) diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index 49fe91c1..60275d0a 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -370,6 +370,7 @@ def _get_all_songs(self): get_next_chunk = lib_chunk.HasField('continuation_token') + @utils.enforce_id_param def download_song(self, song_id): """Returns a tuple ``(u'suggested_filename', 'audio_bytestring')``. The filename @@ -693,7 +694,7 @@ def change_playlist_name(self, playlist_id, new_name): return playlist_id # the call actually doesn't return anything. @utils.accept_singleton(dict) - @utils.empty_arg_shortcircuit() + @utils.empty_arg_shortcircuit def change_song_metadata(self, songs): """Changes the metadata for some :ref:`song dictionaries `. Returns a list of the song ids changed. @@ -741,7 +742,8 @@ def delete_playlist(self, playlist_id): return res['deleteId'] @utils.accept_singleton(basestring) - @utils.empty_arg_shortcircuit() + @utils.empty_arg_shortcircuit + @utils.enforce_ids_param def delete_songs(self, song_ids): """Deletes songs from the entire library. Returns a list of deleted song ids. @@ -865,6 +867,7 @@ def _get_auto_playlists(self): u'Last added': u'auto-playlist-recent', u'Free and purchased': u'auto-playlist-promo'} + @utils.enforce_id_param def get_song_download_info(self, song_id): """Returns a tuple: ``('', )``. @@ -885,6 +888,7 @@ def get_song_download_info(self, song_id): return (url, info["downloadCounts"][song_id]) + @utils.enforce_id_param def get_stream_urls(self, song_id): """Returns a list of urls that point to a streamable version of this song. @@ -918,6 +922,7 @@ def get_stream_urls(self, song_id): except KeyError: return res['urls'] + @utils.enforce_id_param def get_stream_audio(self, song_id): """Returns a bytestring containing mp3 audio for this song. @@ -1090,6 +1095,7 @@ def change_playlist(self, playlist_id, desired_playlist, safe=True): @utils.accept_singleton(basestring, 2) @utils.empty_arg_shortcircuit(position=2) + @utils.enforce_ids_param(position=2) def add_songs_to_playlist(self, playlist_id, song_ids): """Appends songs to a playlist. Returns a list of (song id, playlistEntryId) tuples that were added. @@ -1225,7 +1231,8 @@ def search(self, query): "song_hits": res["songs"]} @utils.accept_singleton(basestring) - @utils.empty_arg_shortcircuit() + @utils.empty_arg_shortcircuit + @utils.enforce_id_param def report_incorrect_match(self, song_ids): """Equivalent to the 'Fix Incorrect Match' button, this requests re-uploading of songs. Returns the song_ids provided. @@ -1244,7 +1251,8 @@ def report_incorrect_match(self, song_ids): return song_ids @utils.accept_singleton(basestring) - @utils.empty_arg_shortcircuit() + @utils.empty_arg_shortcircuit + @utils.enforce_ids_param def upload_album_art(self, song_ids, image_filepath): """Uploads an image and sets it as the album art for songs. diff --git a/gmusicapi/utils/utils.py b/gmusicapi/utils/utils.py index 8f725dee..e048e98e 100644 --- a/gmusicapi/utils/utils.py +++ b/gmusicapi/utils/utils.py @@ -48,6 +48,59 @@ _mac_pattern = re.compile("^({pair}:){{5}}{pair}$".format(pair='[0-9A-F]' * 2)) +class DynamicClientLogger(object): + """Dynamically proxies to the logger of a Client higher in the call stack. + + This is a ridiculous hack needed because + logging is, in the eyes of a user, per-client. + + So, logging from static code (eg protocol, utils) needs to log using the + config of the calling client's logger. + + There can be multiple clients, so we can't just use a globally-available + logger. + + Instead of refactoring every function to receieve a logger, we introspect + the callstack at runtime to figure out who's calling us, then use their + logger. + + This probably won't work on non-CPython implementations. + """ + + def __init__(self, caller_name): + self.caller_name = caller_name + + def __getattr__(self, name): + # this isn't a totally foolproof way to proxy, but it's fine for + # the usual logger.debug, etc methods. + + logger = logging.getLogger(self.caller_name) + + if per_client_logging: + # search upwards for a client instance + for frame_rec in inspect.getouterframes(inspect.currentframe()): + frame = frame_rec[0] + + try: + if 'self' in frame.f_locals: + f_self = frame.f_locals['self'] + if ((f_self.__module__ == 'gmusicapi.clients' and + type(f_self).__name__ in ('Musicmanager', 'Webclient'))): + logger = f_self.logger + break + finally: + del frame # avoid circular references + + else: + stack = traceback.extract_stack() + logger.info('could not locate client caller in stack:\n%s', + '\n'.join(traceback.format_list(stack))) + + return getattr(logger, name) + + +log = DynamicClientLogger(__name__) + def is_valid_mac(mac_string): """Return True if mac_string is of form eg '00:11:22:33:AA:BB'. @@ -118,73 +171,59 @@ def __new__(meta, name, bases, clsdict): return type.__new__(meta, name, bases, clsdict) -class DynamicClientLogger(object): - """Dynamically proxies to the logger of a Client higher in the call stack. - - This is a ridiculous hack needed because - logging is, in the eyes of a user, per-client. +def dual_decorator(func): + """This is a decorator that converts a paramaterized decorator for no-param use. - So, logging from static code (eg protocol, utils) needs to log using the - config of the calling client's logger. + source: http://stackoverflow.com/questions/3888158. + """ + @functools.wraps(func) + def inner(*args, **kw): + if ((len(args) == 1 and not kw and callable(args[0]) + and not (type(args[0]) == type and issubclass(args[0], BaseException)))): + return func()(args[0]) + else: + return func(*args, **kw) + return inner - There can be multiple clients, so we can't just use a globally-available - logger. - Instead of refactoring every function to receieve a logger, we introspect - the callstack at runtime to figure out who's calling us, then use their - logger. +@dual_decorator +def enforce_id_param(position=1): + """Verifies that the caller is passing a single song id, and not + a song dictionary. - This probably won't work on non-CPython implementations. + :param position: (optional) the position of the expected id - defaults to 1. """ - def __init__(self, caller_name): - self.caller_name = caller_name - - def __getattr__(self, name): - # this isn't a totally foolproof way to proxy, but it's fine for - # the usual logger.debug, etc methods. - - logger = logging.getLogger(self.caller_name) + @decorator + def wrapper(function, *args, **kw): - if per_client_logging: - # search upwards for a client instance - for frame_rec in inspect.getouterframes(inspect.currentframe()): - frame = frame_rec[0] + if not isinstance(args[position], basestring): + raise ValueError("Invalid param type in position %s;" + " expected a song id (did you pass a song dictionary?)" % position) - try: - if 'self' in frame.f_locals: - f_self = frame.f_locals['self'] - if ((f_self.__module__ == 'gmusicapi.clients' and - type(f_self).__name__ in ('Musicmanager', 'Webclient'))): - logger = f_self.logger - break - finally: - del frame # avoid circular references + return function(*args, **kw) - else: - stack = traceback.extract_stack() - logger.info('could not locate client caller in stack:\n%s', - '\n'.join(traceback.format_list(stack))) + return wrapper - return getattr(logger, name) +@dual_decorator +def enforce_ids_param(position=1): + """Verifies that the caller is passing a list of song ids, and not a + list of song dictionaries. + :param position: (optional) the position of the expected list - defaults to 1. + """ -log = DynamicClientLogger(__name__) + @decorator + def wrapper(function, *args, **kw): + if (not isinstance(args[position], (list, tuple)) or + not all([isinstance(e, basestring) for e in args[position]])): + raise ValueError("Invalid param type in position %s;" + " expected song ids (did you pass song dictionaries?)" % position) -def dual_decorator(func): - """This is a decorator that converts a paramaterized decorator for no-param use. + return function(*args, **kw) - source: http://stackoverflow.com/questions/3888158. - """ - @functools.wraps(func) - def inner(*args, **kw): - if ((len(args) == 1 and not kw and callable(args[0]) - and not (type(args[0]) == type and issubclass(args[0], BaseException)))): - return func()(args[0]) - else: - return func(*args, **kw) - return inner + return wrapper def configure_debug_log_handlers(logger): @@ -392,6 +431,7 @@ def truncate(x, max_els=100, recurse_levels=0): return x +@dual_decorator def empty_arg_shortcircuit(return_code='[]', position=1): """Decorate a function to shortcircuit and return something immediately if the length of a positional arg is 0. From bcd42756f5b09ac2bc1ac3fcdf7a6098b190b903 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Thu, 13 Jun 2013 17:20:49 -0400 Subject: [PATCH 15/72] update history --- HISTORY.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index db4aed68..928bcbec 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,8 +7,10 @@ As of 1.0.0, `semantic versioning `__ is used. 1.2.0-dev +++++++++ -release 2013-XX-XX +released 2013-XX-XX +- add support for streaming All Access songs +- terminate early when a song dictionary is passed instead of an id 1.2.0 +++++ From da1cf7b51bfc93411f937bd145b6c6939b21afee Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 17 Jun 2013 10:50:58 -0400 Subject: [PATCH 16/72] close #133 --- gmusicapi/protocol/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gmusicapi/protocol/metadata.py b/gmusicapi/protocol/metadata.py index deda759b..de42c102 100644 --- a/gmusicapi/protocol/metadata.py +++ b/gmusicapi/protocol/metadata.py @@ -124,7 +124,6 @@ def get_schema(self): ('id', 'string', 'a per-user unique id for this song; sometimes referred to as *server id* or *song id*.'), - ('deleted', 'boolean', ''), ('creationDate', 'integer', ''), ('type', 'integer', 'An enum: 1: free/purchased, 2: uploaded/not matched, 6: uploaded/matched'), @@ -134,7 +133,6 @@ def get_schema(self): ('subjectToCuration', 'boolean', 'meaning unknown.'), ('matchedId', 'string', 'meaning unknown; related to scan and match?'), - ('recentTimestamp', 'integer', 'UTC/microsecond timestamp: meaning unknown.'), ) ] + [ Expectation(name, type_str, mutable=False, optional=True, explanation=explain) @@ -152,6 +150,8 @@ def get_schema(self): ('albumPlaybackTimestamp', 'integer', 'UTC/microsecond timestamp: the last time this album was played?'), ('origin', 'array', '???'), ('artistImageBaseUrl', 'string', 'like albumArtUrl, but for the artist. May be blank.'), + ('recentTimestamp', 'integer', 'UTC/microsecond timestamp: meaning unknown.'), + ('deleted', 'boolean', ''), ) ] + [ Expectation(name + 'Norm', 'string', mutable=False, optional=False, From 7528daaba2a9478ae793e8dcad912df3fc68a187 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 17 Jun 2013 11:43:11 -0400 Subject: [PATCH 17/72] close #121 --- gmusicapi/test/run_tests.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/gmusicapi/test/run_tests.py b/gmusicapi/test/run_tests.py index b2e9f6d6..17b2f3cc 100644 --- a/gmusicapi/test/run_tests.py +++ b/gmusicapi/test/run_tests.py @@ -10,7 +10,7 @@ from proboscis import TestProgram -from gmusicapi.clients import Webclient, Musicmanager +from gmusicapi.clients import Webclient, Musicmanager, OAUTH_FILEPATH from gmusicapi.protocol.musicmanager import credentials_from_refresh_token from gmusicapi.test import local_tests, server_tests from gmusicapi.test.utils import NoticeLogging @@ -65,26 +65,31 @@ def freeze_login_details(): else: # no travis, no credentials - - # we need to login here to verify their credentials. - # the authenticated api is then thrown away. - - wclient = Webclient() - valid_auth = False - print ("These tests will never delete or modify your music." "\n\n" "If the tests fail, you *might* end up with a test" " song/playlist in your library, though." - "You must have oauth credentials stored at the default" - " path by Musicmanager.perform_oauth prior to running.") + "\n") + + # check for oauth + try: + with open(OAUTH_FILEPATH) as f: + pass # assume they're valid + + except IOError: + print ("You must have oauth credentials stored at the default" + " path by Musicmanager.perform_oauth prior to running.") + sys.exit(1) + + wclient = Webclient() + valid_wc_auth = False - while not valid_auth: + while not valid_wc_auth: print email = raw_input("Email: ") passwd = getpass() - valid_auth = wclient.login(email, passwd) + valid_wc_auth = wclient.login(email, passwd) wc_kwargs.update({'email': email, 'password': passwd}) From 25b07722aaf1c0c2a01e7331c210dba281feea65 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 17 Jun 2013 14:46:24 -0400 Subject: [PATCH 18/72] add validation toggle in client init (#100, #101) --- gmusicapi/clients.py | 20 ++++++++++++++------ gmusicapi/protocol/shared.py | 6 ++++-- gmusicapi/session.py | 4 ++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index 60275d0a..068945bb 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -38,7 +38,7 @@ class _Base(object): num_clients = 0 # used to disambiguate loggers - def __init__(self, logger_basename, debug_logging): + def __init__(self, logger_basename, debug_logging, validate): """ :param debug_logging: each Client has a ``logger`` member. @@ -63,6 +63,10 @@ def __init__(self, logger_basename, debug_logging): The Google Music protocol can change at any time; if something were to go wrong, the logs would be necessary for recovery. + + :param validate: if False, do not validate server responses against + known schemas. This helps to catch protocol changes, but requires + significant cpu work. """ # this isn't correct if init is called more than once, so we log the # client name below to avoid confusion for people reading logs @@ -71,6 +75,7 @@ def __init__(self, logger_basename, debug_logging): logger_name = "gmusicapi.%s%s" % (logger_basename, _Base.num_clients) self.logger = logging.getLogger(logger_name) + self.validate = validate if debug_logging: utils.configure_debug_log_handlers(self.logger) @@ -84,7 +89,10 @@ def _make_call(self, protocol, *args, **kwargs): CallFailure may be raised.""" - return protocol.perform(self.session, *args, **kwargs) + print protocol + print args, kwargs + + return protocol.perform(self.session, self.validate, *args, **kwargs) def is_authenticated(self): """Returns ``True`` if the Api can make an authenticated request.""" @@ -166,10 +174,10 @@ def perform_oauth(storage_filepath=OAUTH_FILEPATH, open_browser=False): return credentials - def __init__(self, debug_logging=True): + def __init__(self, debug_logging=True, validate=True): self.session = gmusicapi.session.Musicmanager() - super(Musicmanager, self).__init__(self.__class__.__name__, debug_logging) + super(Musicmanager, self).__init__(self.__class__.__name__, debug_logging, validate) self.logout() def login(self, oauth_credentials=OAUTH_FILEPATH, @@ -653,10 +661,10 @@ class Webclient(_Base): to upload). """ - def __init__(self, debug_logging=True): + def __init__(self, debug_logging=True, validate=True): self.session = gmusicapi.session.Webclient() - super(Webclient, self).__init__(self.__class__.__name__, debug_logging) + super(Webclient, self).__init__(self.__class__.__name__, debug_logging, validate) self.logout() def login(self, email, password): diff --git a/gmusicapi/protocol/shared.py b/gmusicapi/protocol/shared.py index 598c1cc8..054493ad 100644 --- a/gmusicapi/protocol/shared.py +++ b/gmusicapi/protocol/shared.py @@ -184,11 +184,12 @@ def filter_response(cls, msg): return msg # default to identity @classmethod - def perform(cls, session, *args, **kwargs): + def perform(cls, session, validate, *args, **kwargs): """Send, parse, validate and check success of this call. *args and **kwargs are passed to protocol.build_transaction. :param session: a PlaySession used to send this request. + :param validate: if False, do not validate """ #TODO link up these docs @@ -238,7 +239,8 @@ def perform(cls, session, *args, **kwargs): try: #order is important; validate only has a schema for a successful response cls.check_success(response, parsed_response) - cls.validate(response, parsed_response) + if validate: + cls.validate(response, parsed_response) except CallFailure: raise except ValidationException as e: diff --git a/gmusicapi/session.py b/gmusicapi/session.py index 4a4e33ad..9987d88c 100644 --- a/gmusicapi/session.py +++ b/gmusicapi/session.py @@ -83,7 +83,7 @@ def login(self, email, password, *args, **kwargs): super(Webclient, self).login() - res = ClientLogin.perform(self, email, password) + res = ClientLogin.perform(self, True, email, password) if 'SID' not in res or 'Auth' not in res: return False @@ -95,7 +95,7 @@ def login(self, email, password, *args, **kwargs): # Get webclient cookies. # They're stored automatically by requests on the webclient session. try: - webclient.Init.perform(self) + webclient.Init.perform(self, True) except CallFailure: # throw away clientlogin credentials self.logout() From 62b84e9fe2868296aa928144272ecc651577c1d7 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 17 Jun 2013 14:48:09 -0400 Subject: [PATCH 19/72] add validation toggle to changelog --- HISTORY.rst | 3 ++- gmusicapi/clients.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 928bcbec..5b311366 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,7 +10,8 @@ As of 1.0.0, `semantic versioning `__ is used. released 2013-XX-XX - add support for streaming All Access songs -- terminate early when a song dictionary is passed instead of an id +- add a toggle to turn off validation per client +- raise an exception when a song dictionary is passed instead of an id 1.2.0 +++++ diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index 068945bb..204e91d9 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -67,6 +67,9 @@ def __init__(self, logger_basename, debug_logging, validate): :param validate: if False, do not validate server responses against known schemas. This helps to catch protocol changes, but requires significant cpu work. + + This arg is stored as ``self.validate`` and can be safely + modified at runtime. """ # this isn't correct if init is called more than once, so we log the # client name below to avoid confusion for people reading logs @@ -89,9 +92,6 @@ def _make_call(self, protocol, *args, **kwargs): CallFailure may be raised.""" - print protocol - print args, kwargs - return protocol.perform(self.session, self.validate, *args, **kwargs) def is_authenticated(self): From 7d4faa2edcb6e887c2850f27835deb9d54678e3e Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 17 Jun 2013 15:19:43 -0400 Subject: [PATCH 20/72] defer default oauth folder creation (#115) --- gmusicapi/clients.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index 204e91d9..02a6e471 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -27,10 +27,6 @@ OAUTH_FILEPATH = os.path.join(utils.my_appdirs.user_data_dir, 'oauth.cred') -# oauth client breaks if the dir doesn't exist -utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700) - - class _Base(object): """Factors out common client setup.""" @@ -169,6 +165,8 @@ def perform_oauth(storage_filepath=OAUTH_FILEPATH, open_browser=False): credentials = flow.step2_exchange(code) if storage_filepath is not None: + if storage_filepath == OAUTH_FILEPATH: + utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700) storage = oauth2client.file.Storage(storage_filepath) storage.put(credentials) @@ -243,6 +241,8 @@ def _oauth_login(self, oauth_credentials): if isinstance(oauth_credentials, basestring): oauth_file = oauth_credentials + if oauth_file == OAUTH_FILEPATH: + utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700) storage = oauth2client.file.Storage(oauth_file) oauth_credentials = storage.get() From 04200d768bfc738b55ff7863313fe30e61a3cb37 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 17 Jun 2013 18:08:32 -0400 Subject: [PATCH 21/72] refactor test envarg parsing --- .travis.yml | 3 + gmusicapi/test/run_tests.py | 138 ++++++++++++++++-------------------- 2 files changed, 65 insertions(+), 76 deletions(-) diff --git a/.travis.yml b/.travis.yml index c151e709..9c13f880 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,5 +16,8 @@ branches: - develop - master env: + global: + - GM_UP_ID="E9:40:01:0E:51:7A" + - GM_UP_NAME="Travis-CI (gmusicapi)" matrix: - secure: "UAXrAYzrYRVFWSRldd5o9NrreexCsXdBna/kcKLlAQ8ygRahpqP7qXX8qiNU\nVhz0kdgBxgS84AEE3H/30o9v2IDWqmJCI5OqMfH1o5Pm+CelBt+8FHu3SMjm\nNvNcm/Vmip7WCSx7P2FfOf8HboSH/kVXuF0iPlOdozrTR1wPUq8=" diff --git a/gmusicapi/test/run_tests.py b/gmusicapi/test/run_tests.py index 17b2f3cc..2ed356b8 100644 --- a/gmusicapi/test/run_tests.py +++ b/gmusicapi/test/run_tests.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from collections import namedtuple from functools import partial, update_wrapper from getpass import getpass import logging @@ -12,113 +13,98 @@ from gmusicapi.clients import Webclient, Musicmanager, OAUTH_FILEPATH from gmusicapi.protocol.musicmanager import credentials_from_refresh_token -from gmusicapi.test import local_tests, server_tests +from gmusicapi.test import local_tests, server_tests # noqa from gmusicapi.test.utils import NoticeLogging -travis_id = 'E9:40:01:0E:51:7A' -travis_name = "Travis-CI (gmusicapi)" +EnvArg = namedtuple('EnvArg', 'envarg kwarg description') -# pretend to use test modules to appease flake8 -# these need to be imported for implicit test discovery -_, _ = local_tests, server_tests +wc_envargs = ( + EnvArg('GM_USER', 'email', 'WC user. If not present, user will be prompted.'), + EnvArg('GM_PASS', 'password', 'WC password. If not present, user will be prompted.'), +) +mm_envargs = ( + EnvArg('GM_OAUTH', 'oauth_credentials', 'MM refresh token. Defaults to MM.login default.'), + EnvArg('GM_UP_ID', 'uploader_id', 'MM uploader id. Defaults to MM.login default.'), + EnvArg('GM_UP_NAME', 'uploader_name', 'MM uploader name. Default to MM.login default.'), +) -def freeze_login_details(): - """Searches the environment for credentials, and freezes them to - client.login if found. - If no auth is present in the env, the user is prompted. OAuth is read from - the default path. +def prompt_for_wc_auth(): + """Return a valid (user, pass) tuple by continually + prompting the user.""" - If running on Travis, the prompt will never be fired; sys.exit is called - if the envvars are not present. - """ + print ("These tests will never delete or modify your music." + "\n\n" + "If the tests fail, you *might* end up with a test" + " song/playlist in your library, though." + "\n") - #Attempt to get auth from environ. - user, passwd, refresh_tok = [os.environ.get(name) for name in - ('GM_USER', - 'GM_PASS', - 'GM_OAUTH')] + wclient = Webclient() + valid_wc_auth = False - on_travis = os.environ.get('TRAVIS') + while not valid_wc_auth: + print + email = raw_input("Email: ") + passwd = getpass() - mm_kwargs = {} - wc_kwargs = {} + valid_wc_auth = wclient.login(email, passwd) - has_env_auth = user and passwd and refresh_tok + return email, passwd - if not has_env_auth and on_travis: - print 'on Travis but could not read auth from environ; quitting.' - sys.exit(1) - if os.environ.get('TRAVIS'): - #Travis runs on VMs with no "real" mac - we have to provide one. - mm_kwargs.update({'uploader_id': travis_id, - 'uploader_name': travis_name}) +def retrieve_auth(): + """Searches the env for auth, prompting the user if necessary. - if has_env_auth: - wc_kwargs.update({'email': user, 'password': passwd}) + On success, return (wc_kwargs, mm_kwargs). On failure, raise ValueError.""" - # mm expects a full OAuth2Credentials object - credentials = credentials_from_refresh_token(refresh_tok) - mm_kwargs.update({'oauth_credentials': credentials}) + get_kwargs = lambda envargs: {arg.kwarg: os.environ.get(arg.envarg) + for arg in envargs} - else: - # no travis, no credentials - print ("These tests will never delete or modify your music." - "\n\n" - "If the tests fail, you *might* end up with a test" - " song/playlist in your library, though." - "\n") - - # check for oauth - try: - with open(OAUTH_FILEPATH) as f: - pass # assume they're valid - - except IOError: - print ("You must have oauth credentials stored at the default" - " path by Musicmanager.perform_oauth prior to running.") - sys.exit(1) + wc_kwargs = get_kwargs(wc_envargs) + mm_kwargs = get_kwargs(mm_envargs) - wclient = Webclient() - valid_wc_auth = False + if not all([wc_kwargs[arg] for arg in ('email', 'password')]): + if os.environ.get('TRAVIS'): + print 'on Travis but could not read auth from environ; quitting.' + sys.exit(1) - while not valid_wc_auth: - print - email = raw_input("Email: ") - passwd = getpass() + wc_kwargs.update(zip(['email', 'password'], prompt_for_wc_auth())) - valid_wc_auth = wclient.login(email, passwd) + if mm_kwargs['oauth_credentials'] is None: + # ignoring race + if not os.path.isfile(OAUTH_FILEPATH): + raise ValueError("You must have oauth credentials stored at the default" + " path by Musicmanager.perform_oauth prior to running.") + del mm_kwargs['oauth_credentials'] # mm default is not None + else: + mm_kwargs['oauth_credentials'] = \ + credentials_from_refresh_token(mm_kwargs['oauth_credentials']) - wc_kwargs.update({'email': email, 'password': passwd}) + return (wc_kwargs, mm_kwargs) - # globally freeze our params in place. - # they can still be overridden manually; they're just the defaults now. - Musicmanager.login = MethodType( - update_wrapper(partial(Musicmanager.login, **mm_kwargs), Musicmanager.login), - None, Musicmanager - ) - Webclient.login = MethodType( - update_wrapper(partial(Webclient.login, **wc_kwargs), Webclient.login), - None, Webclient - ) +def freeze_login_details(wc_kwargs, mm_kwargs): + """Set the given kwargs to be the default for client login methods.""" + for cls, kwargs in ((Musicmanager, mm_kwargs), + (Webclient, wc_kwargs)): + cls.login = MethodType( + update_wrapper(partial(cls.login, **kwargs), cls.login), + None, cls) def main(): - if '--group=local' not in sys.argv: - freeze_login_details() + """Search env for auth envargs and run tests.""" - root_logger = logging.getLogger('gmusicapi') - # using DynamicClientLoggers eliminates the need for root handlers - # configure_debug_log_handlers(root_logger) + if '--group=local' not in sys.argv: + # hack: assume we're just running the proboscis local group + freeze_login_details(*retrieve_auth()) # warnings typically signal a change in protocol, # so fail the build if anything >= warning are sent, - noticer = NoticeLogging() noticer.setLevel(logging.WARNING) + root_logger = logging.getLogger('gmusicapi') root_logger.addHandler(noticer) # proboscis does not have an exit=False equivalent, From 5ff576f6d64644786aa4a41eed1eb9a3f4908a96 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 17 Jun 2013 18:18:22 -0400 Subject: [PATCH 22/72] remove dict comp for 2.6 --- gmusicapi/test/run_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gmusicapi/test/run_tests.py b/gmusicapi/test/run_tests.py index 2ed356b8..511e9a26 100644 --- a/gmusicapi/test/run_tests.py +++ b/gmusicapi/test/run_tests.py @@ -58,8 +58,8 @@ def retrieve_auth(): On success, return (wc_kwargs, mm_kwargs). On failure, raise ValueError.""" - get_kwargs = lambda envargs: {arg.kwarg: os.environ.get(arg.envarg) - for arg in envargs} + get_kwargs = lambda envargs: dict([(arg.kwarg, os.environ.get(arg.envarg)) + for arg in envargs]) wc_kwargs = get_kwargs(wc_envargs) mm_kwargs = get_kwargs(mm_envargs) From 7ef2f9521b3e5ab5c475f3eb286e42f971108d79 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 18 Jun 2013 13:11:13 -0400 Subject: [PATCH 23/72] move appdirs to compat --- gmusicapi/clients.py | 6 ++++-- gmusicapi/compat.py | 27 +++++++++++++++++++++++---- gmusicapi/utils/utils.py | 10 +++++----- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index 02a6e471..3996338c 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -19,13 +19,15 @@ import oauth2client.file import gmusicapi +from gmusicapi.compat import my_appdirs from gmusicapi.gmtools import tools from gmusicapi.exceptions import CallFailure, NotLoggedIn from gmusicapi.protocol import webclient, musicmanager, upload_pb2, locker_pb2 from gmusicapi.utils import utils import gmusicapi.session -OAUTH_FILEPATH = os.path.join(utils.my_appdirs.user_data_dir, 'oauth.cred') +OAUTH_FILEPATH = os.path.join(my_appdirs.user_data_dir, 'oauth.cred') + class _Base(object): """Factors out common client setup.""" @@ -500,7 +502,7 @@ def upload(self, filepaths, transcode_quality=3, enable_matching=False): #To keep behavior consistent, make no effort to guess - require users # to decode first. user_err_msg = ("nonascii bytestrings must be decoded to unicode" - " (error: '%s')" % err_msg) + " (error: '%s')" % user_err_msg) not_uploaded[path] = user_err_msg else: diff --git a/gmusicapi/compat.py b/gmusicapi/compat.py index 3c30c56b..d5445825 100644 --- a/gmusicapi/compat.py +++ b/gmusicapi/compat.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -Single interface for code that varies across Python versions +Single interface for code that varies across Python environments. """ import sys @@ -14,6 +14,25 @@ import unittest2 as unittest import simplejson as json else: # 2.7 - from collections import Counter - import unittest - import json + from collections import Counter # noqa + import unittest # noqa + import json # noqa + +try: + from appdirs import AppDirs + my_appdirs = AppDirs('gmusicapi', 'Simon Weber') +except ImportError: + print 'warning: could not import appdirs; will use current directory' + + class FakeAppDirs(object): + to_spoof = set([base + '_dir' for base in + ('user_data', 'site_data', 'user_config', + 'site_config', 'user_cache', 'user_log')]) + + def __getattr__(self, name): + if name in self.to_spoof: + return '' # current dir + else: + raise AttributeError + + my_appdirs = FakeAppDirs() diff --git a/gmusicapi/utils/utils.py b/gmusicapi/utils/utils.py index e048e98e..f7208542 100644 --- a/gmusicapi/utils/utils.py +++ b/gmusicapi/utils/utils.py @@ -12,11 +12,11 @@ import time import traceback -from appdirs import AppDirs from decorator import decorator from google.protobuf.descriptor import FieldDescriptor from gmusicapi import __version__ +from gmusicapi.compat import my_appdirs from gmusicapi.exceptions import CallFailure # this controls the crazy logging setup that checks the callstack; @@ -24,8 +24,6 @@ # when False, static code will simply log in the standard way under the root. per_client_logging = True -my_appdirs = AppDirs('gmusicapi', 'Simon Weber') - #Map descriptor.CPPTYPE -> python type. _python_to_cpp_types = { long: ('int32', 'int64', 'uint32', 'uint64'), @@ -101,6 +99,7 @@ def __getattr__(self, name): log = DynamicClientLogger(__name__) + def is_valid_mac(mac_string): """Return True if mac_string is of form eg '00:11:22:33:AA:BB'. @@ -205,6 +204,7 @@ def wrapper(function, *args, **kw): return wrapper + @dual_decorator def enforce_ids_param(position=1): """Verifies that the caller is passing a list of song ids, and not a @@ -216,8 +216,8 @@ def enforce_ids_param(position=1): @decorator def wrapper(function, *args, **kw): - if (not isinstance(args[position], (list, tuple)) or - not all([isinstance(e, basestring) for e in args[position]])): + if ((not isinstance(args[position], (list, tuple)) or + not all([isinstance(e, basestring) for e in args[position]]))): raise ValueError("Invalid param type in position %s;" " expected song ids (did you pass song dictionaries?)" % position) From 36db448a5ff47997c98515e826df09c38ca7ea7d Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 18 Jun 2013 13:17:14 -0400 Subject: [PATCH 24/72] fix indent typo in compat --- gmusicapi/compat.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gmusicapi/compat.py b/gmusicapi/compat.py index d5445825..4d1b114c 100644 --- a/gmusicapi/compat.py +++ b/gmusicapi/compat.py @@ -29,10 +29,10 @@ class FakeAppDirs(object): ('user_data', 'site_data', 'user_config', 'site_config', 'user_cache', 'user_log')]) - def __getattr__(self, name): - if name in self.to_spoof: - return '' # current dir - else: - raise AttributeError + def __getattr__(self, name): + if name in self.to_spoof: + return '.' # current dir + else: + raise AttributeError my_appdirs = FakeAppDirs() From 7ce93593f643a9323ad977a5410b3ca8498ea3ed Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 18 Jun 2013 13:33:19 -0400 Subject: [PATCH 25/72] get test files without chdir --- gmusicapi/test/utils.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/gmusicapi/test/utils.py b/gmusicapi/test/utils.py index 582fdb97..65283946 100644 --- a/gmusicapi/test/utils.py +++ b/gmusicapi/test/utils.py @@ -2,7 +2,6 @@ """Utilities used in testing.""" -from glob import glob import logging import numbers import os @@ -27,14 +26,9 @@ # directory as this file. cwd = os.getcwd() test_file_dir = os.path.dirname(os.path.abspath(__file__)) -os.chdir(test_file_dir) -_audio_filenames = glob(u'audiotest*') -mp3_filenames = [os.path.abspath(fn) for fn in _audio_filenames if fn.endswith('.mp3')] -small_mp3 = os.path.abspath(u'audiotest_small.mp3') -image_filename = os.path.abspath(u'imagetest_10x10_check.png') - -os.chdir(cwd) +small_mp3 = os.path.join(test_file_dir, u'audiotest_small.mp3') +image_filename = os.path.join(test_file_dir, u'imagetest_10x10_check.png') class NoticeLogging(logging.Handler): @@ -124,4 +118,4 @@ def is_id_list(lst): def is_id_pair_list(lst): """Returns True if the given list is made up of all (id, id) pairs.""" a, b = zip(*lst) - return is_id_list(a+b) + return is_id_list(a + b) From a9370aed9e9c10c4ee08c731c4fa28b089816313 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 18 Jun 2013 14:42:34 -0400 Subject: [PATCH 26/72] tests run without filesystem write access --- gmusicapi/test/run_tests.py | 2 +- gmusicapi/utils/utils.py | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/gmusicapi/test/run_tests.py b/gmusicapi/test/run_tests.py index 511e9a26..3342591c 100644 --- a/gmusicapi/test/run_tests.py +++ b/gmusicapi/test/run_tests.py @@ -111,7 +111,7 @@ def main(): # so SystemExit must be caught instead (we need # to check the log noticer) try: - TestProgram().run_and_exit() + TestProgram(module=sys.modules[__name__]).run_and_exit() except SystemExit as e: print if noticer.seen_message: diff --git a/gmusicapi/utils/utils.py b/gmusicapi/utils/utils.py index f7208542..0cf79b8a 100644 --- a/gmusicapi/utils/utils.py +++ b/gmusicapi/utils/utils.py @@ -227,35 +227,43 @@ def wrapper(function, *args, **kw): def configure_debug_log_handlers(logger): - """Warnings and above to stderr, below to gmusicapi.log. + """Warnings and above to stderr, below to gmusicapi.log when possible. Output includes line number.""" global printed_log_start_message logger.setLevel(logging.DEBUG) - make_sure_path_exists(os.path.dirname(log_filepath), 0o700) - fh = logging.FileHandler(log_filepath) - fh.setLevel(logging.DEBUG) + logging_to_file = True + try: + make_sure_path_exists(os.path.dirname(log_filepath), 0o700) + debug_handler = logging.FileHandler(log_filepath) + except OSError: + logging_to_file = False + debug_handler = logging.StreamHandler() + + debug_handler.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.WARNING) + important_handler = logging.StreamHandler() + important_handler.setLevel(logging.WARNING) - logger.addHandler(fh) - logger.addHandler(ch) + logger.addHandler(debug_handler) + logger.addHandler(important_handler) if not printed_log_start_message: #print out startup message without verbose formatting logger.info("!-- begin debug log --!") logger.info("version: " + __version__) - logger.info("logging to: " + log_filepath) + if logging_to_file: + logger.info("logging to: " + log_filepath) + printed_log_start_message = True formatter = logging.Formatter( '%(asctime)s - %(name)s (%(module)s:%(lineno)s) [%(levelname)s]: %(message)s' ) - fh.setFormatter(formatter) - ch.setFormatter(formatter) + debug_handler.setFormatter(formatter) + important_handler.setFormatter(formatter) @dual_decorator From 931d3a66e19be037b5da861137efa307bf8af22a Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 18 Jun 2013 14:58:00 -0400 Subject: [PATCH 27/72] all access tests off by default, enabled with an envarg --- .travis.yml | 1 + gmusicapi/test/server_tests.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c13f880..554dc327 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,5 +19,6 @@ env: global: - GM_UP_ID="E9:40:01:0E:51:7A" - GM_UP_NAME="Travis-CI (gmusicapi)" + - GM_TEST_ALLACCESS=TRUE matrix: - secure: "UAXrAYzrYRVFWSRldd5o9NrreexCsXdBna/kcKLlAQ8ygRahpqP7qXX8qiNU\nVhz0kdgBxgS84AEE3H/30o9v2IDWqmJCI5OqMfH1o5Pm+CelBt+8FHu3SMjm\nNvNcm/Vmip7WCSx7P2FfOf8HboSH/kVXuF0iPlOdozrTR1wPUq8=" diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 61ff8585..9149697b 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -9,6 +9,7 @@ from copy import copy from collections import namedtuple +import os import re import types @@ -311,12 +312,14 @@ def get_normal_stream_urls(self): assert_is_not_none(url) assert_equal(url[:7], 'http://') - @song_test - def get_aa_stream_urls(self): - # that dumb little intro track on Conspiracy of One - urls = self.wc.get_stream_urls('Tqqufr34tuqojlvkolsrwdwx7pe') + # TODO there must be a better way + if os.environ.get('GM_TEST_ALLACCESS') == 'TRUE': + @song_test + def get_aa_stream_urls(self): + # that dumb little intro track on Conspiracy of One + urls = self.wc.get_stream_urls('Tqqufr34tuqojlvkolsrwdwx7pe') - assert_true(len(urls) > 1) + assert_true(len(urls) > 1) @song_test def upload_album_art(self): From a9e753ea0a4c17e0a2bb1032b4178a59292c7310 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 24 Jun 2013 21:28:38 -0400 Subject: [PATCH 28/72] completely disable search tests --- gmusicapi/test/server_tests.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 9149697b..99a1b742 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -346,23 +346,23 @@ def _assert_search_hit(res, hit_type, hit_key, val): #hitmap = (hit[hit_key] == val for hit in res[hit_type]) #assert_equal(sum(hitmap), 1) # eg sum(True, False, True) == 2 - @song_test - def search_title(self): - res = self.wc.search(self.song.title) + #@song_test + #def search_title(self): + # res = self.wc.search(self.song.title) - self._assert_search_hit(res, 'song_hits', 'id', self.song.sid) + # self._assert_search_hit(res, 'song_hits', 'id', self.song.sid) - @song_test - def search_artist(self): - res = self.wc.search(self.song.artist) + #@song_test + #def search_artist(self): + # res = self.wc.search(self.song.artist) - self._assert_search_hit(res, 'artist_hits', 'id', self.song.sid) + # self._assert_search_hit(res, 'artist_hits', 'id', self.song.sid) - @song_test - def search_album(self): - res = self.wc.search(self.song.album) + #@song_test + #def search_album(self): + # res = self.wc.search(self.song.album) - self._assert_search_hit(res, 'album_hits', 'albumName', self.song.album) + # self._assert_search_hit(res, 'album_hits', 'albumName', self.song.album) #--------------- # Playlist tests From 47a42f14bd3f4c2defb4969e546cc054ba75af8a Mon Sep 17 00:00:00 2001 From: devsda Date: Wed, 26 Jun 2013 12:07:24 -0500 Subject: [PATCH 29/72] [issue 137] fix for streaming AA tracks --- gmusicapi/protocol/webclient.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 36921060..94cf54e9 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -2,8 +2,13 @@ """Calls made by the web client.""" +import binascii import copy +import hmac +import random +import string import sys +from hashlib import sha1 import validictory @@ -510,9 +515,19 @@ class GetStreamUrl(WcCall): @staticmethod def dynamic_params(song_id): + + # https://github.com/simon-weber/Unofficial-Google-Music-API/issues/137 + # And technically, slt/sig aren't required for tracks you upload, + # but without the track's type field, we can't tell the difference. + key = '27f7313e-f75d-445a-ac99-56386a5fe879' + salt = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(12)) + sig = binascii.b2a_base64(hmac.new(key, (song_id + salt), sha1).digest())[:-1].replace('+', '-').replace('/', '_').replace('=', '.') + params = { 'u': 0, - 'pt': 'e' + 'pt': 'e', + 'slt': salt, + 'sig': sig } # all access streams use a different param From c1d38dd9db066a18629dbb65fdab0a158ba57525 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 26 Jun 2013 13:49:59 -0400 Subject: [PATCH 30/72] integrate #138 [close #137] --- gmusicapi/protocol/webclient.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 94cf54e9..03e49185 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -517,11 +517,19 @@ class GetStreamUrl(WcCall): def dynamic_params(song_id): # https://github.com/simon-weber/Unofficial-Google-Music-API/issues/137 - # And technically, slt/sig aren't required for tracks you upload, - # but without the track's type field, we can't tell the difference. + # there are three cases when streaming: + # | track type | guid songid? | slt/sig needed? | + # user-uploaded yes no + # AA track in library yes yes + # AA track not in library no yes + + # without the track['type'] field we can't tell between 1 and 2, but + # include slt/sig anyway; the server ignores the extra params. key = '27f7313e-f75d-445a-ac99-56386a5fe879' salt = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(12)) - sig = binascii.b2a_base64(hmac.new(key, (song_id + salt), sha1).digest())[:-1].replace('+', '-').replace('/', '_').replace('=', '.') + sig = binascii.b2a_base64(hmac.new(key, (song_id + salt), sha1).digest())[:-1] + urlsafe_b64_trans = string.maketrans("+/=", "-_.") + sig = sig.translate(urlsafe_b64_trans) params = { 'u': 0, @@ -530,10 +538,7 @@ def dynamic_params(song_id): 'sig': sig } - # all access streams use a different param - # https://github.com/simon-weber/Unofficial-Google-Music-API/pull/131#issuecomment-18843993 - # thanks @lukegb! - + # TODO match guid instead, should be more robust if song_id[0] == 'T': # all access params['mjck'] = song_id From d63c151f43c49a60098cdf4f7d762231f0942bc2 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 2 Jul 2013 13:48:19 -0400 Subject: [PATCH 31/72] add webclient.get_registered_devices --- gmusicapi/clients.py | 25 +++++++++++++ gmusicapi/protocol/webclient.py | 65 +++++++++++++++++++++++++++++++++ gmusicapi/test/server_tests.py | 5 +++ 3 files changed, 95 insertions(+) diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index 3996338c..db9fe553 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -692,6 +692,31 @@ def login(self, email, password): def logout(self): return super(Webclient, self).logout() + def get_registered_devices(self): + """Returns a list of dictionaries, eg:: + [ + { + u'date': 1367470393588, # utc-millisecond + u'id': u'AA:BB:CC:11:22:33', + u'name': u'my-hostname', + u'type': u'DESKTOP_APP' + }, + { + u'carrier': u'Google', + u'date': 1344808742774 + u'id': u'0x00112233aabbccdd', + u'manufacturer': u'Asus', + u'model': u'Nexus 7', + u'name': u'', + u'type': u'PHONE', + } + ] + """ + + #TODO sessionid stuff + res = self._make_call(webclient.GetSettings, '') + return res['settings']['devices'] + def change_playlist_name(self, playlist_id, new_name): """Changes the name of a playlist. Returns the changed id. diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 03e49185..3da69d10 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -632,3 +632,68 @@ def dynamic_files(image_filepath): contents = f.read() return {'albumArt': (image_filepath, contents)} + + +class GetSettings(WcCall): + """Get data that populates the settings tab: labs and devices.""" + + static_method = 'POST' + static_url = service_url + 'loadsettings' + + _device_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'date': {'type': 'integer', + 'format': 'utc-millisec'}, + 'id': {'type': 'string'}, + 'name': {'type': 'string'}, + 'type': {'type': 'string'}, + # only for type == PHONE: + 'model': {'type': 'string', 'required': False}, + 'manufacturer': {'type': 'string', 'required': False}, + 'name': {'type': 'string', 'required': False}, + 'carrier': {'type': 'string', 'required': False}, + }, + } + + _lab_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'description': {'type': 'string'}, + 'enabled': {'type': 'boolean'}, + 'name': {'type': 'string'}, + 'title': {'type': 'string'}, + }, + } + + _res_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'settings': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'devices': {'type': 'array', 'items': _device_schema}, + 'labs': {'type': 'array', 'items': _lab_schema}, + 'maxTracks': {'type': 'integer'}, + 'expirationMillis': { + 'type': 'integer', + 'format': 'utc-millisec', + 'required': False, + }, + 'isSubscription': {'type': 'boolean', 'required': False}, + 'isTrial': {'type': 'boolean', 'required': False}, + }, + }, + }, + } + + @staticmethod + def dynamic_data(session_id): + """ + :param: session_id + """ + return {'json': json.dumps({'sessionId': session_id})} diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 99a1b742..686220e5 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -163,6 +163,11 @@ def song_delete(self): # Non-wonky tests resume down here. + @test + def get_registered_devices(self): + # no logic; schema does verification + self.wc.get_registered_devices() + #----------- # Song tests #----------- From 5798865eb4cd1261065ccdfbeb9e66425c80fca6 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 2 Jul 2013 13:57:26 -0400 Subject: [PATCH 32/72] add get_registered_devices to docs --- HISTORY.rst | 1 + docs/source/reference/webclient.rst | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 5b311366..145c6bbe 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ As of 1.0.0, `semantic versioning `__ is used. released 2013-XX-XX - add support for streaming All Access songs +- add Webclient.get_registered_devices - add a toggle to turn off validation per client - raise an exception when a song dictionary is passed instead of an id diff --git a/docs/source/reference/webclient.rst b/docs/source/reference/webclient.rst index dab1ea01..f18edd13 100644 --- a/docs/source/reference/webclient.rst +++ b/docs/source/reference/webclient.rst @@ -44,3 +44,7 @@ Playlist content manipulation .. automethod:: Webclient.change_playlist .. automethod:: Webclient.add_songs_to_playlist .. automethod:: Webclient.remove_songs_from_playlist + +Other +----- +.. automethod:: Webclient.get_registered_devices From c354ab8375ec024910d178f6331eede9c34e036d Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 2 Jul 2013 14:05:57 -0400 Subject: [PATCH 33/72] add deauth protocol --- gmusicapi/protocol/webclient.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 3da69d10..2ffe0ec0 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -697,3 +697,18 @@ def dynamic_data(session_id): :param: session_id """ return {'json': json.dumps({'sessionId': session_id})} + + +class DeauthDevice(WcCall): + """Deauthorize a device from GetSettings.""" + static_method = 'POST' + static_url = service_url + 'modifysettings' + + @staticmethod + def dynamic_data(device_id, session_id): + return {'json': json.dumps({'deauth': device_id, 'sessionId': session_id})} + + @classmethod + def validate(cls, response, msg): + if msg.text != '{}': + raise ValidationException("expected an empty object; received %r" % msg.text) From 5ef46941080bf15d2f1c8bc0f23a29c5125b1e3a Mon Sep 17 00:00:00 2001 From: Jaime Cura Date: Tue, 2 Jul 2013 23:42:06 +0200 Subject: [PATCH 34/72] Create Mobile client and initial calls --- gmusicapi/__init__.py | 4 +- gmusicapi/clients.py | 61 +++++++++++- gmusicapi/protocol/mobileclient.py | 150 +++++++++++++++++++++++++++++ gmusicapi/session.py | 48 +++++++++ 4 files changed, 259 insertions(+), 4 deletions(-) create mode 100644 gmusicapi/protocol/mobileclient.py diff --git a/gmusicapi/__init__.py b/gmusicapi/__init__.py index 1b6d6881..09bf967f 100644 --- a/gmusicapi/__init__.py +++ b/gmusicapi/__init__.py @@ -7,11 +7,11 @@ __license__ = 'BSD 3-Clause' __title__ = 'gmusicapi' -from gmusicapi.clients import Webclient, Musicmanager +from gmusicapi.clients import Webclient, Musicmanager, Mobileclient from gmusicapi.exceptions import CallFailure # appease flake8: the imports are purposeful -(__version__, Webclient, Musicmanager, CallFailure) +(__version__, Webclient, Musicmanager, Mobileclient, CallFailure) class Api(object): diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index c8865e91..9c73173a 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -21,7 +21,7 @@ import gmusicapi from gmusicapi.gmtools import tools from gmusicapi.exceptions import CallFailure, NotLoggedIn -from gmusicapi.protocol import webclient, musicmanager, upload_pb2, locker_pb2 +from gmusicapi.protocol import webclient, musicmanager, mobileclient, upload_pb2, locker_pb2 from gmusicapi.utils import utils import gmusicapi.session @@ -491,7 +491,7 @@ def upload(self, filepaths, transcode_quality=3, enable_matching=False): #To keep behavior consistent, make no effort to guess - require users # to decode first. user_err_msg = ("nonascii bytestrings must be decoded to unicode" - " (error: '%s')" % err_msg) + " (error: '%s')" % user_err_msg) not_uploaded[path] = user_err_msg else: @@ -643,6 +643,63 @@ def upload(self, filepaths, transcode_quality=3, enable_matching=False): return uploaded, matched, not_uploaded +class Mobileclient(_Base): + """Allows library management and streaming by posing as the + googleapis.com mobile clients. + + Uploading is not supported by this client (use the :class:`Musicmanager` + to upload). + """ + def __init__(self, debug_logging=True): + self.session = gmusicapi.session.Mobileclient() + + super(Mobileclient, self).__init__(self.__class__.__name__, debug_logging) + self.logout() + + + def login(self, email, password): + """Authenticates the webclient. + Returns ``True`` on success, ``False`` on failure. + + :param email: eg ``'test@gmail.com'`` or just ``'test'``. + :param password: password or app-specific password for 2-factor users. + This is not stored locally, and is sent securely over SSL. + + Users of two-factor authentication will need to set an application-specific password + to log in. + """ + + if not self.session.login(email, password): + self.logger.info("failed to authenticate") + return False + + self.logger.info("authenticated") + + return True + + + def search(self, query, max_results=5): + """Queries the server for songs and albums. + + :param query: a string keyword to search with. Capitalization and punctuation are ignored. + :param max_results: Maximum number of items to be retrieved + + The results are returned in a dictionary, arranged by how they were found. + ``artist_hits`` and ``song_hits`` return a list of + :ref:`song dictionaries `, while ``album_hits`` entries + have a different structure. + """ + + res = self._make_call(mobileclient.Search, query, max_results)['entries'] + + return {"album_hits": [hit for hit in res if hit['type']=="3"], + "artist_hits": [hit for hit in res if hit['type']=="2"], + "song_hits": [hit for hit in res if hit['type']=="1"]} + + def get_artist(self, artistid, albums=True, top_tracks=0, rel_artist=0): + """Retrieve artist data""" + res = self._make_call(mobileclient.GetArtist, artistid, albums, top_tracks, rel_artist) + return res class Webclient(_Base): """Allows library management and streaming by posing as the diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py new file mode 100644 index 00000000..68fe51f9 --- /dev/null +++ b/gmusicapi/protocol/mobileclient.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""Calls made by the web client.""" + +import sys + +import validictory + +from gmusicapi.exceptions import CallFailure, ValidationException +from gmusicapi.protocol.shared import Call, authtypes +from gmusicapi.utils import utils + +# URL for sj service +sj_url = 'https://www.googleapis.com/sj/v1/' + +# Data schema for sj service. Might be incomplete +sj_track = { + 'type':'object', + 'properties':{ + 'kind':{'type':'string'}, + 'title':{'type':'string'}, + 'artist':{'type':'string'}, + 'album':{'type':'string'}, + 'albumArtist':{'type':'string'}, + 'trackNumber':{'type':'integer'}, + 'durationMillis':{'type':'string'}, + 'albumArtRef':{'type':'array', 'items':{'type':'object', 'properties':{'url':{'type':'string'}}}}, + 'discNumber':{'type':'integer'}, + 'estimatedSize':{'type':'string'}, + 'trackType':{'type':'string'}, + 'storeId':{'type':'string'}, + 'albumId':{'type':'string'}, + 'artistId':{'type':'array', 'items':{'type':'string'}}, + 'nid':{'type':'string'}, + 'trackAvailableForPurchase':{'type':'boolean'}, + 'albumAvailableForPurchase':{'type':'boolean'}, + } + } + +sj_album = { + 'type':'object', + 'properties':{ + 'kind':{'type':'string'}, + 'name':{'type':'string'}, + 'albumArtist':{'type':'string'}, + 'albumArtRef':{'type':'string'}, + 'albumId':{'type':'string'}, + 'artist':{'type':'string'}, + 'artistId':{'type':'array', 'items':{'type':'string'}}, + 'year': {'type': 'integer'} + } + } + +sj_artist = { + 'type':'object', + 'properties':{ + 'kind':{'type':'string'}, + 'name':{'type':'string'}, + 'artistArtRef':{'type':'string'}, + 'artistId':{'type':'string'}, + 'albums:':{'type':'array', 'items':sj_album, 'required':False}, + 'topTracks':{'type':'array', 'items':sj_track, 'required':False}, + } + } + +sj_artist['related_artists']= {'type':'array', 'items':sj_artist, 'required':False} +# Result definition may not contain any item. +sj_result = { + "type":"object", + "properties":{ + 'score':{"type":"number"}, + 'artists':sj_artist, + 'album': sj_album, + 'track': sj_track + } + } + +sj_result['properties']['artists']['required']=False +sj_result['properties']['album']['required']=False +sj_result['properties']['track']['required']=False + +class MCall(Call): + """Abstract base for mobile client calls.""" + + required_auth = authtypes(xt=False, sso=True) + + #validictory schema for the response + _res_schema = utils.NotImplementedField + + @classmethod + def validate(cls, response, msg): + """Use validictory and a static schema (stored in cls._res_schema).""" + try: + return validictory.validate(msg, cls._res_schema) + except ValueError as e: + trace = sys.exc_info()[2] + raise ValidationException(str(e)), None, trace + + @classmethod + def check_success(cls, response, msg): + #Failed responses always have a success=False key. + #Some successful responses do not have a success=True key, however. + #TODO remove utils.call_succeeded + + if 'success' in msg and not msg['success']: + raise CallFailure( + "the server reported failure. This is usually" + " caused by bad arguments, but can also happen if requests" + " are made too quickly (eg creating a playlist then" + " modifying it before the server has created it)", + cls.__name__) + + @classmethod + def parse_response(cls, response): + return cls._parse_json(response.text) + + +class Search(MCall): + """Search for All Access tracks.""" + + static_method = 'GET' + + _res_schema = { + "type": "object", + "properties": { + "kind":{"type":"string"}, + "entries": {'type':'array', 'items': sj_result} + }, + "additionalProperties": False + } + + @staticmethod + def dynamic_url(query, max_ret): + return sj_url + 'query?q=%s&max-results=%d' % (query, max_ret) + +class GetArtist(MCall): + static_method = 'GET' + _res_schema = sj_artist + + @staticmethod + def dynamic_url(artistid, albums=True, top_tracks=0, rel_artist=0): + ret = sj_url + 'fetchartist?alt=json' + ret += '&nid=%s' % artistid + ret += '&include-albums=%r' % albums + ret += '&num-top-tracks=%d' % top_tracks + ret += '&num-related-artists=%d' % rel_artist + return ret + + diff --git a/gmusicapi/session.py b/gmusicapi/session.py index 2b88f5bf..e6b515b0 100644 --- a/gmusicapi/session.py +++ b/gmusicapi/session.py @@ -118,6 +118,54 @@ def _send_with_auth(self, req_kwargs, desired_auth, rsession): return rsession.request(**req_kwargs) +class Mobileclient(_Base): + """ This class is almost the same as Webclient. + Defined for future change purposes""" + + def __init__(self): + super(Mobileclient, self).__init__() + self._authtoken = None + + def login(self, email, password, *args, **kwargs): + """ + Perform clientlogin then retrieve webclient cookies. + + :param email: + :param password: + """ + + super(Mobileclient, self).login() + + res = ClientLogin.perform(self, email, password) + + if 'SID' not in res or 'Auth' not in res: + return False + + self._authtoken = res['Auth'] + + self.is_authenticated = True + + # Get mobileclient cookies. This is the same as webclient + try: + webclient.Init.perform(self) + except CallFailure: + # throw away clientlogin credentials + self.logout() + + return self.is_authenticated + + def _send_with_auth(self, req_kwargs, desired_auth, rsession): + req_kwargs['headers'] = req_kwargs.get('headers', {}) + + # Only supported authentication/authorization procedure is + # ClientLogin, but this will change by After April 20, 2015. + # A change must be made before that date. + + req_kwargs['headers']['Authorization'] = \ + 'GoogleLogin auth=' + self._authtoken + + return rsession.request(**req_kwargs) + class Musicmanager(_Base): def __init__(self): super(Musicmanager, self).__init__() From d1beff6f792b1c15e2e0f360f72e0625c7126ea4 Mon Sep 17 00:00:00 2001 From: Jaime Cura Date: Wed, 3 Jul 2013 01:36:20 +0200 Subject: [PATCH 35/72] Added a couple of calls more to mobileclient --- gmusicapi/clients.py | 103 ++++++++++++++++++++++++++++- gmusicapi/protocol/mobileclient.py | 78 +++++++++++++++++++++- 2 files changed, 178 insertions(+), 3 deletions(-) diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index 9c73173a..2c6667e1 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -11,6 +11,7 @@ from socket import gethostname import time import urllib +from urlparse import urlparse, parse_qsl from uuid import getnode as getmac import webbrowser @@ -701,6 +702,71 @@ def get_artist(self, artistid, albums=True, top_tracks=0, rel_artist=0): res = self._make_call(mobileclient.GetArtist, artistid, albums, top_tracks, rel_artist) return res + def get_album(self, albumid, tracks=True): + """Retrieve artist data""" + res = self._make_call(mobileclient.GetAlbum, albumid, tracks) + return res + + def get_track(self, trackid): + """Retrieve artist data""" + res = self._make_call(mobileclient.GetTrack, trackid) + return res + + def get_stream_audio(self, song_id): + """Returns a bytestring containing mp3 audio for this song. + + :param song_id: a single song id + """ + + urls = self.get_stream_urls(song_id) + + if len(urls) == 1: + return self.session._rsession.get(urls[0]).content + + # AA tracks are separated into multiple files + # the url contains the range of each file to be used + + range_pairs = [[int(s) for s in val.split('-')] + for url in urls + for key, val in parse_qsl(urlparse(url)[4]) + if key == 'range'] + + stream_pieces = [] + prev_end = 0 + + for url, (start, end) in zip(urls, range_pairs): + audio = self.session._rsession.get(url).content + stream_pieces.append(audio[prev_end - start:]) + + prev_end = end + 1 + + return ''.join(stream_pieces) + + def get_stream_urls(self, song_id): + """Returns a url that points to a streamable version of this song. + + :param song_id: a single song id. + + While acquiring the url requires authentication, retreiving the + url contents does not. + + However, there are limitation as to how the stream url can be used: + * the url expires after about a minute + * only one IP can be streaming music at once. + Other attempts will get an http 403 with + ``X-Rejected-Reason: ANOTHER_STREAM_BEING_PLAYED``. + + *This is only intended for streaming*. The streamed audio does not contain metadata. + Use :func:`get_song_download_info` to download complete files with metadata. + """ + res = self._make_call(mobileclient.GetStreamUrl, song_id) + + try: + return res['url'] + except KeyError: + return res['urls'] + + class Webclient(_Base): """Allows library management and streaming by posing as the music.google.com webclient. @@ -942,7 +1008,7 @@ def get_song_download_info(self, song_id): return (url, info["downloadCounts"][song_id]) - def get_stream_url(self, song_id): + def get_stream_urls(self, song_id): """Returns a url that points to a streamable version of this song. :param song_id: a single song id. @@ -962,7 +1028,40 @@ def get_stream_url(self, song_id): res = self._make_call(webclient.GetStreamUrl, song_id) - return res['url'] + try: + return res['url'] + except KeyError: + return res['urls'] + + def get_stream_audio(self, song_id): + """Returns a bytestring containing mp3 audio for this song. + + :param song_id: a single song id + """ + + urls = self.get_stream_urls(song_id) + + if len(urls) == 1: + return self.session._rsession.get(urls[0]).content + + # AA tracks are separated into multiple files + # the url contains the range of each file to be used + + range_pairs = [[int(s) for s in val.split('-')] + for url in urls + for key, val in parse_qsl(urlparse(url)[4]) + if key == 'range'] + + stream_pieces = [] + prev_end = 0 + + for url, (start, end) in zip(urls, range_pairs): + audio = self.session._rsession.get(url).content + stream_pieces.append(audio[prev_end - start:]) + + prev_end = end + 1 + + return ''.join(stream_pieces) def copy_playlist(self, playlist_id, copy_name): """Copies the contents of a playlist to a new playlist. Returns the id of the new playlist. diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 68fe51f9..aa04cc8c 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -3,7 +3,12 @@ """Calls made by the web client.""" +import binascii +import hmac +import random +import string import sys +from hashlib import sha1 import validictory @@ -48,7 +53,8 @@ 'albumId':{'type':'string'}, 'artist':{'type':'string'}, 'artistId':{'type':'array', 'items':{'type':'string'}}, - 'year': {'type': 'integer'} + 'year': {'type': 'integer'}, + 'tracks': {'type':'array', 'items':sj_track} } } @@ -147,4 +153,74 @@ def dynamic_url(artistid, albums=True, top_tracks=0, rel_artist=0): ret += '&num-related-artists=%d' % rel_artist return ret +class GetAlbum(MCall): + static_method = 'GET' + _res_schema = sj_album + + @staticmethod + def dynamic_url(albumid, tracks=True): + ret = sj_url + 'fetchalbum?alt=json' + ret += '&nid=%s' % albumid + ret += '&include-tracks=%r' % tracks + return ret + +class GetTrack(MCall): + static_method = 'GET' + _res_schema = sj_track + + @staticmethod + def dynamic_url(trackid): + ret = sj_url + 'fetchtrack?alt=json' + ret += '&nid=%s' % trackid + return ret + +class GetStreamUrl(MCall): + """Used to request a streaming link of a track.""" + + static_method = 'GET' + static_url = 'https://play.google.com/music/play' # note use of base_url, not service_url + + required_auth = authtypes(sso=True) # no xt required + + _res_schema = { + "type": "object", + "properties": { + "url": {"type": "string", "required": False}, + "urls": {"type": "array", "required": False} + }, + "additionalProperties": False + } + + @staticmethod + def dynamic_params(song_id): + + # https://github.com/simon-weber/Unofficial-Google-Music-API/issues/137 + # there are three cases when streaming: + # | track type | guid songid? | slt/sig needed? | + # user-uploaded yes no + # AA track in library yes yes + # AA track not in library no yes + + # without the track['type'] field we can't tell between 1 and 2, but + # include slt/sig anyway; the server ignores the extra params. + key = '27f7313e-f75d-445a-ac99-56386a5fe879' + salt = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(12)) + sig = binascii.b2a_base64(hmac.new(key, (song_id + salt), sha1).digest())[:-1] + urlsafe_b64_trans = string.maketrans("+/=", "-_.") + sig = sig.translate(urlsafe_b64_trans) + + params = { + 'u': 0, + 'pt': 'e', + 'slt': salt, + 'sig': sig + } + + # TODO match guid instead, should be more robust + if song_id[0] == 'T': + # all access + params['mjck'] = song_id + else: + params['songid'] = song_id + return params From cd8bfbcbc6fe6955aa8d67e5e82208c2ce4e2c25 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 3 Jul 2013 12:25:12 -0400 Subject: [PATCH 36/72] break clients into package --- example.py | 4 +- gmusicapi/clients.py | 1429 ----------------------------- gmusicapi/clients/__init__.py | 5 + gmusicapi/clients/clients.py | 0 gmusicapi/clients/mobileclient.py | 127 +++ gmusicapi/clients/musicmanager.py | 570 ++++++++++++ gmusicapi/clients/shared.py | 79 ++ gmusicapi/clients/webclient.py | 664 ++++++++++++++ gmusicapi/utils/utils.py | 7 +- 9 files changed, 1452 insertions(+), 1433 deletions(-) delete mode 100644 gmusicapi/clients.py create mode 100644 gmusicapi/clients/__init__.py create mode 100644 gmusicapi/clients/clients.py create mode 100644 gmusicapi/clients/mobileclient.py create mode 100644 gmusicapi/clients/musicmanager.py create mode 100644 gmusicapi/clients/shared.py create mode 100644 gmusicapi/clients/webclient.py diff --git a/example.py b/example.py index 3c740c7a..887c45b2 100644 --- a/example.py +++ b/example.py @@ -52,8 +52,8 @@ def demonstrate(): # this is essentially a random song. first_song = library[0] print "The first song I see is '{}' by '{}'.".format( - first_song["name"], - first_song["artist"]) + first_song["name"].encode('utf-8'), + first_song["artist"].encode('utf-8')) # We're going to create a new playlist and add a song to it. # Songs are uniquely identified by 'song ids', so let's get the id: diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py deleted file mode 100644 index 0452f2c9..00000000 --- a/gmusicapi/clients.py +++ /dev/null @@ -1,1429 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -The clients module exposes the main user-facing interfaces of gmusicapi. -""" - -import copy -import logging -import os -from socket import gethostname -import time -import urllib -from urlparse import urlparse, parse_qsl -from uuid import getnode as getmac -import webbrowser - -import httplib2 # included with oauth2client -from oauth2client.client import OAuth2WebServerFlow, TokenRevokeError -import oauth2client.file - -import gmusicapi -from gmusicapi.compat import my_appdirs -from gmusicapi.gmtools import tools -from gmusicapi.exceptions import CallFailure, NotLoggedIn -from gmusicapi.protocol import webclient, musicmanager, mobileclient, upload_pb2, locker_pb2 -from gmusicapi.utils import utils -import gmusicapi.session - -OAUTH_FILEPATH = os.path.join(my_appdirs.user_data_dir, 'oauth.cred') - - -class _Base(object): - """Factors out common client setup.""" - - __metaclass__ = utils.DocstringInheritMeta - - num_clients = 0 # used to disambiguate loggers - - def __init__(self, logger_basename, debug_logging, validate): - """ - - :param debug_logging: each Client has a ``logger`` member. - The logger is named ``gmusicapi.`` and - will propogate to the ``gmusicapi`` root logger. - - If this param is ``True``, handlers will be configured to send - this client's debug log output to disk, - with warnings and above printed to stderr. - `Appdirs `__ - ``user_log_dir`` is used by default. Users can run:: - - from gmusicapi.utils import utils - print utils.log_filepath - - to see the exact location on their system. - - If ``False``, no handlers will be configured; - users must create their own handlers. - - Completely ignoring logging is dangerous and not recommended. - The Google Music protocol can change at any time; if - something were to go wrong, the logs would be necessary for - recovery. - - :param validate: if False, do not validate server responses against - known schemas. This helps to catch protocol changes, but requires - significant cpu work. - - This arg is stored as ``self.validate`` and can be safely - modified at runtime. - """ - # this isn't correct if init is called more than once, so we log the - # client name below to avoid confusion for people reading logs - _Base.num_clients += 1 - - logger_name = "gmusicapi.%s%s" % (logger_basename, - _Base.num_clients) - self.logger = logging.getLogger(logger_name) - self.validate = validate - - if debug_logging: - utils.configure_debug_log_handlers(self.logger) - - self.logger.info("initialized") - - def _make_call(self, protocol, *args, **kwargs): - """Returns the response of a protocol.Call. - - args/kwargs are passed to protocol.perform. - - CallFailure may be raised.""" - - return protocol.perform(self.session, self.validate, *args, **kwargs) - - def is_authenticated(self): - """Returns ``True`` if the Api can make an authenticated request.""" - return self.session.is_authenticated - - def logout(self): - """Forgets local authentication in this Api instance. - Returns ``True`` on success.""" - - self.session.logout() - self.logger.info("logged out") - return True - - -class Musicmanager(_Base): - """Allows uploading by posing as Google's Music Manager. - - Musicmanager uses OAuth, so a plaintext email and password are not required - when logging in. - - For most users, :func:`perform_oauth` should be run once per machine to - store credentials to disk. Future calls to :func:`login` can use - use the stored credentials by default. - - Alternatively, users can implement the OAuth flow themselves, then - provide credentials directly to :func:`login`. - """ - - @staticmethod - def perform_oauth(storage_filepath=OAUTH_FILEPATH, open_browser=False): - """Provides a series of prompts for a user to follow to authenticate. - Returns ``oauth2client.client.OAuth2Credentials`` when successful. - - In most cases, this should only be run once per machine to store - credentials to disk, then never be needed again. - - If the user refuses to give access, - ``oauth2client.client.FlowExchangeError`` is raised. - - :param storage_filepath: a filepath to write the credentials to, - or ``None`` - to not write the credentials to disk (which is not recommended). - - `Appdirs `__ - ``user_data_dir`` is used by default. Users can run:: - - import gmusicapi.clients - print gmusicapi.clients.OAUTH_FILEPATH - - to see the exact location on their system. - - :param open_browser: if True, attempt to open the auth url - in the system default web browser. The url will be printed - regardless of this param's setting. - """ - - flow = OAuth2WebServerFlow(*musicmanager.oauth) - - auth_uri = flow.step1_get_authorize_url() - print - print "Visit the following url:\n %s" % auth_uri - - if open_browser: - print - print 'Opening your browser to it now...', - webbrowser.open(auth_uri) - print 'done.' - print "If you don't see your browser, you can just copy and paste the url." - print - - code = raw_input("Follow the prompts," - " then paste the auth code here and hit enter: ") - - credentials = flow.step2_exchange(code) - - if storage_filepath is not None: - if storage_filepath == OAUTH_FILEPATH: - utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700) - storage = oauth2client.file.Storage(storage_filepath) - storage.put(credentials) - - return credentials - - def __init__(self, debug_logging=True, validate=True): - self.session = gmusicapi.session.Musicmanager() - - super(Musicmanager, self).__init__(self.__class__.__name__, debug_logging, validate) - self.logout() - - def login(self, oauth_credentials=OAUTH_FILEPATH, - uploader_id=None, uploader_name=None): - """Authenticates the Music Manager using OAuth. - Returns ``True`` on success, ``False`` on failure. - - Unlike the :class:`Webclient`, OAuth allows authentication without - providing plaintext credentials to the application. - - In most cases, the default parameters should be acceptable. Users on - virtual machines will want to provide `uploader_id`. - - :param oauth_credentials: ``oauth2client.client.OAuth2Credentials`` or the path to a - ``oauth2client.file.Storage`` file. By default, the same default path used by - :func:`perform_oauth` is used. - - Endusers will likely call :func:`perform_oauth` once to write - credentials to disk and then ignore this parameter. - - This param - is mostly intended to allow flexibility for developers of a - 3rd party service who intend to perform their own OAuth flow - (eg on their website). - - :param uploader_id: a unique id as a MAC address, eg ``'00:11:22:33:AA:BB'``. - This should only be provided in cases where the default - (host MAC address incremented by 1) will not work. - - Upload behavior is undefined if a Music Manager uses the same id, especially when - reporting bad matches. - - ``ValueError`` will be raised if this is provided but not in the proper form. - - ``OSError`` will be raised if this is not provided and a real MAC could not be - determined (most common when running on a VPS). - - If provided, use the same id on all future runs for this machine, - because of the upload device limit explained below. - - :param uploader_name: human-readable non-unique id; default is - ``" (gmusicapi-{version})"``. - - This doesn't appear to be a part of authentication at all. - Registering with (id, name = X, Y) and logging in with - (id, name = X, Z) works, and does not change the server-stored - uploader_name. - - There are hard limits on how many upload devices can be registered; refer to `Google's - docs `__. There - have been limits on deauthorizing devices in the past, so it's smart not to register - more devices than necessary. - """ - - return (self._oauth_login(oauth_credentials) and - self._perform_upauth(uploader_id, uploader_name)) - - def _oauth_login(self, oauth_credentials): - """Auth ourselves to the MM oauth endpoint. - - Return True on success; see :py:func:`login` for params. - """ - - if isinstance(oauth_credentials, basestring): - oauth_file = oauth_credentials - if oauth_file == OAUTH_FILEPATH: - utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700) - storage = oauth2client.file.Storage(oauth_file) - - oauth_credentials = storage.get() - if oauth_credentials is None: - self.logger.warning("could not retrieve oauth credentials from '%s'", oauth_file) - return False - - if not self.session.login(oauth_credentials): - self.logger.warning("failed to authenticate") - return False - - self.logger.info("oauth successful") - - return True - - def _perform_upauth(self, uploader_id, uploader_name): - """Auth or register ourselves as an upload client. - - Return True on success; see :py:func:`login` for params. - """ - - if uploader_id is None: - mac_int = getmac() - if (mac_int >> 40) % 2: - raise OSError('a valid MAC could not be determined.' - ' Provide uploader_id (and be' - ' sure to provide the same one on future runs).') - - else: - #distinguish us from a Music Manager on this machine - mac_int = (mac_int + 1) % (1 << 48) - - uploader_id = utils.create_mac_string(mac_int) - - if not utils.is_valid_mac(uploader_id): - raise ValueError('uploader_id is not in a valid form.' - '\nProvide 6 pairs of hex digits' - ' with capital letters', - ' (eg "00:11:22:33:AA:BB")') - - if uploader_name is None: - uploader_name = gethostname() + u" (gmusicapi-%s)" % gmusicapi.__version__ - - try: - # this is a MM-specific step that might register a new device. - self._make_call(musicmanager.AuthenticateUploader, - uploader_id, - uploader_name) - self.logger.info("successful upauth") - self.uploader_id = uploader_id - self.uploader_name = uploader_name - - except CallFailure: - self.logger.exception("upauth failure") - self.session.logout() - return False - - return True - - def logout(self, revoke_oauth=False): - """Forgets local authentication in this Client instance. - - :param revoke_oauth: if True, oauth credentials will be permanently - revoked. If credentials came from a file, it will be deleted. - - Returns ``True`` on success.""" - - # TODO the login/logout stuff is all over the place - - success = True - - if revoke_oauth: - try: - # this automatically deletes a Storage file, if present - self.session._oauth_creds.revoke(httplib2.Http()) - except TokenRevokeError: - self.logger.exception("could not revoke oauth credentials") - success = False - - self.uploader_id = None - self.uploader_name = None - - return success and super(Musicmanager, self).logout() - - # mostly copy-paste from Webclient.get_all_songs. - # not worried about overlap in this case; the logic of either could change. - def get_all_songs(self, incremental=False): - """Returns a list of dictionaries, each with the following keys: - ``('id', 'title', 'album', 'album_artist', 'artist', 'track_number', - 'track_size')``. - - :param incremental: if True, return a generator that yields lists - of at most 1000 dictionaries - as they are retrieved from the server. This can be useful for - presenting a loading bar to a user. - """ - - to_return = self._get_all_songs() - - if not incremental: - to_return = [song for chunk in to_return for song in chunk] - - return to_return - - @staticmethod - def _track_info_to_dict(track_info): - """Given a download_pb2.DownloadTrackInfo, return a dictionary.""" - # figure it's better to hardcode keys here than use introspection - # and risk returning a new field all of a sudden. - - return dict((field, getattr(track_info, field)) for field in - ('id', 'title', 'album', 'album_artist', 'artist', - 'track_number', 'track_size')) - - def _get_all_songs(self): - """Return a generator of song chunks.""" - - get_next_chunk = True - - # need to spoof .continuation_token access, and - # can't add attrs to object(). Can with functions. - - lib_chunk = lambda: 0 - lib_chunk.continuation_token = None - - while get_next_chunk: - lib_chunk = self._make_call(musicmanager.ListTracks, - self.uploader_id, - lib_chunk.continuation_token) - - yield [self._track_info_to_dict(info) - for info in lib_chunk.download_track_info] - - get_next_chunk = lib_chunk.HasField('continuation_token') - - @utils.enforce_id_param - def download_song(self, song_id): - """Returns a tuple ``(u'suggested_filename', 'audio_bytestring')``. - The filename - will be what the Music Manager would save the file as, - presented as a unicode string with the proper file extension. - You don't have to use it if you don't want. - - - :param song_id: a single song id. - - To write the song to disk, use something like:: - - filename, audio = mm.download_song(an_id) - - # if open() throws a UnicodeEncodeError, either use - # filename.encode('utf-8') - # or change your default encoding to something sane =) - with open(filename, 'wb') as f: - f.write(audio) - - Unlike with :py:func:`Webclient.get_song_download_info - `, - there is no download limit when using this interface. - - Also unlike the Webclient, downloading a track requires authentication. - Returning a url does not suffice, since retrieving a track without auth - will produce an http 500. - """ - - url = self._make_call(musicmanager.GetDownloadLink, - song_id, - self.uploader_id)['url'] - - response = self._make_call(musicmanager.DownloadTrack, url) - - cd_header = response.headers['content-disposition'] - - filename = urllib.unquote(cd_header.split("filename*=UTF-8''")[-1]) - filename = filename.decode('utf-8') - - return (filename, response.content) - - # def get_quota(self): - # """Returns a tuple of (allowed number of tracks, total tracks, available tracks).""" - # quota = self._mm_pb_call("client_state").quota - # #protocol incorrect here... - # return (quota.maximumTracks, quota.totalTracks, quota.availableTracks) - - @utils.accept_singleton(basestring) - @utils.empty_arg_shortcircuit(return_code='{}') - def upload(self, filepaths, transcode_quality=3, enable_matching=False): - """Uploads the given filepaths. - Any non-mp3 files will be `transcoded with avconv - `__ before being uploaded. - - Return a 3-tuple ``(uploaded, matched, not_uploaded)`` of dictionaries, eg:: - - ( - {'': ''}, # uploaded - {'': ''}, # matched - {'': ''} # not uploaded - ) - - :param filepaths: a list of filepaths, or a single filepath. - :param transcode_quality: if int, pass to avconv ``-qscale`` for libmp3lame - (lower-better int, roughly corresponding to `hydrogenaudio -vX settings - `__). - If string, pass to avconv ``-ab`` (eg ``'128k'`` for an average bitrate of 128k). The - default is ~175kbs vbr. - - :param enable_matching: if ``True``, attempt to use `scan and match - `__ - to avoid uploading every song. - **WARNING**: currently, mismatched songs can *not* be fixed with the 'Fix Incorrect Match' - button nor :py:func:`report_incorrect_match - `. - They would have to be deleted and reuploaded with matching disabled - (or with the Music Manager). - Fixing matches from gmusicapi may be supported in a future release; see issue `#89 - `__. - - All Google-supported filetypes are supported; see `Google's documentation - `__. - - Unlike Google's Music Manager, this function will currently allow the same song to - be uploaded more than once if its tags are changed. This is subject to change in the future. - - If ``PERMANENT_ERROR`` is given as a not_uploaded reason, attempts to reupload will never - succeed. The file will need to be changed before the server will reconsider it; the easiest - way is to change metadata tags (it's not important that the tag be uploaded, just that the - contents of the file change somehow). - """ - - if self.uploader_id is None or self.uploader_name is None: - raise NotLoggedIn("Not authenticated as an upload device;" - " run Api.login(...perform_upload_auth=True...)" - " first.") - - #TODO there is way too much code in this function. - - #To return. - uploaded = {} - matched = {} - not_uploaded = {} - - #Gather local information on the files. - local_info = {} # {clientid: (path, Track)} - for path in filepaths: - try: - track = musicmanager.UploadMetadata.fill_track_info(path) - except BaseException as e: - self.logger.exception("problem gathering local info of '%r'", path) - - user_err_msg = str(e) - - if 'Non-ASCII strings must be converted to unicode' in str(e): - #This is a protobuf-specific error; they require either ascii or unicode. - #To keep behavior consistent, make no effort to guess - require users - # to decode first. - user_err_msg = ("nonascii bytestrings must be decoded to unicode" - " (error: '%s')" % user_err_msg) - - not_uploaded[path] = user_err_msg - else: - local_info[track.client_id] = (path, track) - - if not local_info: - return uploaded, matched, not_uploaded - - #TODO allow metadata faking - - #Upload metadata; the server tells us what to do next. - res = self._make_call(musicmanager.UploadMetadata, - [track for (path, track) in local_info.values()], - self.uploader_id) - - #TODO checking for proper contents should be handled in verification - md_res = res.metadata_response - - responses = [r for r in md_res.track_sample_response] - sample_requests = [req for req in md_res.signed_challenge_info] - - #Send scan and match samples if requested. - for sample_request in sample_requests: - path, track = local_info[sample_request.challenge_info.client_track_id] - - bogus_sample = None - if not enable_matching: - bogus_sample = '' # just send empty bytes - - try: - res = self._make_call(musicmanager.ProvideSample, - path, sample_request, track, - self.uploader_id, bogus_sample) - - except (IOError, ValueError) as e: - self.logger.warning("couldn't create scan and match sample for '%s': %s", - path, str(e)) - not_uploaded[path] = str(e) - else: - responses.extend(res.sample_response.track_sample_response) - - #Read sample responses and prep upload requests. - to_upload = {} # {serverid: (path, Track, do_not_rematch?)} - for sample_res in responses: - path, track = local_info[sample_res.client_track_id] - - if sample_res.response_code == upload_pb2.TrackSampleResponse.MATCHED: - self.logger.info("matched '%s' to sid %s", path, sample_res.server_track_id) - - if enable_matching: - matched[path] = sample_res.server_track_id - else: - self.logger.exception("'%s' was matched without matching enabled", path) - - elif sample_res.response_code == upload_pb2.TrackSampleResponse.UPLOAD_REQUESTED: - to_upload[sample_res.server_track_id] = (path, track, False) - - else: - # there was a problem - # report the symbolic name of the response code enum for debugging - enum_desc = upload_pb2._TRACKSAMPLERESPONSE.enum_types[0] - res_name = enum_desc.values_by_number[sample_res.response_code].name - - err_msg = "TrackSampleResponse code %s: %s" % (sample_res.response_code, res_name) - - if res_name == 'ALREADY_EXISTS': - # include the sid, too - # this shouldn't be relied on externally, but I use it in - # tests - being surrounded by parens is how it's matched - err_msg += "(%s)" % sample_res.server_track_id - - self.logger.warning("upload of '%s' rejected: %s", path, err_msg) - not_uploaded[path] = err_msg - - #Send upload requests. - if to_upload: - #TODO reordering requests could avoid wasting time waiting for reup sync - self._make_call(musicmanager.UpdateUploadState, 'start', self.uploader_id) - - for server_id, (path, track, do_not_rematch) in to_upload.items(): - #It can take a few tries to get an session. - should_retry = True - attempts = 0 - - while should_retry and attempts < 10: - session = self._make_call(musicmanager.GetUploadSession, - self.uploader_id, len(uploaded), - track, path, server_id, do_not_rematch) - attempts += 1 - - got_session, error_details = \ - musicmanager.GetUploadSession.process_session(session) - - if got_session: - self.logger.info("got an upload session for '%s'", path) - break - - should_retry, reason, error_code = error_details - self.logger.debug("problem getting upload session: %s\ncode=%s retrying=%s", - reason, error_code, should_retry) - - if error_code == 200 and do_not_rematch: - #reupload requests need to wait on a server sync - #200 == already uploaded, so force a retry in this case - should_retry = True - - time.sleep(6) # wait before retrying - else: - err_msg = "GetUploadSession error %s: %s" % (error_code, reason) - - self.logger.warning("giving up on upload session for '%s': %s", path, err_msg) - not_uploaded[path] = err_msg - - continue # to next upload - - #got a session, do the upload - #this terribly inconsistent naming isn't my fault: Google-- - session = session['sessionStatus'] - external = session['externalFieldTransfers'][0] - - session_url = external['putInfo']['url'] - content_type = external.get('content_type', 'audio/mpeg') - - if track.original_content_type != locker_pb2.Track.MP3: - try: - self.logger.info("transcoding '%s' to mp3", path) - contents = utils.transcode_to_mp3(path, quality=transcode_quality) - except (IOError, ValueError) as e: - self.logger.warning("error transcoding %s: %s", path, e) - not_uploaded[path] = "transcoding error: %s" % e - continue - else: - with open(path, 'rb') as f: - contents = f.read() - - upload_response = self._make_call(musicmanager.UploadFile, - session_url, content_type, contents) - - success = upload_response.get('sessionStatus', {}).get('state') - if success: - uploaded[path] = server_id - else: - #404 == already uploaded? serverside check on clientid? - self.logger.debug("could not finalize upload of '%s'. response: %s", - path, upload_response) - not_uploaded[path] = 'could not finalize upload; details in log' - - self._make_call(musicmanager.UpdateUploadState, 'stopped', self.uploader_id) - - return uploaded, matched, not_uploaded - - -class Mobileclient(_Base): - """Allows library management and streaming by posing as the - googleapis.com mobile clients. - - Uploading is not supported by this client (use the :class:`Musicmanager` - to upload). - """ - def __init__(self, debug_logging=True): - self.session = gmusicapi.session.Webclient() # TODO change name; now shared - - super(Mobileclient, self).__init__(self.__class__.__name__, debug_logging) - self.logout() - - def login(self, email, password): - """Authenticates the webclient. - Returns ``True`` on success, ``False`` on failure. - - :param email: eg ``'test@gmail.com'`` or just ``'test'``. - :param password: password or app-specific password for 2-factor users. - This is not stored locally, and is sent securely over SSL. - - Users of two-factor authentication will need to set an application-specific password - to log in. - """ - - if not self.session.login(email, password): - self.logger.info("failed to authenticate") - return False - - self.logger.info("authenticated") - - return True - - def search(self, query, max_results=5): - """Queries the server for songs and albums. - - :param query: a string keyword to search with. Capitalization and punctuation are ignored. - :param max_results: Maximum number of items to be retrieved - - The results are returned in a dictionary, arranged by how they were found. - ``artist_hits`` and ``song_hits`` return a list of - :ref:`song dictionaries `, while ``album_hits`` entries - have a different structure. - """ - - #XXX provide an example - res = self._make_call(mobileclient.Search, query, max_results)['entries'] - - return {"album_hits": [hit for hit in res if hit['type'] == "3"], - "artist_hits": [hit for hit in res if hit['type'] == "2"], - "song_hits": [hit for hit in res if hit['type'] == "1"]} - - def get_artist(self, artistid, albums=True, top_tracks=0, rel_artist=0): - """Retrieve artist data""" - res = self._make_call(mobileclient.GetArtist, artistid, albums, top_tracks, rel_artist) - return res - - def get_album(self, albumid, tracks=True): - """Retrieve artist data""" - res = self._make_call(mobileclient.GetAlbum, albumid, tracks) - return res - - def get_track(self, trackid): - """Retrieve artist data""" - res = self._make_call(mobileclient.GetTrack, trackid) - return res - - def get_stream_audio(self, song_id): - """Returns a bytestring containing mp3 audio for this song. - - :param song_id: a single song id - """ - - urls = self.get_stream_urls(song_id) - - if len(urls) == 1: - return self.session._rsession.get(urls[0]).content - - # AA tracks are separated into multiple files - # the url contains the range of each file to be used - - range_pairs = [[int(s) for s in val.split('-')] - for url in urls - for key, val in parse_qsl(urlparse(url)[4]) - if key == 'range'] - - stream_pieces = [] - prev_end = 0 - - for url, (start, end) in zip(urls, range_pairs): - audio = self.session._rsession.get(url).content - stream_pieces.append(audio[prev_end - start:]) - - prev_end = end + 1 - - return ''.join(stream_pieces) - - def get_stream_urls(self, song_id): - """Returns a url that points to a streamable version of this song. - - :param song_id: a single song id. - - While acquiring the url requires authentication, retreiving the - url contents does not. - - However, there are limitation as to how the stream url can be used: - * the url expires after about a minute - * only one IP can be streaming music at once. - Other attempts will get an http 403 with - ``X-Rejected-Reason: ANOTHER_STREAM_BEING_PLAYED``. - - *This is only intended for streaming*. The streamed audio does not contain metadata. - Use :func:`get_song_download_info` to download complete files with metadata. - """ - res = self._make_call(mobileclient.GetStreamUrl, song_id) - - try: - return res['url'] - except KeyError: - return res['urls'] - - -class Webclient(_Base): - """Allows library management and streaming by posing as the - music.google.com webclient. - - Uploading is not supported by this client (use the :class:`Musicmanager` - to upload). - """ - - def __init__(self, debug_logging=True, validate=True): - self.session = gmusicapi.session.Webclient() - - super(Webclient, self).__init__(self.__class__.__name__, debug_logging, validate) - self.logout() - - def login(self, email, password): - """Authenticates the webclient. - Returns ``True`` on success, ``False`` on failure. - - :param email: eg ``'test@gmail.com'`` or just ``'test'``. - :param password: password or app-specific password for 2-factor users. - This is not stored locally, and is sent securely over SSL. - - Users of two-factor authentication will need to set an application-specific password - to log in. - """ - - if not self.session.login(email, password): - self.logger.info("failed to authenticate") - return False - - self.logger.info("authenticated") - - return True - - def logout(self): - return super(Webclient, self).logout() - - def get_registered_devices(self): - """Returns a list of dictionaries, eg:: - [ - { - u'date': 1367470393588, # utc-millisecond - u'id': u'AA:BB:CC:11:22:33', - u'name': u'my-hostname', - u'type': u'DESKTOP_APP' - }, - { - u'carrier': u'Google', - u'date': 1344808742774 - u'id': u'0x00112233aabbccdd', - u'manufacturer': u'Asus', - u'model': u'Nexus 7', - u'name': u'', - u'type': u'PHONE', - } - ] - """ - - #TODO sessionid stuff - res = self._make_call(webclient.GetSettings, '') - return res['settings']['devices'] - - def change_playlist_name(self, playlist_id, new_name): - """Changes the name of a playlist. Returns the changed id. - - :param playlist_id: id of the playlist to rename. - :param new_title: desired title. - """ - - self._make_call(webclient.ChangePlaylistName, playlist_id, new_name) - - return playlist_id # the call actually doesn't return anything. - - @utils.accept_singleton(dict) - @utils.empty_arg_shortcircuit - def change_song_metadata(self, songs): - """Changes the metadata for some :ref:`song dictionaries `. - Returns a list of the song ids changed. - - :param songs: a list of :ref:`song dictionaries `, - or a single :ref:`song dictionary `. - - Generally, stick to these metadata keys: - - * ``rating``: set to 0 (no thumb), 1 (down thumb), or 5 (up thumb) - * ``name``: use this instead of ``title`` - * ``album`` - * ``albumArtist`` - * ``artist`` - * ``composer`` - * ``disc`` - * ``genre`` - * ``playCount`` - * ``totalDiscs`` - * ``totalTracks`` - * ``track`` - * ``year`` - """ - - res = self._make_call(webclient.ChangeSongMetadata, songs) - - return [s['id'] for s in res['songs']] - - def create_playlist(self, name): - """Creates a new playlist. Returns the new playlist id. - - :param title: the title of the playlist to create. - """ - - return self._make_call(webclient.AddPlaylist, name)['id'] - - def delete_playlist(self, playlist_id): - """Deletes a playlist. Returns the deleted id. - - :param playlist_id: id of the playlist to delete. - """ - - res = self._make_call(webclient.DeletePlaylist, playlist_id) - - return res['deleteId'] - - @utils.accept_singleton(basestring) - @utils.empty_arg_shortcircuit - @utils.enforce_ids_param - def delete_songs(self, song_ids): - """Deletes songs from the entire library. Returns a list of deleted song ids. - - :param song_ids: a list of song ids, or a single song id. - """ - - res = self._make_call(webclient.DeleteSongs, song_ids) - - return res['deleteIds'] - - def get_all_songs(self, incremental=False): - """Returns a list of :ref:`song dictionaries `. - - :param incremental: if True, return a generator that yields lists - of at most 2500 :ref:`song dictionaries ` - as they are retrieved from the server. This can be useful for - presenting a loading bar to a user. - """ - - to_return = self._get_all_songs() - - if not incremental: - to_return = [song for chunk in to_return for song in chunk] - - return to_return - - def _get_all_songs(self): - """Return a generator of song chunks.""" - - get_next_chunk = True - lib_chunk = {'continuationToken': None} - - while get_next_chunk: - lib_chunk = self._make_call(webclient.GetLibrarySongs, - lib_chunk['continuationToken']) - - yield lib_chunk['playlist'] # list of songs of the chunk - - get_next_chunk = 'continuationToken' in lib_chunk - - def get_playlist_songs(self, playlist_id): - """Returns a list of :ref:`song dictionaries `, - which include ``playlistEntryId`` keys for the given playlist. - - :param playlist_id: id of the playlist to load. - - This will return ``[]`` if the playlist id does not exist. - """ - - res = self._make_call(webclient.GetPlaylistSongs, playlist_id) - return res['playlist'] - - def get_all_playlist_ids(self, auto=True, user=True): - """Returns a dictionary that maps playlist types to dictionaries. - - :param auto: create an ``'auto'`` subdictionary entry. - Currently, this will just map to ``{}``; support for 'Shared with me' and - 'Google Play recommends' is on the way ( - `#102 `__). - - Other auto playlists are not stored on the server, but calculated by the client. - See `this gist `__ for sample code for - 'Thumbs Up', 'Last Added', and 'Free and Purchased'. - - :param user: create a user ``'user'`` subdictionary entry for user-created playlists. - This includes anything that appears on the left side 'Playlists' bar (notably, saved - instant mixes). - - User playlist names will be unicode strings. - - Google Music allows multiple user playlists with the same name, so the ``'user'`` dictionary - will map onto lists of ids. Here's an example response:: - - { - 'auto':{}, - - 'user':{ - u'Some Song Mix':[ - u'14814747-efbf-4500-93a1-53291e7a5919' - ], - - u'Two playlists have this name':[ - u'c89078a6-0c35-4f53-88fe-21afdc51a414', - u'86c69009-ea5b-4474-bd2e-c0fe34ff5484' - ] - } - } - - There is currently no support for retrieving automatically-created instant mixes - (see issue `#67 `__). - - """ - - playlists = {} - - if auto: - #playlists['auto'] = self._get_auto_playlists() - playlists['auto'] = {} - if user: - res = self._make_call(webclient.GetPlaylistSongs, 'all') - playlists['user'] = self._playlist_list_to_dict(res['playlists']) - - return playlists - - def _playlist_list_to_dict(self, pl_list): - ret = {} - - for name, pid in ((p["title"], p["playlistId"]) for p in pl_list): - if not name in ret: - ret[name] = [] - ret[name].append(pid) - - return ret - - def _get_auto_playlists(self): - """For auto playlists, returns a dictionary which maps autoplaylist name to id.""" - - #Auto playlist ids are hardcoded in the wc javascript. - #When testing, an incorrect name here will be caught. - return {u'Thumbs up': u'auto-playlist-thumbs-up', - u'Last added': u'auto-playlist-recent', - u'Free and purchased': u'auto-playlist-promo'} - - @utils.enforce_id_param - def get_song_download_info(self, song_id): - """Returns a tuple: ``('', )``. - - :param song_id: a single song id. - - ``url`` will be ``None`` if the download limit is exceeded. - - GM allows 2 downloads per song. The download count may not always be accurate, - and the 2 download limit seems to be loosely enforced. - - This call alone does not count towards a download - - the count is incremented when ``url`` is retrieved. - """ - - #TODO the protocol expects a list of songs - could extend with accept_singleton - info = self._make_call(webclient.GetDownloadInfo, [song_id]) - url = info.get('url') - - return (url, info["downloadCounts"][song_id]) - - @utils.enforce_id_param - def get_stream_urls(self, song_id): - """Returns a list of urls that point to a streamable version of this song. - - If you just need the audio and are ok with gmusicapi doing the download, - consider using :func:`get_stream_audio` instead. - This abstracts away the differences between different kinds of tracks: - * normal tracks return a single url - * All Access tracks return multiple urls, which must be combined - - :param song_id: a single song id. - - While acquiring the urls requires authentication, retreiving the - contents does not. - - However, there are limitations on how the stream urls can be used: - * the urls expire after a minute - * only one IP can be streaming music at once. - Other attempts will get an http 403 with - ``X-Rejected-Reason: ANOTHER_STREAM_BEING_PLAYED``. - - *This is only intended for streaming*. The streamed audio does not contain metadata. - Use :func:`get_song_download_info` or :func:`Musicmanager.download_song - ` - to download files with metadata. - """ - - res = self._make_call(webclient.GetStreamUrl, song_id) - - try: - return [res['url']] - except KeyError: - return res['urls'] - - @utils.enforce_id_param - def get_stream_audio(self, song_id): - """Returns a bytestring containing mp3 audio for this song. - - :param song_id: a single song id - """ - - urls = self.get_stream_urls(song_id) - - if len(urls) == 1: - return self.session._rsession.get(urls[0]).content - - # AA tracks are separated into multiple files - # the url contains the range of each file to be used - - range_pairs = [[int(s) for s in val.split('-')] - for url in urls - for key, val in parse_qsl(urlparse(url)[4]) - if key == 'range'] - - stream_pieces = [] - prev_end = 0 - - for url, (start, end) in zip(urls, range_pairs): - audio = self.session._rsession.get(url).content - stream_pieces.append(audio[prev_end - start:]) - - prev_end = end + 1 - - return ''.join(stream_pieces) - - def copy_playlist(self, playlist_id, copy_name): - """Copies the contents of a playlist to a new playlist. Returns the id of the new playlist. - - :param playlist_id: id of the playlist to be copied. - :param copy_name: the name of the new copied playlist. - - This is useful for making backups of playlists before modifications. - """ - - orig_tracks = self.get_playlist_songs(playlist_id) - - new_id = self.create_playlist(copy_name) - self.add_songs_to_playlist(new_id, [t["id"] for t in orig_tracks]) - - return new_id - - def change_playlist(self, playlist_id, desired_playlist, safe=True): - """Changes the order and contents of an existing playlist. - Returns the id of the playlist when finished - - this may be the same as the argument in the case of a failure and recovery. - - :param playlist_id: the id of the playlist being modified. - :param desired_playlist: the desired contents and order as a list of - :ref:`song dictionaries `, like is returned - from :func:`get_playlist_songs`. - - :param safe: if ``True``, ensure playlists will not be lost if a problem occurs. - This may slow down updates. - - The server only provides 3 basic playlist mutations: addition, deletion, and reordering. - This function will use these to automagically apply the desired changes. - - However, this might involve multiple calls to the server, and if a call fails, - the playlist will be left in an inconsistent state. - The ``safe`` option makes a backup of the playlist before doing anything, so it can be - rolled back if a problem occurs. This is enabled by default. - This might slow down updates of very large playlists. - - There will always be a warning logged if a problem occurs, even if ``safe`` is ``False``. - """ - - #We'll be modifying the entries in the playlist, and need to copy it. - #Copying ensures two things: - # 1. the user won't see our changes - # 2. changing a key for one entry won't change it for another - which would be the case - # if the user appended the same song twice, for example. - desired_playlist = [copy.deepcopy(t) for t in desired_playlist] - server_tracks = self.get_playlist_songs(playlist_id) - - if safe: - #Make a backup. - #The backup is stored on the server as a new playlist with "_gmusicapi_backup" - # appended to the backed up name. - names_to_ids = self.get_all_playlist_ids()['user'] - playlist_name = (ni_pair[0] - for ni_pair in names_to_ids.iteritems() - if playlist_id in ni_pair[1]).next() - - backup_id = self.copy_playlist(playlist_id, playlist_name + u"_gmusicapi_backup") - - try: - #Counter, Counter, and set of id pairs to delete, add, and keep. - to_del, to_add, to_keep = \ - tools.find_playlist_changes(server_tracks, desired_playlist) - - ##Delete unwanted entries. - to_del_eids = [pair[1] for pair in to_del.elements()] - if to_del_eids: - self._remove_entries_from_playlist(playlist_id, to_del_eids) - - ##Add new entries. - to_add_sids = [pair[0] for pair in to_add.elements()] - if to_add_sids: - new_pairs = self.add_songs_to_playlist(playlist_id, to_add_sids) - - ##Update desired tracks with added tracks server-given eids. - #Map new sid -> [eids] - new_sid_to_eids = {} - for sid, eid in new_pairs: - if not sid in new_sid_to_eids: - new_sid_to_eids[sid] = [] - new_sid_to_eids[sid].append(eid) - - for d_t in desired_playlist: - if d_t["id"] in new_sid_to_eids: - #Found a matching sid. - match = d_t - sid = match["id"] - eid = match.get("playlistEntryId") - pair = (sid, eid) - - if pair in to_keep: - to_keep.remove(pair) # only keep one of the to_keep eids. - else: - match["playlistEntryId"] = new_sid_to_eids[sid].pop() - if len(new_sid_to_eids[sid]) == 0: - del new_sid_to_eids[sid] - - ##Now, the right eids are in the playlist. - ##Set the order of the tracks: - - #The web client has no way to dictate the order without block insertion, - # but the api actually supports setting the order to a given list. - #For whatever reason, though, it needs to be set backwards; might be - # able to get around this by messing with afterEntry and beforeEntry parameters. - if desired_playlist: - #can't *-unpack an empty list - sids, eids = zip(*tools.get_id_pairs(desired_playlist[::-1])) - - if sids: - self._make_call(webclient.ChangePlaylistOrder, playlist_id, sids, eids) - - ##Clean up the backup. - if safe: - self.delete_playlist(backup_id) - - except CallFailure: - self.logger.info("a subcall of change_playlist failed - " - "playlist %s is in an inconsistent state", playlist_id) - - if not safe: - raise # there's nothing we can do - else: # try to revert to the backup - self.logger.info("attempting to revert changes from playlist " - "'%s_gmusicapi_backup'", playlist_name) - - try: - self.delete_playlist(playlist_id) - self.change_playlist_name(backup_id, playlist_name) - except CallFailure: - self.logger.warning("failed to revert failed change_playlist call on '%s'", - playlist_name) - raise - else: - self.logger.info("reverted changes safely; playlist id of '%s' is now '%s'", - playlist_name, backup_id) - playlist_id = backup_id - - return playlist_id - - @utils.accept_singleton(basestring, 2) - @utils.empty_arg_shortcircuit(position=2) - @utils.enforce_ids_param(position=2) - def add_songs_to_playlist(self, playlist_id, song_ids): - """Appends songs to a playlist. - Returns a list of (song id, playlistEntryId) tuples that were added. - - :param playlist_id: id of the playlist to add to. - :param song_ids: a list of song ids, or a single song id. - """ - - res = self._make_call(webclient.AddToPlaylist, playlist_id, song_ids) - new_entries = res['songIds'] - - return [(e['songId'], e['playlistEntryId']) for e in new_entries] - - @utils.accept_singleton(basestring, 2) - @utils.empty_arg_shortcircuit(position=2) - def remove_songs_from_playlist(self, playlist_id, sids_to_match): - """Removes all copies of the given song ids from a playlist. - Returns a list of removed (sid, eid) pairs. - - :param playlist_id: id of the playlist to remove songs from. - :param sids_to_match: a list of song ids to match, or a single song id. - - This does *not always* the inverse of a call to :func:`add_songs_to_playlist`, - since multiple copies of the same song are removed. For more control in this case, - get the playlist tracks with :func:`get_playlist_songs`, modify the list of tracks, - then use :func:`change_playlist` to push changes to the server. - """ - - playlist_tracks = self.get_playlist_songs(playlist_id) - sid_set = set(sids_to_match) - - matching_eids = [t["playlistEntryId"] - for t in playlist_tracks - if t["id"] in sid_set] - - if matching_eids: - #Call returns "sid_eid" strings. - sid_eids = self._remove_entries_from_playlist(playlist_id, - matching_eids) - return [s.split("_") for s in sid_eids] - else: - return [] - - @utils.accept_singleton(basestring, 2) - @utils.empty_arg_shortcircuit(position=2) - def _remove_entries_from_playlist(self, playlist_id, entry_ids_to_remove): - """Removes entries from a playlist. Returns a list of removed "sid_eid" strings. - - :param playlist_id: the playlist to be modified. - :param entry_ids: a list of entry ids, or a single entry id. - """ - - #GM requires the song ids in the call as well; find them. - playlist_tracks = self.get_playlist_songs(playlist_id) - remove_eid_set = set(entry_ids_to_remove) - - e_s_id_pairs = [(t["id"], t["playlistEntryId"]) - for t in playlist_tracks - if t["playlistEntryId"] in remove_eid_set] - - num_not_found = len(entry_ids_to_remove) - len(e_s_id_pairs) - if num_not_found > 0: - self.logger.warning("when removing, %d entry ids could not be found in playlist id %s", - num_not_found, playlist_id) - - #Unzip the pairs. - sids, eids = zip(*e_s_id_pairs) - - res = self._make_call(webclient.DeleteSongs, sids, playlist_id, eids) - - return res['deleteIds'] - - def search(self, query): - """Queries the server for songs and albums. - - **WARNING**: Google no longer uses this endpoint in their client; - it may stop working or be removed from gmusicapi without warning. - In addition, it is known to occasionally return unexpected results. - See `#114 - `__ - for more information. - - Instead of using this call, retrieve all tracks with :func:`get_all_songs` - and search them locally. `This gist - `__ has some examples of - simple linear-time searches. - - :param query: a string keyword to search with. Capitalization and punctuation are ignored. - - The results are returned in a dictionary, arranged by how they were found. - ``artist_hits`` and ``song_hits`` return a list of - :ref:`song dictionaries `, while ``album_hits`` entries - have a different structure. - - For example, a search on ``'cat'`` could return:: - - { - "album_hits": [ - { - "albumArtist": "The Cat Empire", - "albumName": "Cities: The Cat Empire Project", - "artistName": "The Cat Empire", - "imageUrl": "//ssl.gstatic.com/music/fe/[...].png" - # no more entries - }, - ], - "artist_hits": [ - { - "album": "Cinema", - "artist": "The Cat Empire", - "id": "c9214fc1-91fa-3bd2-b25d-693727a5f978", - "title": "Waiting" - # ... normal song dictionary - }, - ], - "song_hits": [ - { - "album": "Mandala", - "artist": "RX Bandits", - "id": "a7781438-8ec3-37ab-9c67-0ddb4115f60a", - "title": "Breakfast Cat", - # ... normal song dictionary - }, - ] - } - - """ - - res = self._make_call(webclient.Search, query)['results'] - - return {"album_hits": res["albums"], - "artist_hits": res["artists"], - "song_hits": res["songs"]} - - @utils.accept_singleton(basestring) - @utils.empty_arg_shortcircuit - @utils.enforce_id_param - def report_incorrect_match(self, song_ids): - """Equivalent to the 'Fix Incorrect Match' button, this requests re-uploading of songs. - Returns the song_ids provided. - - :param song_ids: a list of song ids to report, or a single song id. - - Note that if you uploaded a song through gmusicapi, it won't be reuploaded - automatically - this currently only works for songs uploaded with the Music Manager. - See issue `#89 `__. - - This should only be used on matched tracks (``song['type'] == 6``). - """ - - self._make_call(webclient.ReportBadSongMatch, song_ids) - - return song_ids - - @utils.accept_singleton(basestring) - @utils.empty_arg_shortcircuit - @utils.enforce_ids_param - def upload_album_art(self, song_ids, image_filepath): - """Uploads an image and sets it as the album art for songs. - - :param song_ids: a list of song ids, or a single song id. - :param image_filepath: filepath of the art to use. jpg and png are known to work. - - This function will *always* upload the provided image, even if it's already uploaded. - If the art is already uploaded and set for another song, copy over the - value of the ``'albumArtUrl'`` key using :func:`change_song_metadata` instead. - """ - - res = self._make_call(webclient.UploadImage, image_filepath) - url = res['imageUrl'] - - song_dicts = [dict((('id', id), ('albumArtUrl', url))) for id in song_ids] - - return self.change_song_metadata(song_dicts) diff --git a/gmusicapi/clients/__init__.py b/gmusicapi/clients/__init__.py new file mode 100644 index 00000000..9a8fab0d --- /dev/null +++ b/gmusicapi/clients/__init__.py @@ -0,0 +1,5 @@ +from gmusicapi.clients.webclient import Webclient +from gmusicapi.clients.musicmanager import Musicmanager, OAUTH_FILEPATH +from gmusicapi.clients.mobileclient import Mobileclient + +(Webclient, Musicmanager, Mobileclient, OAUTH_FILEPATH) # noqa diff --git a/gmusicapi/clients/clients.py b/gmusicapi/clients/clients.py new file mode 100644 index 00000000..e69de29b diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py new file mode 100644 index 00000000..c806b2ec --- /dev/null +++ b/gmusicapi/clients/mobileclient.py @@ -0,0 +1,127 @@ +from urlparse import urlparse, parse_qsl + +from gmusicapi.clients.shared import _Base +from gmusicapi.protocol import mobileclient +from gmusicapi import session + + +class Mobileclient(_Base): + """Allows library management and streaming by posing as the + googleapis.com mobile clients. + + Uploading is not supported by this client (use the :class:`Musicmanager` + to upload). + """ + def __init__(self, debug_logging=True): + self.session = session.Webclient() # TODO change name; now shared + + super(Mobileclient, self).__init__(self.__class__.__name__, debug_logging) + self.logout() + + def login(self, email, password): + """Authenticates the webclient. + Returns ``True`` on success, ``False`` on failure. + + :param email: eg ``'test@gmail.com'`` or just ``'test'``. + :param password: password or app-specific password for 2-factor users. + This is not stored locally, and is sent securely over SSL. + + Users of two-factor authentication will need to set an application-specific password + to log in. + """ + + if not self.session.login(email, password): + self.logger.info("failed to authenticate") + return False + + self.logger.info("authenticated") + + return True + + def search(self, query, max_results=5): + """Queries the server for songs and albums. + + :param query: a string keyword to search with. Capitalization and punctuation are ignored. + :param max_results: Maximum number of items to be retrieved + + The results are returned in a dictionary, arranged by how they were found. + ``artist_hits`` and ``song_hits`` return a list of + :ref:`song dictionaries `, while ``album_hits`` entries + have a different structure. + """ + + #XXX provide an example + res = self._make_call(mobileclient.Search, query, max_results)['entries'] + + return {"album_hits": [hit for hit in res if hit['type'] == "3"], + "artist_hits": [hit for hit in res if hit['type'] == "2"], + "song_hits": [hit for hit in res if hit['type'] == "1"]} + + def get_artist(self, artistid, albums=True, top_tracks=0, rel_artist=0): + """Retrieve artist data""" + res = self._make_call(mobileclient.GetArtist, artistid, albums, top_tracks, rel_artist) + return res + + def get_album(self, albumid, tracks=True): + """Retrieve artist data""" + res = self._make_call(mobileclient.GetAlbum, albumid, tracks) + return res + + def get_track(self, trackid): + """Retrieve artist data""" + res = self._make_call(mobileclient.GetTrack, trackid) + return res + + def get_stream_audio(self, song_id): + """Returns a bytestring containing mp3 audio for this song. + + :param song_id: a single song id + """ + + urls = self.get_stream_urls(song_id) + + if len(urls) == 1: + return self.session._rsession.get(urls[0]).content + + # AA tracks are separated into multiple files + # the url contains the range of each file to be used + + range_pairs = [[int(s) for s in val.split('-')] + for url in urls + for key, val in parse_qsl(urlparse(url)[4]) + if key == 'range'] + + stream_pieces = [] + prev_end = 0 + + for url, (start, end) in zip(urls, range_pairs): + audio = self.session._rsession.get(url).content + stream_pieces.append(audio[prev_end - start:]) + + prev_end = end + 1 + + return ''.join(stream_pieces) + + def get_stream_urls(self, song_id): + """Returns a url that points to a streamable version of this song. + + :param song_id: a single song id. + + While acquiring the url requires authentication, retreiving the + url contents does not. + + However, there are limitation as to how the stream url can be used: + * the url expires after about a minute + * only one IP can be streaming music at once. + Other attempts will get an http 403 with + ``X-Rejected-Reason: ANOTHER_STREAM_BEING_PLAYED``. + + *This is only intended for streaming*. The streamed audio does not contain metadata. + Use :func:`get_song_download_info` to download complete files with metadata. + """ + res = self._make_call(mobileclient.GetStreamUrl, song_id) + + try: + return res['url'] + except KeyError: + return res['urls'] diff --git a/gmusicapi/clients/musicmanager.py b/gmusicapi/clients/musicmanager.py new file mode 100644 index 00000000..a0e229f0 --- /dev/null +++ b/gmusicapi/clients/musicmanager.py @@ -0,0 +1,570 @@ +import os +from socket import gethostname +import time +import urllib +from uuid import getnode as getmac +import webbrowser + +import httplib2 # included with oauth2client +from oauth2client.client import OAuth2WebServerFlow, TokenRevokeError +import oauth2client.file + +import gmusicapi +from gmusicapi.clients.shared import _Base +from gmusicapi.compat import my_appdirs +from gmusicapi.exceptions import CallFailure, NotLoggedIn +from gmusicapi.protocol import musicmanager, upload_pb2, locker_pb2 +from gmusicapi.utils import utils +from gmusicapi import session + +OAUTH_FILEPATH = os.path.join(my_appdirs.user_data_dir, 'oauth.cred') + + +class Musicmanager(_Base): + """Allows uploading by posing as Google's Music Manager. + + Musicmanager uses OAuth, so a plaintext email and password are not required + when logging in. + + For most users, :func:`perform_oauth` should be run once per machine to + store credentials to disk. Future calls to :func:`login` can use + use the stored credentials by default. + + Alternatively, users can implement the OAuth flow themselves, then + provide credentials directly to :func:`login`. + """ + + @staticmethod + def perform_oauth(storage_filepath=OAUTH_FILEPATH, open_browser=False): + """Provides a series of prompts for a user to follow to authenticate. + Returns ``oauth2client.client.OAuth2Credentials`` when successful. + + In most cases, this should only be run once per machine to store + credentials to disk, then never be needed again. + + If the user refuses to give access, + ``oauth2client.client.FlowExchangeError`` is raised. + + :param storage_filepath: a filepath to write the credentials to, + or ``None`` + to not write the credentials to disk (which is not recommended). + + `Appdirs `__ + ``user_data_dir`` is used by default. Users can run:: + + import gmusicapi.clients + print gmusicapi.clients.OAUTH_FILEPATH + + to see the exact location on their system. + + :param open_browser: if True, attempt to open the auth url + in the system default web browser. The url will be printed + regardless of this param's setting. + """ + + flow = OAuth2WebServerFlow(*musicmanager.oauth) + + auth_uri = flow.step1_get_authorize_url() + print + print "Visit the following url:\n %s" % auth_uri + + if open_browser: + print + print 'Opening your browser to it now...', + webbrowser.open(auth_uri) + print 'done.' + print "If you don't see your browser, you can just copy and paste the url." + print + + code = raw_input("Follow the prompts," + " then paste the auth code here and hit enter: ") + + credentials = flow.step2_exchange(code) + + if storage_filepath is not None: + if storage_filepath == OAUTH_FILEPATH: + utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700) + storage = oauth2client.file.Storage(storage_filepath) + storage.put(credentials) + + return credentials + + def __init__(self, debug_logging=True, validate=True): + self.session = session.Musicmanager() + + super(Musicmanager, self).__init__(self.__class__.__name__, debug_logging, validate) + self.logout() + + def login(self, oauth_credentials=OAUTH_FILEPATH, + uploader_id=None, uploader_name=None): + """Authenticates the Music Manager using OAuth. + Returns ``True`` on success, ``False`` on failure. + + Unlike the :class:`Webclient`, OAuth allows authentication without + providing plaintext credentials to the application. + + In most cases, the default parameters should be acceptable. Users on + virtual machines will want to provide `uploader_id`. + + :param oauth_credentials: ``oauth2client.client.OAuth2Credentials`` or the path to a + ``oauth2client.file.Storage`` file. By default, the same default path used by + :func:`perform_oauth` is used. + + Endusers will likely call :func:`perform_oauth` once to write + credentials to disk and then ignore this parameter. + + This param + is mostly intended to allow flexibility for developers of a + 3rd party service who intend to perform their own OAuth flow + (eg on their website). + + :param uploader_id: a unique id as a MAC address, eg ``'00:11:22:33:AA:BB'``. + This should only be provided in cases where the default + (host MAC address incremented by 1) will not work. + + Upload behavior is undefined if a Music Manager uses the same id, especially when + reporting bad matches. + + ``ValueError`` will be raised if this is provided but not in the proper form. + + ``OSError`` will be raised if this is not provided and a real MAC could not be + determined (most common when running on a VPS). + + If provided, use the same id on all future runs for this machine, + because of the upload device limit explained below. + + :param uploader_name: human-readable non-unique id; default is + ``" (gmusicapi-{version})"``. + + This doesn't appear to be a part of authentication at all. + Registering with (id, name = X, Y) and logging in with + (id, name = X, Z) works, and does not change the server-stored + uploader_name. + + There are hard limits on how many upload devices can be registered; refer to `Google's + docs `__. There + have been limits on deauthorizing devices in the past, so it's smart not to register + more devices than necessary. + """ + + return (self._oauth_login(oauth_credentials) and + self._perform_upauth(uploader_id, uploader_name)) + + def _oauth_login(self, oauth_credentials): + """Auth ourselves to the MM oauth endpoint. + + Return True on success; see :py:func:`login` for params. + """ + + if isinstance(oauth_credentials, basestring): + oauth_file = oauth_credentials + if oauth_file == OAUTH_FILEPATH: + utils.make_sure_path_exists(os.path.dirname(OAUTH_FILEPATH), 0o700) + storage = oauth2client.file.Storage(oauth_file) + + oauth_credentials = storage.get() + if oauth_credentials is None: + self.logger.warning("could not retrieve oauth credentials from '%s'", oauth_file) + return False + + if not self.session.login(oauth_credentials): + self.logger.warning("failed to authenticate") + return False + + self.logger.info("oauth successful") + + return True + + def _perform_upauth(self, uploader_id, uploader_name): + """Auth or register ourselves as an upload client. + + Return True on success; see :py:func:`login` for params. + """ + + if uploader_id is None: + mac_int = getmac() + if (mac_int >> 40) % 2: + raise OSError('a valid MAC could not be determined.' + ' Provide uploader_id (and be' + ' sure to provide the same one on future runs).') + + else: + #distinguish us from a Music Manager on this machine + mac_int = (mac_int + 1) % (1 << 48) + + uploader_id = utils.create_mac_string(mac_int) + + if not utils.is_valid_mac(uploader_id): + raise ValueError('uploader_id is not in a valid form.' + '\nProvide 6 pairs of hex digits' + ' with capital letters', + ' (eg "00:11:22:33:AA:BB")') + + if uploader_name is None: + uploader_name = gethostname() + u" (gmusicapi-%s)" % gmusicapi.__version__ + + try: + # this is a MM-specific step that might register a new device. + self._make_call(musicmanager.AuthenticateUploader, + uploader_id, + uploader_name) + self.logger.info("successful upauth") + self.uploader_id = uploader_id + self.uploader_name = uploader_name + + except CallFailure: + self.logger.exception("upauth failure") + self.session.logout() + return False + + return True + + def logout(self, revoke_oauth=False): + """Forgets local authentication in this Client instance. + + :param revoke_oauth: if True, oauth credentials will be permanently + revoked. If credentials came from a file, it will be deleted. + + Returns ``True`` on success.""" + + # TODO the login/logout stuff is all over the place + + success = True + + if revoke_oauth: + try: + # this automatically deletes a Storage file, if present + self.session._oauth_creds.revoke(httplib2.Http()) + except TokenRevokeError: + self.logger.exception("could not revoke oauth credentials") + success = False + + self.uploader_id = None + self.uploader_name = None + + return success and super(Musicmanager, self).logout() + + # mostly copy-paste from Webclient.get_all_songs. + # not worried about overlap in this case; the logic of either could change. + def get_all_songs(self, incremental=False): + """Returns a list of dictionaries, each with the following keys: + ``('id', 'title', 'album', 'album_artist', 'artist', 'track_number', + 'track_size')``. + + :param incremental: if True, return a generator that yields lists + of at most 1000 dictionaries + as they are retrieved from the server. This can be useful for + presenting a loading bar to a user. + """ + + to_return = self._get_all_songs() + + if not incremental: + to_return = [song for chunk in to_return for song in chunk] + + return to_return + + @staticmethod + def _track_info_to_dict(track_info): + """Given a download_pb2.DownloadTrackInfo, return a dictionary.""" + # figure it's better to hardcode keys here than use introspection + # and risk returning a new field all of a sudden. + + return dict((field, getattr(track_info, field)) for field in + ('id', 'title', 'album', 'album_artist', 'artist', + 'track_number', 'track_size')) + + def _get_all_songs(self): + """Return a generator of song chunks.""" + + get_next_chunk = True + + # need to spoof .continuation_token access, and + # can't add attrs to object(). Can with functions. + + lib_chunk = lambda: 0 + lib_chunk.continuation_token = None + + while get_next_chunk: + lib_chunk = self._make_call(musicmanager.ListTracks, + self.uploader_id, + lib_chunk.continuation_token) + + yield [self._track_info_to_dict(info) + for info in lib_chunk.download_track_info] + + get_next_chunk = lib_chunk.HasField('continuation_token') + + @utils.enforce_id_param + def download_song(self, song_id): + """Returns a tuple ``(u'suggested_filename', 'audio_bytestring')``. + The filename + will be what the Music Manager would save the file as, + presented as a unicode string with the proper file extension. + You don't have to use it if you don't want. + + + :param song_id: a single song id. + + To write the song to disk, use something like:: + + filename, audio = mm.download_song(an_id) + + # if open() throws a UnicodeEncodeError, either use + # filename.encode('utf-8') + # or change your default encoding to something sane =) + with open(filename, 'wb') as f: + f.write(audio) + + Unlike with :py:func:`Webclient.get_song_download_info + `, + there is no download limit when using this interface. + + Also unlike the Webclient, downloading a track requires authentication. + Returning a url does not suffice, since retrieving a track without auth + will produce an http 500. + """ + + url = self._make_call(musicmanager.GetDownloadLink, + song_id, + self.uploader_id)['url'] + + response = self._make_call(musicmanager.DownloadTrack, url) + + cd_header = response.headers['content-disposition'] + + filename = urllib.unquote(cd_header.split("filename*=UTF-8''")[-1]) + filename = filename.decode('utf-8') + + return (filename, response.content) + + # def get_quota(self): + # """Returns a tuple of (allowed number of tracks, total tracks, available tracks).""" + # quota = self._mm_pb_call("client_state").quota + # #protocol incorrect here... + # return (quota.maximumTracks, quota.totalTracks, quota.availableTracks) + + @utils.accept_singleton(basestring) + @utils.empty_arg_shortcircuit(return_code='{}') + def upload(self, filepaths, transcode_quality=3, enable_matching=False): + """Uploads the given filepaths. + Any non-mp3 files will be `transcoded with avconv + `__ before being uploaded. + + Return a 3-tuple ``(uploaded, matched, not_uploaded)`` of dictionaries, eg:: + + ( + {'': ''}, # uploaded + {'': ''}, # matched + {'': ''} # not uploaded + ) + + :param filepaths: a list of filepaths, or a single filepath. + :param transcode_quality: if int, pass to avconv ``-qscale`` for libmp3lame + (lower-better int, roughly corresponding to `hydrogenaudio -vX settings + `__). + If string, pass to avconv ``-ab`` (eg ``'128k'`` for an average bitrate of 128k). The + default is ~175kbs vbr. + + :param enable_matching: if ``True``, attempt to use `scan and match + `__ + to avoid uploading every song. + **WARNING**: currently, mismatched songs can *not* be fixed with the 'Fix Incorrect Match' + button nor :py:func:`report_incorrect_match + `. + They would have to be deleted and reuploaded with matching disabled + (or with the Music Manager). + Fixing matches from gmusicapi may be supported in a future release; see issue `#89 + `__. + + All Google-supported filetypes are supported; see `Google's documentation + `__. + + Unlike Google's Music Manager, this function will currently allow the same song to + be uploaded more than once if its tags are changed. This is subject to change in the future. + + If ``PERMANENT_ERROR`` is given as a not_uploaded reason, attempts to reupload will never + succeed. The file will need to be changed before the server will reconsider it; the easiest + way is to change metadata tags (it's not important that the tag be uploaded, just that the + contents of the file change somehow). + """ + + if self.uploader_id is None or self.uploader_name is None: + raise NotLoggedIn("Not authenticated as an upload device;" + " run Api.login(...perform_upload_auth=True...)" + " first.") + + #TODO there is way too much code in this function. + + #To return. + uploaded = {} + matched = {} + not_uploaded = {} + + #Gather local information on the files. + local_info = {} # {clientid: (path, Track)} + for path in filepaths: + try: + track = musicmanager.UploadMetadata.fill_track_info(path) + except BaseException as e: + self.logger.exception("problem gathering local info of '%r'", path) + + user_err_msg = str(e) + + if 'Non-ASCII strings must be converted to unicode' in str(e): + #This is a protobuf-specific error; they require either ascii or unicode. + #To keep behavior consistent, make no effort to guess - require users + # to decode first. + user_err_msg = ("nonascii bytestrings must be decoded to unicode" + " (error: '%s')" % user_err_msg) + + not_uploaded[path] = user_err_msg + else: + local_info[track.client_id] = (path, track) + + if not local_info: + return uploaded, matched, not_uploaded + + #TODO allow metadata faking + + #Upload metadata; the server tells us what to do next. + res = self._make_call(musicmanager.UploadMetadata, + [track for (path, track) in local_info.values()], + self.uploader_id) + + #TODO checking for proper contents should be handled in verification + md_res = res.metadata_response + + responses = [r for r in md_res.track_sample_response] + sample_requests = [req for req in md_res.signed_challenge_info] + + #Send scan and match samples if requested. + for sample_request in sample_requests: + path, track = local_info[sample_request.challenge_info.client_track_id] + + bogus_sample = None + if not enable_matching: + bogus_sample = '' # just send empty bytes + + try: + res = self._make_call(musicmanager.ProvideSample, + path, sample_request, track, + self.uploader_id, bogus_sample) + + except (IOError, ValueError) as e: + self.logger.warning("couldn't create scan and match sample for '%s': %s", + path, str(e)) + not_uploaded[path] = str(e) + else: + responses.extend(res.sample_response.track_sample_response) + + #Read sample responses and prep upload requests. + to_upload = {} # {serverid: (path, Track, do_not_rematch?)} + for sample_res in responses: + path, track = local_info[sample_res.client_track_id] + + if sample_res.response_code == upload_pb2.TrackSampleResponse.MATCHED: + self.logger.info("matched '%s' to sid %s", path, sample_res.server_track_id) + + if enable_matching: + matched[path] = sample_res.server_track_id + else: + self.logger.exception("'%s' was matched without matching enabled", path) + + elif sample_res.response_code == upload_pb2.TrackSampleResponse.UPLOAD_REQUESTED: + to_upload[sample_res.server_track_id] = (path, track, False) + + else: + # there was a problem + # report the symbolic name of the response code enum for debugging + enum_desc = upload_pb2._TRACKSAMPLERESPONSE.enum_types[0] + res_name = enum_desc.values_by_number[sample_res.response_code].name + + err_msg = "TrackSampleResponse code %s: %s" % (sample_res.response_code, res_name) + + if res_name == 'ALREADY_EXISTS': + # include the sid, too + # this shouldn't be relied on externally, but I use it in + # tests - being surrounded by parens is how it's matched + err_msg += "(%s)" % sample_res.server_track_id + + self.logger.warning("upload of '%s' rejected: %s", path, err_msg) + not_uploaded[path] = err_msg + + #Send upload requests. + if to_upload: + #TODO reordering requests could avoid wasting time waiting for reup sync + self._make_call(musicmanager.UpdateUploadState, 'start', self.uploader_id) + + for server_id, (path, track, do_not_rematch) in to_upload.items(): + #It can take a few tries to get an session. + should_retry = True + attempts = 0 + + while should_retry and attempts < 10: + session = self._make_call(musicmanager.GetUploadSession, + self.uploader_id, len(uploaded), + track, path, server_id, do_not_rematch) + attempts += 1 + + got_session, error_details = \ + musicmanager.GetUploadSession.process_session(session) + + if got_session: + self.logger.info("got an upload session for '%s'", path) + break + + should_retry, reason, error_code = error_details + self.logger.debug("problem getting upload session: %s\ncode=%s retrying=%s", + reason, error_code, should_retry) + + if error_code == 200 and do_not_rematch: + #reupload requests need to wait on a server sync + #200 == already uploaded, so force a retry in this case + should_retry = True + + time.sleep(6) # wait before retrying + else: + err_msg = "GetUploadSession error %s: %s" % (error_code, reason) + + self.logger.warning("giving up on upload session for '%s': %s", path, err_msg) + not_uploaded[path] = err_msg + + continue # to next upload + + #got a session, do the upload + #this terribly inconsistent naming isn't my fault: Google-- + session = session['sessionStatus'] + external = session['externalFieldTransfers'][0] + + session_url = external['putInfo']['url'] + content_type = external.get('content_type', 'audio/mpeg') + + if track.original_content_type != locker_pb2.Track.MP3: + try: + self.logger.info("transcoding '%s' to mp3", path) + contents = utils.transcode_to_mp3(path, quality=transcode_quality) + except (IOError, ValueError) as e: + self.logger.warning("error transcoding %s: %s", path, e) + not_uploaded[path] = "transcoding error: %s" % e + continue + else: + with open(path, 'rb') as f: + contents = f.read() + + upload_response = self._make_call(musicmanager.UploadFile, + session_url, content_type, contents) + + success = upload_response.get('sessionStatus', {}).get('state') + if success: + uploaded[path] = server_id + else: + #404 == already uploaded? serverside check on clientid? + self.logger.debug("could not finalize upload of '%s'. response: %s", + path, upload_response) + not_uploaded[path] = 'could not finalize upload; details in log' + + self._make_call(musicmanager.UpdateUploadState, 'stopped', self.uploader_id) + + return uploaded, matched, not_uploaded diff --git a/gmusicapi/clients/shared.py b/gmusicapi/clients/shared.py new file mode 100644 index 00000000..6787500f --- /dev/null +++ b/gmusicapi/clients/shared.py @@ -0,0 +1,79 @@ +import logging + +from gmusicapi.utils import utils + + +class _Base(object): + """Factors out common client setup.""" + + __metaclass__ = utils.DocstringInheritMeta + + num_clients = 0 # used to disambiguate loggers + + def __init__(self, logger_basename, debug_logging, validate): + """ + + :param debug_logging: each Client has a ``logger`` member. + The logger is named ``gmusicapi.`` and + will propogate to the ``gmusicapi`` root logger. + + If this param is ``True``, handlers will be configured to send + this client's debug log output to disk, + with warnings and above printed to stderr. + `Appdirs `__ + ``user_log_dir`` is used by default. Users can run:: + + from gmusicapi.utils import utils + print utils.log_filepath + + to see the exact location on their system. + + If ``False``, no handlers will be configured; + users must create their own handlers. + + Completely ignoring logging is dangerous and not recommended. + The Google Music protocol can change at any time; if + something were to go wrong, the logs would be necessary for + recovery. + + :param validate: if False, do not validate server responses against + known schemas. This helps to catch protocol changes, but requires + significant cpu work. + + This arg is stored as ``self.validate`` and can be safely + modified at runtime. + """ + # this isn't correct if init is called more than once, so we log the + # client name below to avoid confusion for people reading logs + _Base.num_clients += 1 + + logger_name = "gmusicapi.%s%s" % (logger_basename, + _Base.num_clients) + self.logger = logging.getLogger(logger_name) + self.validate = validate + + if debug_logging: + utils.configure_debug_log_handlers(self.logger) + + self.logger.info("initialized") + + def _make_call(self, protocol, *args, **kwargs): + """Returns the response of a protocol.Call. + + args/kwargs are passed to protocol.perform. + + CallFailure may be raised.""" + + return protocol.perform(self.session, self.validate, *args, **kwargs) + + def is_authenticated(self): + """Returns ``True`` if the Api can make an authenticated request.""" + return self.session.is_authenticated + + def logout(self): + """Forgets local authentication in this Api instance. + Returns ``True`` on success.""" + + self.session.logout() + self.logger.info("logged out") + return True diff --git a/gmusicapi/clients/webclient.py b/gmusicapi/clients/webclient.py new file mode 100644 index 00000000..3fa40f70 --- /dev/null +++ b/gmusicapi/clients/webclient.py @@ -0,0 +1,664 @@ +# -*- coding: utf-8 -*- + +import copy +from urlparse import urlparse, parse_qsl + +import gmusicapi +from gmusicapi.clients.shared import _Base +from gmusicapi.gmtools import tools +from gmusicapi.exceptions import CallFailure +from gmusicapi.protocol import webclient +from gmusicapi.utils import utils +import gmusicapi.session + + +class Webclient(_Base): + """Allows library management and streaming by posing as the + music.google.com webclient. + + Uploading is not supported by this client (use the :class:`Musicmanager` + to upload). + """ + + def __init__(self, debug_logging=True, validate=True): + self.session = gmusicapi.session.Webclient() + + super(Webclient, self).__init__(self.__class__.__name__, debug_logging, validate) + self.logout() + + def login(self, email, password): + """Authenticates the webclient. + Returns ``True`` on success, ``False`` on failure. + + :param email: eg ``'test@gmail.com'`` or just ``'test'``. + :param password: password or app-specific password for 2-factor users. + This is not stored locally, and is sent securely over SSL. + + Users of two-factor authentication will need to set an application-specific password + to log in. + """ + + if not self.session.login(email, password): + self.logger.info("failed to authenticate") + return False + + self.logger.info("authenticated") + + return True + + def logout(self): + return super(Webclient, self).logout() + + def get_registered_devices(self): + """Returns a list of dictionaries, eg:: + [ + { + u'date': 1367470393588, # utc-millisecond + u'id': u'AA:BB:CC:11:22:33', + u'name': u'my-hostname', + u'type': u'DESKTOP_APP' + }, + { + u'carrier': u'Google', + u'date': 1344808742774 + u'id': u'0x00112233aabbccdd', + u'manufacturer': u'Asus', + u'model': u'Nexus 7', + u'name': u'', + u'type': u'PHONE', + } + ] + """ + + #TODO sessionid stuff + res = self._make_call(webclient.GetSettings, '') + return res['settings']['devices'] + + def change_playlist_name(self, playlist_id, new_name): + """Changes the name of a playlist. Returns the changed id. + + :param playlist_id: id of the playlist to rename. + :param new_title: desired title. + """ + + self._make_call(webclient.ChangePlaylistName, playlist_id, new_name) + + return playlist_id # the call actually doesn't return anything. + + @utils.accept_singleton(dict) + @utils.empty_arg_shortcircuit + def change_song_metadata(self, songs): + """Changes the metadata for some :ref:`song dictionaries `. + Returns a list of the song ids changed. + + :param songs: a list of :ref:`song dictionaries `, + or a single :ref:`song dictionary `. + + Generally, stick to these metadata keys: + + * ``rating``: set to 0 (no thumb), 1 (down thumb), or 5 (up thumb) + * ``name``: use this instead of ``title`` + * ``album`` + * ``albumArtist`` + * ``artist`` + * ``composer`` + * ``disc`` + * ``genre`` + * ``playCount`` + * ``totalDiscs`` + * ``totalTracks`` + * ``track`` + * ``year`` + """ + + res = self._make_call(webclient.ChangeSongMetadata, songs) + + return [s['id'] for s in res['songs']] + + def create_playlist(self, name): + """Creates a new playlist. Returns the new playlist id. + + :param title: the title of the playlist to create. + """ + + return self._make_call(webclient.AddPlaylist, name)['id'] + + def delete_playlist(self, playlist_id): + """Deletes a playlist. Returns the deleted id. + + :param playlist_id: id of the playlist to delete. + """ + + res = self._make_call(webclient.DeletePlaylist, playlist_id) + + return res['deleteId'] + + @utils.accept_singleton(basestring) + @utils.empty_arg_shortcircuit + @utils.enforce_ids_param + def delete_songs(self, song_ids): + """Deletes songs from the entire library. Returns a list of deleted song ids. + + :param song_ids: a list of song ids, or a single song id. + """ + + res = self._make_call(webclient.DeleteSongs, song_ids) + + return res['deleteIds'] + + def get_all_songs(self, incremental=False): + """Returns a list of :ref:`song dictionaries `. + + :param incremental: if True, return a generator that yields lists + of at most 2500 :ref:`song dictionaries ` + as they are retrieved from the server. This can be useful for + presenting a loading bar to a user. + """ + + to_return = self._get_all_songs() + + if not incremental: + to_return = [song for chunk in to_return for song in chunk] + + return to_return + + def _get_all_songs(self): + """Return a generator of song chunks.""" + + get_next_chunk = True + lib_chunk = {'continuationToken': None} + + while get_next_chunk: + lib_chunk = self._make_call(webclient.GetLibrarySongs, + lib_chunk['continuationToken']) + + yield lib_chunk['playlist'] # list of songs of the chunk + + get_next_chunk = 'continuationToken' in lib_chunk + + def get_playlist_songs(self, playlist_id): + """Returns a list of :ref:`song dictionaries `, + which include ``playlistEntryId`` keys for the given playlist. + + :param playlist_id: id of the playlist to load. + + This will return ``[]`` if the playlist id does not exist. + """ + + res = self._make_call(webclient.GetPlaylistSongs, playlist_id) + return res['playlist'] + + def get_all_playlist_ids(self, auto=True, user=True): + """Returns a dictionary that maps playlist types to dictionaries. + + :param auto: create an ``'auto'`` subdictionary entry. + Currently, this will just map to ``{}``; support for 'Shared with me' and + 'Google Play recommends' is on the way ( + `#102 `__). + + Other auto playlists are not stored on the server, but calculated by the client. + See `this gist `__ for sample code for + 'Thumbs Up', 'Last Added', and 'Free and Purchased'. + + :param user: create a user ``'user'`` subdictionary entry for user-created playlists. + This includes anything that appears on the left side 'Playlists' bar (notably, saved + instant mixes). + + User playlist names will be unicode strings. + + Google Music allows multiple user playlists with the same name, so the ``'user'`` dictionary + will map onto lists of ids. Here's an example response:: + + { + 'auto':{}, + + 'user':{ + u'Some Song Mix':[ + u'14814747-efbf-4500-93a1-53291e7a5919' + ], + + u'Two playlists have this name':[ + u'c89078a6-0c35-4f53-88fe-21afdc51a414', + u'86c69009-ea5b-4474-bd2e-c0fe34ff5484' + ] + } + } + + There is currently no support for retrieving automatically-created instant mixes + (see issue `#67 `__). + + """ + + playlists = {} + + if auto: + #playlists['auto'] = self._get_auto_playlists() + playlists['auto'] = {} + if user: + res = self._make_call(webclient.GetPlaylistSongs, 'all') + playlists['user'] = self._playlist_list_to_dict(res['playlists']) + + return playlists + + def _playlist_list_to_dict(self, pl_list): + ret = {} + + for name, pid in ((p["title"], p["playlistId"]) for p in pl_list): + if not name in ret: + ret[name] = [] + ret[name].append(pid) + + return ret + + def _get_auto_playlists(self): + """For auto playlists, returns a dictionary which maps autoplaylist name to id.""" + + #Auto playlist ids are hardcoded in the wc javascript. + #When testing, an incorrect name here will be caught. + return {u'Thumbs up': u'auto-playlist-thumbs-up', + u'Last added': u'auto-playlist-recent', + u'Free and purchased': u'auto-playlist-promo'} + + @utils.enforce_id_param + def get_song_download_info(self, song_id): + """Returns a tuple: ``('', )``. + + :param song_id: a single song id. + + ``url`` will be ``None`` if the download limit is exceeded. + + GM allows 2 downloads per song. The download count may not always be accurate, + and the 2 download limit seems to be loosely enforced. + + This call alone does not count towards a download - + the count is incremented when ``url`` is retrieved. + """ + + #TODO the protocol expects a list of songs - could extend with accept_singleton + info = self._make_call(webclient.GetDownloadInfo, [song_id]) + url = info.get('url') + + return (url, info["downloadCounts"][song_id]) + + @utils.enforce_id_param + def get_stream_urls(self, song_id): + """Returns a list of urls that point to a streamable version of this song. + + If you just need the audio and are ok with gmusicapi doing the download, + consider using :func:`get_stream_audio` instead. + This abstracts away the differences between different kinds of tracks: + * normal tracks return a single url + * All Access tracks return multiple urls, which must be combined + + :param song_id: a single song id. + + While acquiring the urls requires authentication, retreiving the + contents does not. + + However, there are limitations on how the stream urls can be used: + * the urls expire after a minute + * only one IP can be streaming music at once. + Other attempts will get an http 403 with + ``X-Rejected-Reason: ANOTHER_STREAM_BEING_PLAYED``. + + *This is only intended for streaming*. The streamed audio does not contain metadata. + Use :func:`get_song_download_info` or :func:`Musicmanager.download_song + ` + to download files with metadata. + """ + + res = self._make_call(webclient.GetStreamUrl, song_id) + + try: + return [res['url']] + except KeyError: + return res['urls'] + + @utils.enforce_id_param + def get_stream_audio(self, song_id): + """Returns a bytestring containing mp3 audio for this song. + + :param song_id: a single song id + """ + + urls = self.get_stream_urls(song_id) + + if len(urls) == 1: + return self.session._rsession.get(urls[0]).content + + # AA tracks are separated into multiple files + # the url contains the range of each file to be used + + range_pairs = [[int(s) for s in val.split('-')] + for url in urls + for key, val in parse_qsl(urlparse(url)[4]) + if key == 'range'] + + stream_pieces = [] + prev_end = 0 + + for url, (start, end) in zip(urls, range_pairs): + audio = self.session._rsession.get(url).content + stream_pieces.append(audio[prev_end - start:]) + + prev_end = end + 1 + + return ''.join(stream_pieces) + + def copy_playlist(self, playlist_id, copy_name): + """Copies the contents of a playlist to a new playlist. Returns the id of the new playlist. + + :param playlist_id: id of the playlist to be copied. + :param copy_name: the name of the new copied playlist. + + This is useful for making backups of playlists before modifications. + """ + + orig_tracks = self.get_playlist_songs(playlist_id) + + new_id = self.create_playlist(copy_name) + self.add_songs_to_playlist(new_id, [t["id"] for t in orig_tracks]) + + return new_id + + def change_playlist(self, playlist_id, desired_playlist, safe=True): + """Changes the order and contents of an existing playlist. + Returns the id of the playlist when finished - + this may be the same as the argument in the case of a failure and recovery. + + :param playlist_id: the id of the playlist being modified. + :param desired_playlist: the desired contents and order as a list of + :ref:`song dictionaries `, like is returned + from :func:`get_playlist_songs`. + + :param safe: if ``True``, ensure playlists will not be lost if a problem occurs. + This may slow down updates. + + The server only provides 3 basic playlist mutations: addition, deletion, and reordering. + This function will use these to automagically apply the desired changes. + + However, this might involve multiple calls to the server, and if a call fails, + the playlist will be left in an inconsistent state. + The ``safe`` option makes a backup of the playlist before doing anything, so it can be + rolled back if a problem occurs. This is enabled by default. + This might slow down updates of very large playlists. + + There will always be a warning logged if a problem occurs, even if ``safe`` is ``False``. + """ + + #We'll be modifying the entries in the playlist, and need to copy it. + #Copying ensures two things: + # 1. the user won't see our changes + # 2. changing a key for one entry won't change it for another - which would be the case + # if the user appended the same song twice, for example. + desired_playlist = [copy.deepcopy(t) for t in desired_playlist] + server_tracks = self.get_playlist_songs(playlist_id) + + if safe: + #Make a backup. + #The backup is stored on the server as a new playlist with "_gmusicapi_backup" + # appended to the backed up name. + names_to_ids = self.get_all_playlist_ids()['user'] + playlist_name = (ni_pair[0] + for ni_pair in names_to_ids.iteritems() + if playlist_id in ni_pair[1]).next() + + backup_id = self.copy_playlist(playlist_id, playlist_name + u"_gmusicapi_backup") + + try: + #Counter, Counter, and set of id pairs to delete, add, and keep. + to_del, to_add, to_keep = \ + tools.find_playlist_changes(server_tracks, desired_playlist) + + ##Delete unwanted entries. + to_del_eids = [pair[1] for pair in to_del.elements()] + if to_del_eids: + self._remove_entries_from_playlist(playlist_id, to_del_eids) + + ##Add new entries. + to_add_sids = [pair[0] for pair in to_add.elements()] + if to_add_sids: + new_pairs = self.add_songs_to_playlist(playlist_id, to_add_sids) + + ##Update desired tracks with added tracks server-given eids. + #Map new sid -> [eids] + new_sid_to_eids = {} + for sid, eid in new_pairs: + if not sid in new_sid_to_eids: + new_sid_to_eids[sid] = [] + new_sid_to_eids[sid].append(eid) + + for d_t in desired_playlist: + if d_t["id"] in new_sid_to_eids: + #Found a matching sid. + match = d_t + sid = match["id"] + eid = match.get("playlistEntryId") + pair = (sid, eid) + + if pair in to_keep: + to_keep.remove(pair) # only keep one of the to_keep eids. + else: + match["playlistEntryId"] = new_sid_to_eids[sid].pop() + if len(new_sid_to_eids[sid]) == 0: + del new_sid_to_eids[sid] + + ##Now, the right eids are in the playlist. + ##Set the order of the tracks: + + #The web client has no way to dictate the order without block insertion, + # but the api actually supports setting the order to a given list. + #For whatever reason, though, it needs to be set backwards; might be + # able to get around this by messing with afterEntry and beforeEntry parameters. + if desired_playlist: + #can't *-unpack an empty list + sids, eids = zip(*tools.get_id_pairs(desired_playlist[::-1])) + + if sids: + self._make_call(webclient.ChangePlaylistOrder, playlist_id, sids, eids) + + ##Clean up the backup. + if safe: + self.delete_playlist(backup_id) + + except CallFailure: + self.logger.info("a subcall of change_playlist failed - " + "playlist %s is in an inconsistent state", playlist_id) + + if not safe: + raise # there's nothing we can do + else: # try to revert to the backup + self.logger.info("attempting to revert changes from playlist " + "'%s_gmusicapi_backup'", playlist_name) + + try: + self.delete_playlist(playlist_id) + self.change_playlist_name(backup_id, playlist_name) + except CallFailure: + self.logger.warning("failed to revert failed change_playlist call on '%s'", + playlist_name) + raise + else: + self.logger.info("reverted changes safely; playlist id of '%s' is now '%s'", + playlist_name, backup_id) + playlist_id = backup_id + + return playlist_id + + @utils.accept_singleton(basestring, 2) + @utils.empty_arg_shortcircuit(position=2) + @utils.enforce_ids_param(position=2) + def add_songs_to_playlist(self, playlist_id, song_ids): + """Appends songs to a playlist. + Returns a list of (song id, playlistEntryId) tuples that were added. + + :param playlist_id: id of the playlist to add to. + :param song_ids: a list of song ids, or a single song id. + """ + + res = self._make_call(webclient.AddToPlaylist, playlist_id, song_ids) + new_entries = res['songIds'] + + return [(e['songId'], e['playlistEntryId']) for e in new_entries] + + @utils.accept_singleton(basestring, 2) + @utils.empty_arg_shortcircuit(position=2) + def remove_songs_from_playlist(self, playlist_id, sids_to_match): + """Removes all copies of the given song ids from a playlist. + Returns a list of removed (sid, eid) pairs. + + :param playlist_id: id of the playlist to remove songs from. + :param sids_to_match: a list of song ids to match, or a single song id. + + This does *not always* the inverse of a call to :func:`add_songs_to_playlist`, + since multiple copies of the same song are removed. For more control in this case, + get the playlist tracks with :func:`get_playlist_songs`, modify the list of tracks, + then use :func:`change_playlist` to push changes to the server. + """ + + playlist_tracks = self.get_playlist_songs(playlist_id) + sid_set = set(sids_to_match) + + matching_eids = [t["playlistEntryId"] + for t in playlist_tracks + if t["id"] in sid_set] + + if matching_eids: + #Call returns "sid_eid" strings. + sid_eids = self._remove_entries_from_playlist(playlist_id, + matching_eids) + return [s.split("_") for s in sid_eids] + else: + return [] + + @utils.accept_singleton(basestring, 2) + @utils.empty_arg_shortcircuit(position=2) + def _remove_entries_from_playlist(self, playlist_id, entry_ids_to_remove): + """Removes entries from a playlist. Returns a list of removed "sid_eid" strings. + + :param playlist_id: the playlist to be modified. + :param entry_ids: a list of entry ids, or a single entry id. + """ + + #GM requires the song ids in the call as well; find them. + playlist_tracks = self.get_playlist_songs(playlist_id) + remove_eid_set = set(entry_ids_to_remove) + + e_s_id_pairs = [(t["id"], t["playlistEntryId"]) + for t in playlist_tracks + if t["playlistEntryId"] in remove_eid_set] + + num_not_found = len(entry_ids_to_remove) - len(e_s_id_pairs) + if num_not_found > 0: + self.logger.warning("when removing, %d entry ids could not be found in playlist id %s", + num_not_found, playlist_id) + + #Unzip the pairs. + sids, eids = zip(*e_s_id_pairs) + + res = self._make_call(webclient.DeleteSongs, sids, playlist_id, eids) + + return res['deleteIds'] + + def search(self, query): + """Queries the server for songs and albums. + + **WARNING**: Google no longer uses this endpoint in their client; + it may stop working or be removed from gmusicapi without warning. + In addition, it is known to occasionally return unexpected results. + See `#114 + `__ + for more information. + + Instead of using this call, retrieve all tracks with :func:`get_all_songs` + and search them locally. `This gist + `__ has some examples of + simple linear-time searches. + + :param query: a string keyword to search with. Capitalization and punctuation are ignored. + + The results are returned in a dictionary, arranged by how they were found. + ``artist_hits`` and ``song_hits`` return a list of + :ref:`song dictionaries `, while ``album_hits`` entries + have a different structure. + + For example, a search on ``'cat'`` could return:: + + { + "album_hits": [ + { + "albumArtist": "The Cat Empire", + "albumName": "Cities: The Cat Empire Project", + "artistName": "The Cat Empire", + "imageUrl": "//ssl.gstatic.com/music/fe/[...].png" + # no more entries + }, + ], + "artist_hits": [ + { + "album": "Cinema", + "artist": "The Cat Empire", + "id": "c9214fc1-91fa-3bd2-b25d-693727a5f978", + "title": "Waiting" + # ... normal song dictionary + }, + ], + "song_hits": [ + { + "album": "Mandala", + "artist": "RX Bandits", + "id": "a7781438-8ec3-37ab-9c67-0ddb4115f60a", + "title": "Breakfast Cat", + # ... normal song dictionary + }, + ] + } + + """ + + res = self._make_call(webclient.Search, query)['results'] + + return {"album_hits": res["albums"], + "artist_hits": res["artists"], + "song_hits": res["songs"]} + + @utils.accept_singleton(basestring) + @utils.empty_arg_shortcircuit + @utils.enforce_id_param + def report_incorrect_match(self, song_ids): + """Equivalent to the 'Fix Incorrect Match' button, this requests re-uploading of songs. + Returns the song_ids provided. + + :param song_ids: a list of song ids to report, or a single song id. + + Note that if you uploaded a song through gmusicapi, it won't be reuploaded + automatically - this currently only works for songs uploaded with the Music Manager. + See issue `#89 `__. + + This should only be used on matched tracks (``song['type'] == 6``). + """ + + self._make_call(webclient.ReportBadSongMatch, song_ids) + + return song_ids + + @utils.accept_singleton(basestring) + @utils.empty_arg_shortcircuit + @utils.enforce_ids_param + def upload_album_art(self, song_ids, image_filepath): + """Uploads an image and sets it as the album art for songs. + + :param song_ids: a list of song ids, or a single song id. + :param image_filepath: filepath of the art to use. jpg and png are known to work. + + This function will *always* upload the provided image, even if it's already uploaded. + If the art is already uploaded and set for another song, copy over the + value of the ``'albumArtUrl'`` key using :func:`change_song_metadata` instead. + """ + + res = self._make_call(webclient.UploadImage, image_filepath) + url = res['imageUrl'] + + song_dicts = [dict((('id', id), ('albumArtUrl', url))) for id in song_ids] + + return self.change_song_metadata(song_dicts) diff --git a/gmusicapi/utils/utils.py b/gmusicapi/utils/utils.py index 0cf79b8a..c31ebf12 100644 --- a/gmusicapi/utils/utils.py +++ b/gmusicapi/utils/utils.py @@ -82,8 +82,11 @@ def __getattr__(self, name): try: if 'self' in frame.f_locals: f_self = frame.f_locals['self'] - if ((f_self.__module__ == 'gmusicapi.clients' and - type(f_self).__name__ in ('Musicmanager', 'Webclient'))): + + if (f_self is not None and + f_self.__module__ == 'gmusicapi.clients' and + type(f_self).__name__ in ('Musicmanager', + 'Webclient', 'Mobileclient')): logger = f_self.logger break finally: From 1a7932fb56bcea8d0fa02be6cdf263b1b2bdd4f1 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 3 Jul 2013 13:32:41 -0400 Subject: [PATCH 37/72] wip; integrating search --- gmusicapi/clients/mobileclient.py | 135 ++++++++++++++++++++++++++---- gmusicapi/test/run_tests.py | 6 +- gmusicapi/test/server_tests.py | 23 ++++- gmusicapi/utils/utils.py | 12 ++- 4 files changed, 155 insertions(+), 21 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index c806b2ec..dba0e240 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -12,10 +12,10 @@ class Mobileclient(_Base): Uploading is not supported by this client (use the :class:`Musicmanager` to upload). """ - def __init__(self, debug_logging=True): - self.session = session.Webclient() # TODO change name; now shared + def __init__(self, debug_logging=True, validate=True): + self.session = session.Webclient() - super(Mobileclient, self).__init__(self.__class__.__name__, debug_logging) + super(Mobileclient, self).__init__(self.__class__.__name__, debug_logging, validate) self.logout() def login(self, email, password): @@ -38,24 +38,131 @@ def login(self, email, password): return True - def search(self, query, max_results=5): - """Queries the server for songs and albums. + def search_all_access(self, query, max_results=5): + """Queries the server for All Access songs and albums. + Using this method without an All Access subscription will always result in + CallFailure being raised. :param query: a string keyword to search with. Capitalization and punctuation are ignored. :param max_results: Maximum number of items to be retrieved - The results are returned in a dictionary, arranged by how they were found. - ``artist_hits`` and ``song_hits`` return a list of - :ref:`song dictionaries `, while ``album_hits`` entries - have a different structure. + The results are returned in a dictionary, arranged by how they were found, eg:: + { + 'album_hits':[ + { + u'album':{ + u'albumArtRef':u'http://lh6.ggpht.com/...', + u'albumId':u'Bfr2onjv7g7tm4rzosewnnwxxyy', + u'artist':u'Amorphis', + u'artistId':[ + u'Apoecs6off3y6k4h5nvqqos4b5e' + ], + u'kind':u'sj#album', + u'name':u'Circle', + u'year':2013 + }, + u'best_result':True, + u'score':385.55609130859375, + u'type':u'3' + }, + { + u'album':{ + u'albumArtRef':u'http://lh3.ggpht.com/...', + u'albumArtist':u'Amorphis', + u'albumId':u'Bqzxfykbqcqmjjtdom7ukegaf2u', + u'artist':u'Amorphis', + u'artistId':[ + u'Apoecs6off3y6k4h5nvqqos4b5e' + ], + u'kind':u'sj#album', + u'name':u'Elegy', + u'year':1996 + }, + u'score':236.33485412597656, + u'type':u'3' + }, + ], + 'artist_hits':[ + { + u'artist':{ + u'artistArtRef':u'http://lh6.ggpht.com/...', + u'artistId':u'Apoecs6off3y6k4h5nvqqos4b5e', + u'kind':u'sj#artist', + u'name':u'Amorphis' + }, + u'score':237.86375427246094, + u'type':u'2' + } + ], + 'song_hits':[ + { + u'score':105.23198699951172, + u'track':{ + u'album':u'Skyforger', + u'albumArtRef':[ + { + u'url':u'http://lh4.ggpht.com/...' + } + ], + u'albumArtist':u'Amorphis', + u'albumAvailableForPurchase':True, + u'albumId':u'B5nc22xlcmdwi3zn5htkohstg44', + u'artist':u'Amorphis', + u'artistId':[ + u'Apoecs6off3y6k4h5nvqqos4b5e' + ], + u'discNumber':1, + u'durationMillis':u'253000', + u'estimatedSize':u'10137633', + u'kind':u'sj#track', + u'nid':u'Tn2ugrgkeinrrb2a4ji7khungoy', + u'playCount':1, + u'storeId':u'Tn2ugrgkeinrrb2a4ji7khungoy', + u'title':u'Silver Bride', + u'trackAvailableForPurchase':True, + u'trackNumber':2, + u'trackType':u'7' + }, + u'type':u'1' + }, + { + u'score':96.23717498779297, + u'track':{ + u'album':u'Magic And Mayhem - Tales From The Early Years', + u'albumArtRef':[ + { + u'url':u'http://lh4.ggpht.com/...' + } + ], + u'albumArtist':u'Amorphis', + u'albumAvailableForPurchase':True, + u'albumId':u'B7dplgr5h2jzzkcyrwhifgwl2v4', + u'artist':u'Amorphis', + u'artistId':[ + u'Apoecs6off3y6k4h5nvqqos4b5e' + ], + u'discNumber':1, + u'durationMillis':u'235000', + u'estimatedSize':u'9405159', + u'kind':u'sj#track', + u'nid':u'T4j5jxodzredqklxxhncsua5oba', + u'storeId':u'T4j5jxodzredqklxxhncsua5oba', + u'title':u'Black Winter Day', + u'trackAvailableForPurchase':True, + u'trackNumber':4, + u'trackType':u'7', + u'year':2010 + }, + u'type':u'1' + }, + ] + } """ - - #XXX provide an example res = self._make_call(mobileclient.Search, query, max_results)['entries'] - return {"album_hits": [hit for hit in res if hit['type'] == "3"], - "artist_hits": [hit for hit in res if hit['type'] == "2"], - "song_hits": [hit for hit in res if hit['type'] == "1"]} + return {'album_hits': [hit for hit in res if hit['type'] == '3'], + 'artist_hits': [hit for hit in res if hit['type'] == '2'], + 'song_hits': [hit for hit in res if hit['type'] == '1']} def get_artist(self, artistid, albums=True, top_tracks=0, rel_artist=0): """Retrieve artist data""" diff --git a/gmusicapi/test/run_tests.py b/gmusicapi/test/run_tests.py index 3342591c..01f79de0 100644 --- a/gmusicapi/test/run_tests.py +++ b/gmusicapi/test/run_tests.py @@ -11,7 +11,7 @@ from proboscis import TestProgram -from gmusicapi.clients import Webclient, Musicmanager, OAUTH_FILEPATH +from gmusicapi.clients import Webclient, Musicmanager, Mobileclient, OAUTH_FILEPATH from gmusicapi.protocol.musicmanager import credentials_from_refresh_token from gmusicapi.test import local_tests, server_tests # noqa from gmusicapi.test.utils import NoticeLogging @@ -87,7 +87,9 @@ def retrieve_auth(): def freeze_login_details(wc_kwargs, mm_kwargs): """Set the given kwargs to be the default for client login methods.""" for cls, kwargs in ((Musicmanager, mm_kwargs), - (Webclient, wc_kwargs)): + (Webclient, wc_kwargs), + (Mobileclient, wc_kwargs), + ): cls.login = MethodType( update_wrapper(partial(cls.login, **kwargs), cls.login), None, cls) diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 686220e5..556b6283 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -19,7 +19,7 @@ ) from proboscis import test, before_class, after_class, SkipTest -from gmusicapi.clients import Webclient, Musicmanager +from gmusicapi import Webclient, Musicmanager, Mobileclient, CallFailure #from gmusicapi.exceptions import NotLoggedIn from gmusicapi.protocol.metadata import md_expectations from gmusicapi.utils.utils import retry @@ -47,6 +47,9 @@ def login(self): self.mm = test_utils.new_test_client(Musicmanager) assert_true(self.mm.is_authenticated()) + self.mc = test_utils.new_test_client(Mobileclient) + assert_true(self.mc.is_authenticated()) + @after_class(always_run=True) def logout(self): if self.wc is None: @@ -57,6 +60,10 @@ def logout(self): raise SkipTest('did not create mm') assert_true(self.mm.logout()) + if self.mc is None: + raise SkipTest('did not create mc') + assert_true(self.mc.logout()) + # This next section is a bit odd: it nests playlist tests inside song tests. # The intuitition: starting from an empty library, you need to have @@ -160,6 +167,7 @@ def song_delete(self): song_test = test(groups=['song', 'song.exists'], depends_on=[song_create]) playlist_test = test(groups=['playlist', 'playlist.exists'], depends_on=[playlist_create]) + mc_test = test(groups=['mobile']) # Non-wonky tests resume down here. @@ -168,6 +176,19 @@ def get_registered_devices(self): # no logic; schema does verification self.wc.get_registered_devices() + #--------- + # MC tests + #--------- + @mc_test + def mc_search_aa(self): + if os.environ.get('GM_TEST_ALLACCESS') == 'TRUE': + res = self.mc.search_all_access('amorphis') + with Check() as check: + for hits in res.values(): + check.true(len(hits) > 0) + else: + self.assert_raises(CallFailure, self.mc.search_all_access, 'amorphis') + #----------- # Song tests #----------- diff --git a/gmusicapi/utils/utils.py b/gmusicapi/utils/utils.py index c31ebf12..f60f5d0a 100644 --- a/gmusicapi/utils/utils.py +++ b/gmusicapi/utils/utils.py @@ -67,6 +67,7 @@ class DynamicClientLogger(object): def __init__(self, caller_name): self.caller_name = caller_name + print 'dynamic logger created for', self.caller_name def __getattr__(self, name): # this isn't a totally foolproof way to proxy, but it's fine for @@ -83,16 +84,19 @@ def __getattr__(self, name): if 'self' in frame.f_locals: f_self = frame.f_locals['self'] - if (f_self is not None and - f_self.__module__ == 'gmusicapi.clients' and - type(f_self).__name__ in ('Musicmanager', - 'Webclient', 'Mobileclient')): + # can't import and check against classes; that causes an import cycle + if ((f_self is not None and + f_self.__module__.startswith('gmusicapi.clients') and + f_self.__class__.__name__ in ('Musicmanager', 'Webclient', + 'Mobileclient'))): logger = f_self.logger break finally: del frame # avoid circular references else: + # log to root logger. + # should this be stronger? There's no default root logger set up. stack = traceback.extract_stack() logger.info('could not locate client caller in stack:\n%s', '\n'.join(traceback.format_list(stack))) From cf233b6358b3a1aecd34ac425f5e5d95e8ad09eb Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 3 Jul 2013 14:21:02 -0400 Subject: [PATCH 38/72] aa search integrated --- gmusicapi/clients/mobileclient.py | 2 +- gmusicapi/protocol/mobileclient.py | 136 ++++++++++++++++------------- gmusicapi/test/server_tests.py | 50 ++++++++--- 3 files changed, 113 insertions(+), 75 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index dba0e240..1e554c1d 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -38,7 +38,7 @@ def login(self, email, password): return True - def search_all_access(self, query, max_results=5): + def search_all_access(self, query, max_results=50): """Queries the server for All Access songs and albums. Using this method without an All Access subscription will always result in CallFailure being raised. diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index c47a1974..8f1f47be 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -19,72 +19,82 @@ # URL for sj service sj_url = 'https://www.googleapis.com/sj/v1/' -# Data schema for sj service. Might be incomplete +# shared schemas sj_track = { - 'type':'object', - 'properties':{ - 'kind':{'type':'string'}, - 'title':{'type':'string'}, - 'artist':{'type':'string'}, - 'album':{'type':'string'}, - 'albumArtist':{'type':'string'}, - 'trackNumber':{'type':'integer'}, - 'durationMillis':{'type':'string'}, - 'albumArtRef':{'type':'array', 'items':{'type':'object', 'properties':{'url':{'type':'string'}}}}, - 'discNumber':{'type':'integer'}, - 'estimatedSize':{'type':'string'}, - 'trackType':{'type':'string'}, - 'storeId':{'type':'string'}, - 'albumId':{'type':'string'}, - 'artistId':{'type':'array', 'items':{'type':'string'}}, - 'nid':{'type':'string'}, - 'trackAvailableForPurchase':{'type':'boolean'}, - 'albumAvailableForPurchase':{'type':'boolean'}, - } + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'title': {'type': 'string'}, + 'artist': {'type': 'string'}, + 'album': {'type': 'string'}, + 'albumArtist': {'type': 'string'}, + 'trackNumber': {'type': 'integer'}, + 'durationMillis': {'type': 'string'}, + 'albumArtRef': {'type': 'array', + 'items': {'type': 'object', 'properties': {'url': {'type': 'string'}}}}, + 'discNumber': {'type': 'integer'}, + 'estimatedSize': {'type': 'string'}, + 'trackType': {'type': 'string'}, + 'storeId': {'type': 'string'}, + 'albumId': {'type': 'string'}, + 'artistId': {'type': 'array', 'items': {'type': 'string'}}, + 'nid': {'type': 'string'}, + 'trackAvailableForPurchase': {'type': 'boolean'}, + 'albumAvailableForPurchase': {'type': 'boolean'}, + 'playCount': {'type': 'integer', 'required': False}, + 'year': {'type': 'integer', 'required': False}, } +} sj_album = { - 'type':'object', - 'properties':{ - 'kind':{'type':'string'}, - 'name':{'type':'string'}, - 'albumArtist':{'type':'string'}, - 'albumArtRef':{'type':'string'}, - 'albumId':{'type':'string'}, - 'artist':{'type':'string'}, - 'artistId':{'type':'array', 'items':{'type':'string'}}, - 'year': {'type': 'integer'}, - 'tracks': {'type':'array', 'items':sj_track} - } + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'name': {'type': 'string'}, + 'albumArtist': {'type': 'string'}, + 'albumArtRef': {'type': 'string'}, + 'albumId': {'type': 'string'}, + 'artist': {'type': 'string'}, + 'artistId': {'type': 'array', 'items': {'type': 'string'}}, + 'year': {'type': 'integer'}, + 'tracks': {'type': 'array', 'items': sj_track, 'required': False} } +} sj_artist = { - 'type':'object', - 'properties':{ - 'kind':{'type':'string'}, - 'name':{'type':'string'}, - 'artistArtRef':{'type':'string'}, - 'artistId':{'type':'string'}, - 'albums:':{'type':'array', 'items':sj_album, 'required':False}, - 'topTracks':{'type':'array', 'items':sj_track, 'required':False}, - } + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'name': {'type': 'string'}, + 'artistArtRef': {'type': 'string'}, + 'artistId': {'type': 'string'}, + 'albums: ': {'type': 'array', 'items': sj_album, 'required': False}, + 'topTracks': {'type': 'array', 'items': sj_track, 'required': False}, } +} -sj_artist['related_artists']= {'type':'array', 'items':sj_artist, 'required':False} +sj_artist['related_artists'] = {'type': 'array', 'items': sj_artist, 'required': False} # Result definition may not contain any item. sj_result = { - "type":"object", - "properties":{ - 'score':{"type":"number"}, - 'artists':sj_artist, - 'album': sj_album, - 'track': sj_track - } + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'score': {'type': 'number'}, + 'type': {'type': 'string'}, + 'best_result': {'type': 'boolean', 'required': False}, + 'artist': sj_artist.copy(), + 'album': sj_album.copy(), + 'track': sj_track.copy(), } +} + +sj_result['properties']['artist']['required'] = False +sj_result['properties']['album']['required'] = False +sj_result['properties']['track']['required'] = False -sj_result['properties']['artists']['required']=False -sj_result['properties']['album']['required']=False -sj_result['properties']['track']['required']=False class McCall(Call): """Abstract base for mobile client calls.""" @@ -128,17 +138,18 @@ class Search(McCall): static_method = 'GET' _res_schema = { - "type": "object", - "properties": { - "kind":{"type":"string"}, - "entries": {'type':'array', 'items': sj_result} - }, - "additionalProperties": False + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'entries': {'type': 'array', 'items': sj_result} + }, } @staticmethod def dynamic_url(query, max_ret): - return sj_url + 'query?q=%s&max-results=%d' % (query, max_ret) + return sj_url + 'query?q=%s&max-results=%d' % (query, max_ret) + class GetArtist(McCall): static_method = 'GET' @@ -153,6 +164,7 @@ def dynamic_url(artistid, albums=True, top_tracks=0, rel_artist=0): ret += '&num-related-artists=%d' % rel_artist return ret + class GetAlbum(McCall): static_method = 'GET' _res_schema = sj_album @@ -164,6 +176,7 @@ def dynamic_url(albumid, tracks=True): ret += '&include-tracks=%r' % tracks return ret + class GetTrack(McCall): static_method = 'GET' _res_schema = sj_track @@ -174,11 +187,12 @@ def dynamic_url(trackid): ret += '&nid=%s' % trackid return ret + class GetStreamUrl(McCall): """Used to request a streaming link of a track.""" static_method = 'GET' - static_url = 'https://play.google.com/music/play' # note use of base_url, not service_url + static_url = 'https://play.google.com/music/play' required_auth = authtypes(sso=True) # no xt required diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 556b6283..40ee8e52 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -15,7 +15,7 @@ from proboscis.asserts import ( assert_true, assert_equal, assert_is_not_none, - assert_not_equal, Check + assert_not_equal, Check, assert_raises ) from proboscis import test, before_class, after_class, SkipTest @@ -164,6 +164,7 @@ def song_delete(self): # They won't work right with additional settings; if that's needed this # pattern should be factored out. + #TODO it'd be nice to have per-client test groups song_test = test(groups=['song', 'song.exists'], depends_on=[song_create]) playlist_test = test(groups=['playlist', 'playlist.exists'], depends_on=[playlist_create]) @@ -177,7 +178,7 @@ def get_registered_devices(self): self.wc.get_registered_devices() #--------- - # MC tests + # MC/AA tests #--------- @mc_test def mc_search_aa(self): @@ -187,7 +188,39 @@ def mc_search_aa(self): for hits in res.values(): check.true(len(hits) > 0) else: - self.assert_raises(CallFailure, self.mc.search_all_access, 'amorphis') + assert_raises(CallFailure, self.mc.search_all_access, 'amorphis') + + @mc_test + def mc_search_aa_with_limit(self): + if os.environ.get('GM_TEST_ALLACCESS') == 'TRUE': + res_unlimited = self.mc.search_all_access('cat empire') + res_5 = self.mc.search_all_access('cat empire', max_results=5) + + assert_equal(len(res_5['song_hits']), 5) + assert_true(len(res_unlimited['song_hits']) > len(res_5['song_hits'])) + + else: + raise SkipTest('AA testing not enabled') + + @test + def get_aa_stream_urls(self): + if os.environ.get('GM_TEST_ALLACCESS') == 'TRUE': + # that dumb little intro track on Conspiracy of One + urls = self.wc.get_stream_urls('Tqqufr34tuqojlvkolsrwdwx7pe') + + assert_true(len(urls) > 1) + #TODO test getting the stream + else: + raise SkipTest('AA testing not enabled') + + @test + def stream_aa_track(self): + if os.environ.get('GM_TEST_ALLACCESS') == 'TRUE': + # that dumb little intro track on Conspiracy of One + audio = self.wc.get_stream_audio('Tqqufr34tuqojlvkolsrwdwx7pe') + assert_is_not_none(audio) + else: + raise SkipTest('AA testing not enabled') #----------- # Song tests @@ -328,7 +361,7 @@ def assert_download(sid=self.song.sid): assert_download() @song_test - def get_normal_stream_urls(self): + def get_uploaded_stream_urls(self): urls = self.wc.get_stream_urls(self.song.sid) assert_equal(len(urls), 1) @@ -338,15 +371,6 @@ def get_normal_stream_urls(self): assert_is_not_none(url) assert_equal(url[:7], 'http://') - # TODO there must be a better way - if os.environ.get('GM_TEST_ALLACCESS') == 'TRUE': - @song_test - def get_aa_stream_urls(self): - # that dumb little intro track on Conspiracy of One - urls = self.wc.get_stream_urls('Tqqufr34tuqojlvkolsrwdwx7pe') - - assert_true(len(urls) > 1) - @song_test def upload_album_art(self): orig_md = self._assert_get_song(self.song.sid) From 9fd97a7232c66634912e14935dfe2f18664f8c18 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 3 Jul 2013 15:40:35 -0400 Subject: [PATCH 39/72] integrate get_artist_info --- gmusicapi/clients/mobileclient.py | 72 ++++++++++++++++++++++++++++-- gmusicapi/protocol/mobileclient.py | 32 ++++++++----- gmusicapi/protocol/shared.py | 2 + gmusicapi/test/server_tests.py | 26 +++++++++++ gmusicapi/utils/utils.py | 1 - 5 files changed, 117 insertions(+), 16 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 1e554c1d..cb97b4e5 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -40,6 +40,7 @@ def login(self, email, password): def search_all_access(self, query, max_results=50): """Queries the server for All Access songs and albums. + Using this method without an All Access subscription will always result in CallFailure being raised. @@ -164,9 +165,74 @@ def search_all_access(self, query, max_results=50): 'artist_hits': [hit for hit in res if hit['type'] == '2'], 'song_hits': [hit for hit in res if hit['type'] == '1']} - def get_artist(self, artistid, albums=True, top_tracks=0, rel_artist=0): - """Retrieve artist data""" - res = self._make_call(mobileclient.GetArtist, artistid, albums, top_tracks, rel_artist) + def get_artist_info(self, artist_id, include_albums=True, max_top_tracks=5, max_rel_artist=5): + """Retrieve details on an artist. + + Using this method without an All Access subscription will always result in + CallFailure being raised. + + Returns a dict, eg:: + { + u'albums':[ # only if include_albums is True + { + u'albumArtRef':u'http://lh6.ggpht.com/...', + u'albumArtist':u'Amorphis', + u'albumId':u'Bfr2onjv7g7tm4rzosewnnwxxyy', + u'artist':u'Amorphis', + u'artistId':[ + u'Apoecs6off3y6k4h5nvqqos4b5e' + ], + u'kind':u'sj#album', + u'name':u'Circle', + u'year':2013 + }, + ], + u'artistArtRef': u'http://lh6.ggpht.com/...', + u'artistId':u'Apoecs6off3y6k4h5nvqqos4b5e', + u'kind':u'sj#artist', + u'name':u'Amorphis', + u'related_artists':[ # only if max_rel_artists > 0 + { + u'artistArtRef': u'http://lh5.ggpht.com/...', + u'artistId':u'Aheqc7kveljtq7rptd7cy5gvk2q', + u'kind':u'sj#artist', + u'name':u'Dark Tranquillity' + } + ], + u'topTracks':[ # only if max_top_tracks > 0 + { + u'album':u'Skyforger', + u'albumArtRef':[ + { + u'url': u'http://lh4.ggpht.com/...' + } + ], + u'albumArtist':u'Amorphis', + u'albumAvailableForPurchase':True, + u'albumId':u'B5nc22xlcmdwi3zn5htkohstg44', + u'artist':u'Amorphis', + u'artistId':[ + u'Apoecs6off3y6k4h5nvqqos4b5e' + ], + u'discNumber':1, + u'durationMillis':u'253000', + u'estimatedSize':u'10137633', + u'kind':u'sj#track', + u'nid':u'Tn2ugrgkeinrrb2a4ji7khungoy', + u'playCount':1, + u'storeId':u'Tn2ugrgkeinrrb2a4ji7khungoy', + u'title':u'Silver Bride', + u'trackAvailableForPurchase':True, + u'trackNumber':2, + u'trackType':u'7' + } + ], + u'total_albums':21 + } + """ + + res = self._make_call(mobileclient.GetArtist, + artist_id, include_albums, max_top_tracks, max_rel_artist) return res def get_album(self, albumid, tracks=True): diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 8f1f47be..3f18b698 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -71,12 +71,18 @@ 'name': {'type': 'string'}, 'artistArtRef': {'type': 'string'}, 'artistId': {'type': 'string'}, - 'albums: ': {'type': 'array', 'items': sj_album, 'required': False}, + 'albums': {'type': 'array', 'items': sj_album, 'required': False}, 'topTracks': {'type': 'array', 'items': sj_track, 'required': False}, + 'total_albums': {'type': 'integer', 'required': False}, } } -sj_artist['related_artists'] = {'type': 'array', 'items': sj_artist, 'required': False} +sj_artist['properties']['related_artists'] = { + 'type': 'array', + 'items': sj_artist, # note the recursion + 'required': False +} + # Result definition may not contain any item. sj_result = { 'type': 'object', @@ -134,8 +140,8 @@ def parse_response(cls, response): class Search(McCall): """Search for All Access tracks.""" - static_method = 'GET' + static_url = sj_url + 'query' _res_schema = { 'type': 'object', @@ -147,22 +153,24 @@ class Search(McCall): } @staticmethod - def dynamic_url(query, max_ret): - return sj_url + 'query?q=%s&max-results=%d' % (query, max_ret) + def dynamic_params(query, max_results): + return {'q': query, 'max-results': max_results} class GetArtist(McCall): static_method = 'GET' + static_url = sj_url + 'fetchartist' + static_params = {'alt': 'json'} + _res_schema = sj_artist @staticmethod - def dynamic_url(artistid, albums=True, top_tracks=0, rel_artist=0): - ret = sj_url + 'fetchartist?alt=json' - ret += '&nid=%s' % artistid - ret += '&include-albums=%r' % albums - ret += '&num-top-tracks=%d' % top_tracks - ret += '&num-related-artists=%d' % rel_artist - return ret + def dynamic_params(artist_id, include_albums, num_top_tracks, num_rel_artist): + return {'nid': artist_id, + 'include-albums': include_albums, + 'num-top-tracks': num_top_tracks, + 'num-related-artists': num_rel_artist, + } class GetAlbum(McCall): diff --git a/gmusicapi/protocol/shared.py b/gmusicapi/protocol/shared.py index 054493ad..5ac13ff4 100644 --- a/gmusicapi/protocol/shared.py +++ b/gmusicapi/protocol/shared.py @@ -216,6 +216,7 @@ def perform(cls, session, validate, *args, **kwargs): err_msg = str(e) if cls.gets_logged: + err_msg += "\n(request args, kwargs: %r, %r)" % (args, kwargs) err_msg += "\n(response was: %r)" % response.content raise CallFailure(err_msg, call_name) @@ -226,6 +227,7 @@ def perform(cls, session, validate, *args, **kwargs): err_msg = ("the server's response could not be understood." " The call may still have succeeded, but it's unlikely.") if cls.gets_logged: + err_msg += "\n(request args, kwargs: %r, %r)" % (args, kwargs) err_msg += "\n(response was: %r)" % response.content log.exception("could not parse %s response: %r", call_name, response.content) else: diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 40ee8e52..b461697c 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -202,6 +202,32 @@ def mc_search_aa_with_limit(self): else: raise SkipTest('AA testing not enabled') + @mc_test + def mc_artist_info(self): + if os.environ.get('GM_TEST_ALLACCESS') == 'TRUE': + aid = 'Apoecs6off3y6k4h5nvqqos4b5e' # amorphis + optional_keys = set(('albums', 'topTracks', 'related_artists')) + + include_all_res = self.mc.get_artist_info(aid, include_albums=True, + max_top_tracks=1, max_rel_artist=1) + + no_albums_res = self.mc.get_artist_info(aid, include_albums=False) + no_rel_res = self.mc.get_artist_info(aid, max_rel_artist=0) + no_tracks_res = self.mc.get_artist_info(aid, max_top_tracks=0) + + with Check() as check: + check.true(set(include_all_res.keys()) & optional_keys == optional_keys) + + check.true(set(no_albums_res.keys()) & optional_keys == + optional_keys - set(['albums'])) + check.true(set(no_rel_res.keys()) & optional_keys == + optional_keys - set(['related_artists'])) + check.true(set(no_tracks_res.keys()) & optional_keys == + optional_keys - set(['topTracks'])) + + else: + assert_raises(CallFailure, self.mc.search_all_access, 'amorphis') + @test def get_aa_stream_urls(self): if os.environ.get('GM_TEST_ALLACCESS') == 'TRUE': diff --git a/gmusicapi/utils/utils.py b/gmusicapi/utils/utils.py index f60f5d0a..3b776d08 100644 --- a/gmusicapi/utils/utils.py +++ b/gmusicapi/utils/utils.py @@ -67,7 +67,6 @@ class DynamicClientLogger(object): def __init__(self, caller_name): self.caller_name = caller_name - print 'dynamic logger created for', self.caller_name def __getattr__(self, name): # this isn't a totally foolproof way to proxy, but it's fine for From bb065debebee8f4dcf059d4960b34ffbfe66af13 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 3 Jul 2013 15:43:30 -0400 Subject: [PATCH 40/72] remove webclient duplication --- gmusicapi/clients/mobileclient.py | 57 +----------------------------- gmusicapi/protocol/mobileclient.py | 56 ----------------------------- 2 files changed, 1 insertion(+), 112 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index cb97b4e5..6c38c462 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -1,5 +1,3 @@ -from urlparse import urlparse, parse_qsl - from gmusicapi.clients.shared import _Base from gmusicapi.protocol import mobileclient from gmusicapi import session @@ -235,6 +233,7 @@ def get_artist_info(self, artist_id, include_albums=True, max_top_tracks=5, max_ artist_id, include_albums, max_top_tracks, max_rel_artist) return res + #TODO below here def get_album(self, albumid, tracks=True): """Retrieve artist data""" res = self._make_call(mobileclient.GetAlbum, albumid, tracks) @@ -244,57 +243,3 @@ def get_track(self, trackid): """Retrieve artist data""" res = self._make_call(mobileclient.GetTrack, trackid) return res - - def get_stream_audio(self, song_id): - """Returns a bytestring containing mp3 audio for this song. - - :param song_id: a single song id - """ - - urls = self.get_stream_urls(song_id) - - if len(urls) == 1: - return self.session._rsession.get(urls[0]).content - - # AA tracks are separated into multiple files - # the url contains the range of each file to be used - - range_pairs = [[int(s) for s in val.split('-')] - for url in urls - for key, val in parse_qsl(urlparse(url)[4]) - if key == 'range'] - - stream_pieces = [] - prev_end = 0 - - for url, (start, end) in zip(urls, range_pairs): - audio = self.session._rsession.get(url).content - stream_pieces.append(audio[prev_end - start:]) - - prev_end = end + 1 - - return ''.join(stream_pieces) - - def get_stream_urls(self, song_id): - """Returns a url that points to a streamable version of this song. - - :param song_id: a single song id. - - While acquiring the url requires authentication, retreiving the - url contents does not. - - However, there are limitation as to how the stream url can be used: - * the url expires after about a minute - * only one IP can be streaming music at once. - Other attempts will get an http 403 with - ``X-Rejected-Reason: ANOTHER_STREAM_BEING_PLAYED``. - - *This is only intended for streaming*. The streamed audio does not contain metadata. - Use :func:`get_song_download_info` to download complete files with metadata. - """ - res = self._make_call(mobileclient.GetStreamUrl, song_id) - - try: - return res['url'] - except KeyError: - return res['urls'] diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 3f18b698..7afb9442 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -3,12 +3,7 @@ """Calls made by the web client.""" -import binascii -import hmac -import random -import string import sys -from hashlib import sha1 import validictory @@ -194,54 +189,3 @@ def dynamic_url(trackid): ret = sj_url + 'fetchtrack?alt=json' ret += '&nid=%s' % trackid return ret - - -class GetStreamUrl(McCall): - """Used to request a streaming link of a track.""" - - static_method = 'GET' - static_url = 'https://play.google.com/music/play' - - required_auth = authtypes(sso=True) # no xt required - - _res_schema = { - "type": "object", - "properties": { - "url": {"type": "string", "required": False}, - "urls": {"type": "array", "required": False} - }, - "additionalProperties": False - } - - @staticmethod - def dynamic_params(song_id): - - # https://github.com/simon-weber/Unofficial-Google-Music-API/issues/137 - # there are three cases when streaming: - # | track type | guid songid? | slt/sig needed? | - # user-uploaded yes no - # AA track in library yes yes - # AA track not in library no yes - - # without the track['type'] field we can't tell between 1 and 2, but - # include slt/sig anyway; the server ignores the extra params. - key = '27f7313e-f75d-445a-ac99-56386a5fe879' - salt = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(12)) - sig = binascii.b2a_base64(hmac.new(key, (song_id + salt), sha1).digest())[:-1] - urlsafe_b64_trans = string.maketrans("+/=", "-_.") - sig = sig.translate(urlsafe_b64_trans) - - params = { - 'u': 0, - 'pt': 'e', - 'slt': salt, - 'sig': sig - } - - # TODO match guid instead, should be more robust - if song_id[0] == 'T': - # all access - params['mjck'] = song_id - else: - params['songid'] = song_id - return params From 7792c04b6039ba1d319a181d708dda67c5621c5b Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sun, 7 Jul 2013 18:35:38 -0400 Subject: [PATCH 41/72] wip get_all_songs --- gmusicapi/clients/mobileclient.py | 5 +++++ gmusicapi/protocol/mobileclient.py | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 6c38c462..b0c24254 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -36,6 +36,11 @@ def login(self, email, password): return True + def get_all_songs(self): + """Lists the songs in our library.""" + res = self._make_call(mobileclient.GetLibraryTracks) + return res + def search_all_access(self, query, max_results=50): """Queries the server for All Access songs and albums. diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 7afb9442..3a2001a1 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -152,6 +152,15 @@ def dynamic_params(query, max_results): return {'q': query, 'max-results': max_results} +class GetLibraryTracks(McCall): + """List tracks in the library.""" + static_method = 'GET' + static_url = sj_url + 'tracks' + + _res_schema = {} + + +#TODO below here class GetArtist(McCall): static_method = 'GET' static_url = sj_url + 'fetchartist' From f7f8997519c74e085e3704ec310b620b71d8eef0 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sun, 7 Jul 2013 19:17:37 -0400 Subject: [PATCH 42/72] enable non-AA CI; rename envargs to fit key size --- .travis.yml | 8 ++++---- gmusicapi/test/run_tests.py | 13 ++++++++----- gmusicapi/test/server_tests.py | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 554dc327..2f2cd846 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ branches: - master env: global: - - GM_UP_ID="E9:40:01:0E:51:7A" - - GM_UP_NAME="Travis-CI (gmusicapi)" - - GM_TEST_ALLACCESS=TRUE + - GM_N="Travis-CI (gmusicapi)" matrix: - - secure: "UAXrAYzrYRVFWSRldd5o9NrreexCsXdBna/kcKLlAQ8ygRahpqP7qXX8qiNU\nVhz0kdgBxgS84AEE3H/30o9v2IDWqmJCI5OqMfH1o5Pm+CelBt+8FHu3SMjm\nNvNcm/Vmip7WCSx7P2FfOf8HboSH/kVXuF0iPlOdozrTR1wPUq8=" + # first is AA, second is not + - secure: "De4nJrP5O5uL0kWPYCDmTdsiDTHv7Vuvkv02XdwPOFoGG6sOL1svrJhL3Z+G\n3eMFy3DE69mybFf1gZEb9XdDY3LoOX0oAgkHigh+memrvgw/QzaJYN553kmF\n31ose52SFNmtFZH95cwQg1qkSZZsMXlmPxRq510MVl6SA7j6Pi4=" + - secure: "aK3s024vOcybRx+Wq7/9tSd9XGJn0LBAuejTWxIiUul0I8sgt/ry7Pc8r0Gv\nXU3a9P/0LOUAYu1hS3/hH3u2oVIgXoEotrm2wC6r8N2R8xE7ZhYVl1Nf0fOK\ns0jy1ebIU+s/yHQmuboe61FT3lfiFRxQm6yJu5/G/D77O4HGuQQ=" diff --git a/gmusicapi/test/run_tests.py b/gmusicapi/test/run_tests.py index 3342591c..98ad36b2 100644 --- a/gmusicapi/test/run_tests.py +++ b/gmusicapi/test/run_tests.py @@ -18,15 +18,18 @@ EnvArg = namedtuple('EnvArg', 'envarg kwarg description') +# these names needed to be compressed to fit everything into the travisci key size. +# there's also GM_A, which when set (to anything) states that we are testing on +# and All Access account. wc_envargs = ( - EnvArg('GM_USER', 'email', 'WC user. If not present, user will be prompted.'), - EnvArg('GM_PASS', 'password', 'WC password. If not present, user will be prompted.'), + EnvArg('GM_U', 'email', 'WC user. If not present, user will be prompted.'), + EnvArg('GM_P', 'password', 'WC password. If not present, user will be prompted.'), ) mm_envargs = ( - EnvArg('GM_OAUTH', 'oauth_credentials', 'MM refresh token. Defaults to MM.login default.'), - EnvArg('GM_UP_ID', 'uploader_id', 'MM uploader id. Defaults to MM.login default.'), - EnvArg('GM_UP_NAME', 'uploader_name', 'MM uploader name. Default to MM.login default.'), + EnvArg('GM_O', 'oauth_credentials', 'MM refresh token. Defaults to MM.login default.'), + EnvArg('GM_I', 'uploader_id', 'MM uploader id. Defaults to MM.login default.'), + EnvArg('GM_N', 'uploader_name', 'MM uploader name. Default to MM.login default.'), ) diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 686220e5..9547dc1a 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -318,7 +318,7 @@ def get_normal_stream_urls(self): assert_equal(url[:7], 'http://') # TODO there must be a better way - if os.environ.get('GM_TEST_ALLACCESS') == 'TRUE': + if 'GM_A' in os.environ: @song_test def get_aa_stream_urls(self): # that dumb little intro track on Conspiracy of One From 8e7400e9d75c73bcd308b5378db2c747823a8168 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 8 Jul 2013 10:48:55 -0400 Subject: [PATCH 43/72] artistArtRef not required --- gmusicapi/protocol/mobileclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 3a2001a1..bc1eecd0 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -64,7 +64,7 @@ 'properties': { 'kind': {'type': 'string'}, 'name': {'type': 'string'}, - 'artistArtRef': {'type': 'string'}, + 'artistArtRef': {'type': 'string', 'required': False}, 'artistId': {'type': 'string'}, 'albums': {'type': 'array', 'items': sj_album, 'required': False}, 'topTracks': {'type': 'array', 'items': sj_track, 'required': False}, From d7ecc332f24d0de069752e9488c08645110a1ce0 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 8 Jul 2013 12:48:27 -0400 Subject: [PATCH 44/72] add/test mc.get_all_songs --- gmusicapi/clients/mobileclient.py | 35 +++++++++++++++++++++---- gmusicapi/protocol/mobileclient.py | 42 +++++++++++++++++++++++++++--- gmusicapi/test/server_tests.py | 11 +++++++- 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index b0c24254..49df1e11 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -17,7 +17,7 @@ def __init__(self, debug_logging=True, validate=True): self.logout() def login(self, email, password): - """Authenticates the webclient. + """Authenticates the Mobileclient. Returns ``True`` on success, ``False`` on failure. :param email: eg ``'test@gmail.com'`` or just ``'test'``. @@ -36,10 +36,35 @@ def login(self, email, password): return True - def get_all_songs(self): - """Lists the songs in our library.""" - res = self._make_call(mobileclient.GetLibraryTracks) - return res + def get_all_songs(self, incremental=False): + """TODO + + :param incremental: if True, return a generator that yields lists + of at most 1000 tracks + as they are retrieved from the server. This can be useful for + presenting a loading bar to a user. + """ + if not incremental: + # slight optimization; can get all tracks at once with mc + res = self._make_call(mobileclient.GetLibraryTracks, max_results=20000) + return res['data']['items'] + + # otherwise, return a generator + return self._get_all_songs_incremental() + + def _get_all_songs_incremental(self): + """Return a generator of lists of tracks.""" + + get_next_chunk = True + lib_chunk = {'nextPageToken': None} + + while get_next_chunk: + lib_chunk = self._make_call(mobileclient.GetLibraryTracks, + start_token=lib_chunk['nextPageToken']) + + yield lib_chunk['data']['items'] # list of songs of the chunk + + get_next_chunk = 'nextPageToken' in lib_chunk def search_all_access(self, query, max_results=50): """Queries the server for All Access songs and albums. diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index bc1eecd0..c5190fc1 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -3,10 +3,12 @@ """Calls made by the web client.""" +import copy import sys import validictory +from gmusicapi.compat import json from gmusicapi.exceptions import CallFailure, ValidationException from gmusicapi.protocol.shared import Call, authtypes from gmusicapi.utils import utils @@ -102,6 +104,8 @@ class McCall(Call): required_auth = authtypes(xt=False, sso=True) + static_headers = {'Content-Type': 'application/json'} + #validictory schema for the response _res_schema = utils.NotImplementedField @@ -154,10 +158,42 @@ def dynamic_params(query, max_results): class GetLibraryTracks(McCall): """List tracks in the library.""" - static_method = 'GET' - static_url = sj_url + 'tracks' + static_method = 'POST' + static_url = sj_url + 'trackfeed' + + _res_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'nextPageToken': {'type': 'string', 'required': False}, + 'data': {'type': 'object', + 'items': {'type': 'array', 'items': sj_track}, + }, + }, + } + + @staticmethod + def dynamic_data(start_token=None, max_results=None): + """ + :param start_token: nextPageToken from a previous response + :param max_results: a positive int; if not provided, server defaults to 1000 + """ + data = {} - _res_schema = {} + if start_token is not None: + data['start-token'] = start_token + + if max_results is not None: + data['max-results'] = str(max_results) + + return json.dumps(data) + + @staticmethod + def filter_response(msg): + filtered = copy.deepcopy(msg) + filtered['data']['items'] = ["<%s songs>" % len(filtered['data'].get('items', []))] + return filtered #TODO below here diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index e657bb9b..bd0d4220 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -174,12 +174,13 @@ def song_delete(self): @test def get_registered_devices(self): - # no logic; schema does verification + # no logic; just checking schema self.wc.get_registered_devices() #--------- # MC/AA tests #--------- + @mc_test def mc_search_aa(self): if 'GM_A' in os.environ: @@ -281,6 +282,10 @@ def list_songs_wc(self): def list_songs_mm(self): self._assert_get_song(self.song.sid, self.mm) + @song_test + def list_songs_mc(self): + self._assert_get_song(self.song.sid, self.mc) + @staticmethod def _list_songs_incrementally(client): lib_chunk_gen = client.get_all_songs(incremental=True) @@ -297,6 +302,10 @@ def list_songs_incrementally_wc(self): def list_songs_incrementally_mm(self): self._list_songs_incrementally(self.mm) + @mc_test + def list_songs_incrementall_mc(self): + self._list_songs_incrementally(self.mc) + @song_test def change_metadata(self): orig_md = self._assert_get_song(self.song.sid) From 9f2ea86ee43a9c77e8f3e82f8b5433bf2dc267d9 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 8 Jul 2013 16:33:42 -0400 Subject: [PATCH 45/72] add mc.get_stream_url. need to setup testing device --- gmusicapi/clients/mobileclient.py | 32 ++++++++++ gmusicapi/protocol/mobileclient.py | 94 +++++++++++++++++++++++++----- gmusicapi/protocol/shared.py | 6 +- gmusicapi/protocol/webclient.py | 6 +- gmusicapi/session.py | 5 +- gmusicapi/test/local_tests.py | 8 +++ 6 files changed, 127 insertions(+), 24 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 49df1e11..eed9193e 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -66,6 +66,38 @@ def _get_all_songs_incremental(self): get_next_chunk = 'nextPageToken' in lib_chunk + def get_stream_url(self, song_id, device_id): + """Returns a url that will point to an mp3 file. + + :param song_id: a single song id + :param device_id: a registered Android device id, as a string. + If you have already used Google Music on a mobile device, + :func:`Webclient.get_registered_devices + ` will provide + at least one working id. + + Note that this id must be from a mobile device; a registered computer + id (as a MAC address) will not be accepted. + + Providing an invalid id will result in an http 403. + + When handling the resulting url, keep in mind that: + * you will likely need to handle redirects + * the url expires after a minute + * only one IP can be streaming music at once. + This can result in an http 403 with + ``X-Rejected-Reason: ANOTHER_STREAM_BEING_PLAYED``. + + The file will not contain metadata. + Use :func:`Webclient.get_song_download_info + ` + or :func:`Musicmanager.download_song + ` + to download files with metadata. + """ + + return self._make_call(mobileclient.GetStreamUrl, song_id, device_id) + def search_all_access(self, query, max_results=50): """Queries the server for All Access songs and albums. diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index c5190fc1..1e6b0e16 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -1,15 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -"""Calls made by the web client.""" +"""Calls made by the mobile client.""" +import base64 import copy +from hashlib import sha1 +import hmac import sys +import time + import validictory from gmusicapi.compat import json -from gmusicapi.exceptions import CallFailure, ValidationException +from gmusicapi.exceptions import ValidationException from gmusicapi.protocol.shared import Call, authtypes from gmusicapi.utils import utils @@ -104,8 +109,6 @@ class McCall(Call): required_auth = authtypes(xt=False, sso=True) - static_headers = {'Content-Type': 'application/json'} - #validictory schema for the response _res_schema = utils.NotImplementedField @@ -120,17 +123,16 @@ def validate(cls, response, msg): @classmethod def check_success(cls, response, msg): - #Failed responses always have a success=False key. - #Some successful responses do not have a success=True key, however. - #TODO remove utils.call_succeeded - - if 'success' in msg and not msg['success']: - raise CallFailure( - "the server reported failure. This is usually" - " caused by bad arguments, but can also happen if requests" - " are made too quickly (eg creating a playlist then" - " modifying it before the server has created it)", - cls.__name__) + #TODO not sure if this is still valid for mc + pass + + #if 'success' in msg and not msg['success']: + # raise CallFailure( + # "the server reported failure. This is usually" + # " caused by bad arguments, but can also happen if requests" + # " are made too quickly (eg creating a playlist then" + # " modifying it before the server has created it)", + # cls.__name__) @classmethod def parse_response(cls, response): @@ -160,6 +162,7 @@ class GetLibraryTracks(McCall): """List tracks in the library.""" static_method = 'POST' static_url = sj_url + 'trackfeed' + static_headers = {'Content-Type': 'application/json'} _res_schema = { 'type': 'object', @@ -196,6 +199,67 @@ def filter_response(msg): return filtered +class GetStreamUrl(McCall): + static_method = 'GET' + static_url = 'https://android.clients.google.com/music/mplay' + static_verify = False + + # this call will redirect to the mp3 + static_allow_redirects = False + + _s1 = base64.b64decode('VzeC4H4h+T2f0VI180nVX8x+Mb5HiTtGnKgH52Otj8ZCGDz9jRW' + 'yHb6QXK0JskSiOgzQfwTY5xgLLSdUSreaLVMsVVWfxfa8Rw==') + _s2 = base64.b64decode('ZAPnhUkYwQ6y5DdQxWThbvhJHN8msQ1rqJw0ggKdufQjelrKuiG' + 'GJI30aswkgCWTDyHkTGK9ynlqTkJ5L4CiGGUabGeo8M6JTQ==') + + # bitwise and of _s1 and _s2 ascii, converted to string + _key = ''.join([chr(ord(c1) ^ ord(c2)) for (c1, c2) in zip(_s1, _s2)]) + + @classmethod + def get_signature(cls, song_id, salt=None): + """Return a (sig, salt) pair for url signing.""" + + if salt is None: + salt = str(int(time.time() * 1000)) + + mac = hmac.new(cls._key, song_id, sha1) + mac.update(salt) + sig = base64.urlsafe_b64encode(mac.digest())[:-1] + + return sig, salt + + @staticmethod + def dynamic_headers(song_id, device_id): + return {'X-Device-ID': device_id} + + @classmethod + def dynamic_params(cls, song_id, device_id): + sig, salt = cls.get_signature(song_id) + + #TODO which of these should get exposed? + params = {'opt': 'hi', + 'net': 'wifi', + 'pt': 'e', + 'slt': salt, + 'sig': sig, + } + if song_id[0] == 'T': + # all access + params['mjck'] = song_id + else: + params['songid'] = song_id + + return params + + @staticmethod + def parse_response(response): + return response.headers['location'] # ie where we were redirected + + @classmethod + def validate(cls, response, msg): + pass + + #TODO below here class GetArtist(McCall): static_method = 'GET' diff --git a/gmusicapi/protocol/shared.py b/gmusicapi/protocol/shared.py index 5ac13ff4..6e7c8414 100644 --- a/gmusicapi/protocol/shared.py +++ b/gmusicapi/protocol/shared.py @@ -46,7 +46,7 @@ def __new__(cls, name, bases, dct): new_cls = super(BuildRequestMeta, cls).__new__(cls, name, bases, dct) merge_keys = ('headers', 'params') - all_keys = ('method', 'url', 'files', 'data', 'verify') + merge_keys + all_keys = ('method', 'url', 'files', 'data', 'verify', 'allow_redirects') + merge_keys config = {} # stores key: val for static or f(*args, **kwargs) -> val for dyn dyn = lambda key: 'dynamic_' + key @@ -216,7 +216,7 @@ def perform(cls, session, validate, *args, **kwargs): err_msg = str(e) if cls.gets_logged: - err_msg += "\n(request args, kwargs: %r, %r)" % (args, kwargs) + err_msg += "\n(requests kwargs: %r)" % (req_kwargs) err_msg += "\n(response was: %r)" % response.content raise CallFailure(err_msg, call_name) @@ -227,7 +227,7 @@ def perform(cls, session, validate, *args, **kwargs): err_msg = ("the server's response could not be understood." " The call may still have succeeded, but it's unlikely.") if cls.gets_logged: - err_msg += "\n(request args, kwargs: %r, %r)" % (args, kwargs) + err_msg += "\n(requests kwargs: %r)" % (req_kwargs) err_msg += "\n(response was: %r)" % response.content log.exception("could not parse %s response: %r", call_name, response.content) else: diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 2ffe0ec0..54588089 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -2,7 +2,7 @@ """Calls made by the web client.""" -import binascii +import base64 import copy import hmac import random @@ -527,9 +527,7 @@ def dynamic_params(song_id): # include slt/sig anyway; the server ignores the extra params. key = '27f7313e-f75d-445a-ac99-56386a5fe879' salt = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(12)) - sig = binascii.b2a_base64(hmac.new(key, (song_id + salt), sha1).digest())[:-1] - urlsafe_b64_trans = string.maketrans("+/=", "-_.") - sig = sig.translate(urlsafe_b64_trans) + sig = base64.urlsafe_b64encode(hmac.new(key, (song_id + salt), sha1).digest())[:-1] params = { 'u': 0, diff --git a/gmusicapi/session.py b/gmusicapi/session.py index 9987d88c..d9fe743e 100644 --- a/gmusicapi/session.py +++ b/gmusicapi/session.py @@ -104,14 +104,15 @@ def login(self, email, password, *args, **kwargs): def _send_with_auth(self, req_kwargs, desired_auth, rsession): if desired_auth.sso: - req_kwargs['headers'] = req_kwargs.get('headers', {}) + req_kwargs.setdefault('headers', {}) # does this ever expire? would we have to perform clientlogin again? req_kwargs['headers']['Authorization'] = \ 'GoogleLogin auth=' + self._authtoken if desired_auth.xt: - req_kwargs['params'] = req_kwargs.get('params', {}) + req_kwargs.setdefault('params', {}) + req_kwargs['params'].update({'u': 0, 'xt': rsession.cookies['xt']}) return rsession.request(**req_kwargs) diff --git a/gmusicapi/test/local_tests.py b/gmusicapi/test/local_tests.py index 81734d2a..584a32f8 100644 --- a/gmusicapi/test/local_tests.py +++ b/gmusicapi/test/local_tests.py @@ -18,6 +18,7 @@ from gmusicapi.clients import Webclient, Musicmanager from gmusicapi.exceptions import AlreadyLoggedIn # ,NotLoggedIn from gmusicapi.protocol.shared import authtypes +from gmusicapi.protocol import mobileclient from gmusicapi.utils import utils @@ -168,6 +169,13 @@ def authtypes_factory_args(): assert_false(auth.xt) +@test +def mc_url_signing(): + sig, _ = mobileclient.GetStreamUrls.get_signature("Tdr6kq3xznv5kdsphyojox6dtoq", + "1373247112519") + assert_equal(sig, "gua1gInBdaVo7_dSwF9y0kodua0") + + ## # utils ## From 3a71762b0961e051fd254abb2dd878c091c9f59d Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 8 Jul 2013 17:43:15 -0400 Subject: [PATCH 46/72] add mc.get_all_songs docs --- gmusicapi/clients/mobileclient.py | 47 ++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index eed9193e..4571fd87 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -37,13 +37,58 @@ def login(self, email, password): return True def get_all_songs(self, incremental=False): - """TODO + """Returns a list of dictionaries that each represent a song, eg:: :param incremental: if True, return a generator that yields lists of at most 1000 tracks as they are retrieved from the server. This can be useful for presenting a loading bar to a user. + + Here is an example song dictionary:: + { + u'comment':u'', + u'rating':u'0', + u'albumArtRef':[ + { + u'url': u'http://lh6.ggpht.com/...' + } + ], + u'artistId':[ + u'Aod62yyj3u3xsjtooghh2glwsdi' + ], + u'composer':u'', + u'year':2011, + u'creationTimestamp':u'1330879409467830', + u'id':u'5924d75a-931c-30ed-8790-f7fce8943c85', + u'album':u'Heritage ', + u'totalDiscCount':0, + u'title':u'Haxprocess', + u'recentTimestamp':u'1372040508935000', + u'albumArtist':u'', + u'trackNumber':6, + u'discNumber':0, + u'deleted':False, + u'storeId':u'Txsffypukmmeg3iwl3w5a5s3vzy', + u'nid':u'Txsffypukmmeg3iwl3w5a5s3vzy', + u'totalTrackCount':10, + u'estimatedSize':u'17229205', + u'albumId':u'Bdkf6ywxmrhflvtasnayxlkgpcm', + u'beatsPerMinute':0, + u'genre':u'Progressive Metal', + u'playCount':7, + u'artistArtRef':[ + { + u'url': u'http://lh3.ggpht.com/...' + } + ], + u'kind':u'sj#track', + u'artist':u'Opeth', + u'lastModifiedTimestamp':u'1330881158830924', + u'clientId':u'+eGFGTbiyMktbPuvB5MfsA', + u'durationMillis':u'418000' + } """ + if not incremental: # slight optimization; can get all tracks at once with mc res = self._make_call(mobileclient.GetLibraryTracks, max_results=20000) From 8e42063411584655af44c2c3b4957605bbaff218 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Thu, 11 Jul 2013 12:27:00 -0400 Subject: [PATCH 47/72] clarify get_all_songs and all access tracks --- gmusicapi/clients.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gmusicapi/clients.py b/gmusicapi/clients.py index db9fe553..45a7587e 100644 --- a/gmusicapi/clients.py +++ b/gmusicapi/clients.py @@ -796,6 +796,11 @@ def get_all_songs(self, incremental=False): of at most 2500 :ref:`song dictionaries ` as they are retrieved from the server. This can be useful for presenting a loading bar to a user. + + All Access tracks that have been added to the library will be included + in the list. They can be distinguished by their ``'type'`` key, + which will be 7. Refer to the :ref:`song dictionary ` + docs for more information. """ to_return = self._get_all_songs() From ebed4b27f876d4da905696b4fe83f007ba33dd47 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Fri, 12 Jul 2013 21:53:12 -0400 Subject: [PATCH 48/72] add metadata: curatedByUser and curationSuggested [close #143] --- gmusicapi/protocol/metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gmusicapi/protocol/metadata.py b/gmusicapi/protocol/metadata.py index de42c102..7a9b03e6 100644 --- a/gmusicapi/protocol/metadata.py +++ b/gmusicapi/protocol/metadata.py @@ -133,6 +133,8 @@ def get_schema(self): ('subjectToCuration', 'boolean', 'meaning unknown.'), ('matchedId', 'string', 'meaning unknown; related to scan and match?'), + ('curatedByUser', 'boolean', 'meaning unknown'), + ('curationSuggested', 'boolean', 'meaning unknown'), ) ] + [ Expectation(name, type_str, mutable=False, optional=True, explanation=explain) @@ -142,7 +144,7 @@ def get_schema(self): ('reuploading', 'boolean', 'scan-and-match reupload in progress.'), ('albumMatchedId', 'string', 'id of matching album in the Play Store?'), ('pending', 'boolean', 'unsure; server processing (eg for store match) pending?'), - ('url', 'string', 'meaning unknown.'), + ('url', 'string', 'meaning unknown.'), ('bitrate', 'integer', "bitrate in kilobytes/second (eg 320)."), ('playlistEntryId', 'string', 'identifies position in the context of a playlist.'), ('albumArtUrl', 'string', "if present, the url of an image for this song's album art."), From df24d08007d382ccf8680bc4cc39d60234a76acf Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sat, 13 Jul 2013 15:32:54 -0400 Subject: [PATCH 49/72] add mc.get_all_playlists --- .travis.yml | 1 + gmusicapi/clients/mobileclient.py | 80 ++++++++++++++++++++++-------- gmusicapi/protocol/mobileclient.py | 64 ++++++++++++++++++++++++ gmusicapi/test/server_tests.py | 12 ++++- 4 files changed, 134 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f2cd846..b7718a63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ branches: only: - develop - master + - mobileclient env: global: - GM_N="Travis-CI (gmusicapi)" diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 4571fd87..0e02aaba 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -36,8 +36,10 @@ def login(self, email, password): return True + #TODO expose max-results for get_all_* + def get_all_songs(self, incremental=False): - """Returns a list of dictionaries that each represent a song, eg:: + """Returns a list of dictionaries that each represent a song. :param incremental: if True, return a generator that yields lists of at most 1000 tracks @@ -89,27 +91,7 @@ def get_all_songs(self, incremental=False): } """ - if not incremental: - # slight optimization; can get all tracks at once with mc - res = self._make_call(mobileclient.GetLibraryTracks, max_results=20000) - return res['data']['items'] - - # otherwise, return a generator - return self._get_all_songs_incremental() - - def _get_all_songs_incremental(self): - """Return a generator of lists of tracks.""" - - get_next_chunk = True - lib_chunk = {'nextPageToken': None} - - while get_next_chunk: - lib_chunk = self._make_call(mobileclient.GetLibraryTracks, - start_token=lib_chunk['nextPageToken']) - - yield lib_chunk['data']['items'] # list of songs of the chunk - - get_next_chunk = 'nextPageToken' in lib_chunk + return self._get_all_items(mobileclient.GetLibraryTracks, incremental) def get_stream_url(self, song_id, device_id): """Returns a url that will point to an mp3 file. @@ -143,6 +125,33 @@ def get_stream_url(self, song_id, device_id): return self._make_call(mobileclient.GetStreamUrl, song_id, device_id) + def get_all_playlists(self, incremental=False): + """Returns a list of dictionaries that each represent a playlist. + + :param incremental: if True, return a generator that yields lists + of at most 1000 playlists + as they are retrieved from the server. This can be useful for + presenting a loading bar to a user. + + Here is an example playlist dictionary:: + { + u 'kind': u 'sj#playlist', + u 'name': u 'Something Mix', + u 'deleted': False, + u 'type': u 'MAGIC', # if not present, playlist is user-created + u 'lastModifiedTimestamp': u '1325458766483033', + u 'recentTimestamp': u '1325458766479000', + u 'shareToken': u '', + u 'ownerProfilePhotoUrl': u 'http://lh3.googleusercontent.com/...', + u 'ownerName': u 'Simon Weber', + u 'accessControlled': False, # something to do with shared playlists? + u 'creationTimestamp': u '1325285553626172', + u 'id': u '3d72c9b5-baad-4ff7-815d-cdef717e5d61' + }, + """ + + return self._get_all_items(mobileclient.ListPlaylists, incremental) + def search_all_access(self, query, max_results=50): """Queries the server for All Access songs and albums. @@ -340,6 +349,33 @@ def get_artist_info(self, artist_id, include_albums=True, max_top_tracks=5, max_ artist_id, include_albums, max_top_tracks, max_rel_artist) return res + def _get_all_items(self, call, incremental): + """ + :param call: protocol.McCall + :param incremental: bool + """ + if not incremental: + # slight optimization; can get all items at once + res = self._make_call(call, max_results=20000) + return res['data']['items'] + + # otherwise, return a generator + return self._get_all_items_incremental(call) + + def _get_all_items_incremental(self, call): + """Return a generator of lists of tracks.""" + + get_next_chunk = True + lib_chunk = {'nextPageToken': None} + + while get_next_chunk: + lib_chunk = self._make_call(call, + start_token=lib_chunk['nextPageToken']) + + yield lib_chunk['data']['items'] # list of songs of the chunk + + get_next_chunk = 'nextPageToken' in lib_chunk + #TODO below here def get_album(self, albumid, tracks=True): """Retrieve artist data""" diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 1e6b0e16..98340f79 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -49,6 +49,29 @@ } } +sj_playlist = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'name': {'type': 'string'}, + 'deleted': {'type': 'boolean'}, + 'type': {'type': 'string', 'required': False}, + 'lastModifiedTimestamp': {'type': 'string'}, + 'recentTimestamp': {'type': 'string'}, + 'shareToken': {'type': 'string'}, + 'ownerProfilePhotoUrl': {'type': 'string'}, + 'ownerName': {'type': 'string'}, + 'accessControlled': {'type': 'boolean'}, + 'creationTimestamp': {'type': 'string'}, + 'id': {'type': 'string'}, + 'albumArtRef': {'type': 'array', + 'items': {'type': 'object', 'properties': {'url': {'type': 'string'}}}, + 'required': False, + }, + } +} + sj_album = { 'type': 'object', 'additionalProperties': False, @@ -260,6 +283,47 @@ def validate(cls, response, msg): pass +class ListPlaylists(McCall): + """List tracks in the library.""" + static_method = 'POST' + static_url = sj_url + 'playlistfeed' + static_headers = {'Content-Type': 'application/json'} + + _res_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'nextPageToken': {'type': 'string', 'required': False}, + 'data': {'type': 'object', + 'items': {'type': 'array', 'items': sj_playlist}, + }, + }, + } + + @staticmethod + def dynamic_data(start_token=None, max_results=None): + """ + :param start_token: nextPageToken from a previous response + :param max_results: a positive int; if not provided, server defaults to 1000 + """ + data = {} + + if start_token is not None: + data['start-token'] = start_token + + if max_results is not None: + data['max-results'] = str(max_results) + + return json.dumps(data) + + @staticmethod + def filter_response(msg): + filtered = copy.deepcopy(msg) + filtered['data']['items'] = ["<%s playlists>" % len(filtered['data'].get('items', []))] + return filtered + + #TODO below here class GetArtist(McCall): static_method = 'GET' diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index bd0d4220..a4542a41 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -121,6 +121,8 @@ def assert_song_exists(sid): @test(depends_on=[song_create], runs_after_groups=['song.exists']) def playlist_create(self): + raise SkipTest('playlist create broken') + self.playlist_id = self.wc.create_playlist(TEST_PLAYLIST_NAME) # like song_create, retry until the playlist appears @@ -181,6 +183,14 @@ def get_registered_devices(self): # MC/AA tests #--------- + @mc_test + def list_playlists_mc(self): + lib_chunk_gen = self.mc.get_all_playlists(incremental=True) + assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) + + assert_equal([p for chunk in lib_chunk_gen for p in chunk], + self.mc.get_all_playlists(incremental=False)) + @mc_test def mc_search_aa(self): if 'GM_A' in os.environ: @@ -303,7 +313,7 @@ def list_songs_incrementally_mm(self): self._list_songs_incrementally(self.mm) @mc_test - def list_songs_incrementall_mc(self): + def list_songs_incrementally_mc(self): self._list_songs_incrementally(self.mc) @song_test From 55df271739e67e31883f83e0c5f9db559876a6e2 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sat, 13 Jul 2013 15:38:02 -0400 Subject: [PATCH 50/72] update readme with mobileclient status --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index d837d50b..afea4b16 100644 --- a/README.rst +++ b/README.rst @@ -49,6 +49,11 @@ Status and updates .. image:: https://travis-ci.org/simon-weber/Unofficial-Google-Music-API.png?branch=develop :target: https://travis-ci.org/simon-weber/Unofficial-Google-Music-API +The Webclient interface has gotten horrible to maintain lately, so I'm currently working on +switching the the Android app api. This will provide easy All Access support and easier +maintainability going forward. Expect this release before August -- you can follow along +`here `__. + Version 1.2.0 fixes a bug that fixes uploader_id formatting from a mac address. This change may cause another machine to be registered - you can safely remove the old machine (it's the one without the version in the name). From c5403b74e3a0195932f017a8931227ff9cde8d92 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sat, 13 Jul 2013 15:46:14 -0400 Subject: [PATCH 51/72] fix streamurl typo --- gmusicapi/test/local_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gmusicapi/test/local_tests.py b/gmusicapi/test/local_tests.py index 584a32f8..aee7c534 100644 --- a/gmusicapi/test/local_tests.py +++ b/gmusicapi/test/local_tests.py @@ -171,8 +171,8 @@ def authtypes_factory_args(): @test def mc_url_signing(): - sig, _ = mobileclient.GetStreamUrls.get_signature("Tdr6kq3xznv5kdsphyojox6dtoq", - "1373247112519") + sig, _ = mobileclient.GetStreamUrl.get_signature("Tdr6kq3xznv5kdsphyojox6dtoq", + "1373247112519") assert_equal(sig, "gua1gInBdaVo7_dSwF9y0kodua0") From 86c7ecfc7b93e87cf5ab5ced0f7600d268965774 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sat, 13 Jul 2013 22:52:30 -0400 Subject: [PATCH 52/72] add mc.get_all_stations --- gmusicapi/clients/mobileclient.py | 50 ++++++++++++++--- gmusicapi/protocol/mobileclient.py | 90 +++++++++++++++++++++++++++++- gmusicapi/test/server_tests.py | 10 ++++ gmusicapi/utils/utils.py | 9 +++ 4 files changed, 151 insertions(+), 8 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 0e02aaba..debcfe6b 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -147,11 +147,42 @@ def get_all_playlists(self, incremental=False): u 'accessControlled': False, # something to do with shared playlists? u 'creationTimestamp': u '1325285553626172', u 'id': u '3d72c9b5-baad-4ff7-815d-cdef717e5d61' - }, + } """ return self._get_all_items(mobileclient.ListPlaylists, incremental) + def get_all_stations(self, updated_after=None, incremental=False): + """Returns a list of dictionaries that each represent a radio station. + + :param updated_after: a datetime.datetime; defaults to epoch + :param incremental: if True, return a generator that yields lists + of at most 1000 stations + as they are retrieved from the server. This can be useful for + presenting a loading bar to a user. + + Here is an example station dictionary:: + { + u 'imageUrl': u 'http://lh6.ggpht.com/...', + u 'kind': u 'sj#radioStation', + u 'name': u 'station', + u 'deleted': False, + u 'lastModifiedTimestamp': u '1370796487455005', + u 'recentTimestamp': u '1370796487454000', + u 'clientId': u 'c2639bf4-af24-4e4f-ab37-855fc89d15a1', + u 'seed': + { + u 'kind': u 'sj#radioSeed', + u 'trackLockerId': u '7df3aadd-9a18-3dc1-b92e-a7cf7619da7e' + # possible keys: + # albumId, artistId, genreId, trackId, trackLockerId + }, + u 'id': u '69f1bfce-308a-313e-9ed2-e50abe33a25d' + }, + """ + return self._get_all_items(mobileclient.ListStations, incremental, + updated_after=updated_after) + def search_all_access(self, query, max_results=50): """Queries the server for All Access songs and albums. @@ -349,28 +380,33 @@ def get_artist_info(self, artist_id, include_albums=True, max_top_tracks=5, max_ artist_id, include_albums, max_top_tracks, max_rel_artist) return res - def _get_all_items(self, call, incremental): + def _get_all_items(self, call, incremental, **kwargs): """ :param call: protocol.McCall :param incremental: bool + + kwargs are passed to the call. """ if not incremental: # slight optimization; can get all items at once - res = self._make_call(call, max_results=20000) + res = self._make_call(call, max_results=20000, **kwargs) return res['data']['items'] # otherwise, return a generator - return self._get_all_items_incremental(call) + return self._get_all_items_incremental(call, **kwargs) + + def _get_all_items_incremental(self, call, **kwargs): + """Return a generator of lists of tracks. - def _get_all_items_incremental(self, call): - """Return a generator of lists of tracks.""" + kwargs are passed to the call.""" get_next_chunk = True lib_chunk = {'nextPageToken': None} while get_next_chunk: lib_chunk = self._make_call(call, - start_token=lib_chunk['nextPageToken']) + start_token=lib_chunk['nextPageToken'], + **kwargs) yield lib_chunk['data']['items'] # list of songs of the chunk diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 98340f79..47c73e54 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -126,6 +126,38 @@ sj_result['properties']['album']['required'] = False sj_result['properties']['track']['required'] = False +sj_station_seed = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + # one of these will be present + 'albumId': {'type': 'string', 'required': False}, + 'artistId': {'type': 'string', 'required': False}, + 'genreId': {'type': 'string', 'required': False}, + 'trackId': {'type': 'string', 'required': False}, + 'trackLockerId': {'type': 'string', 'required': False}, + } +} + +sj_station = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'imageUrl': {'type': 'string'}, + 'kind': {'type': 'string'}, + 'name': {'type': 'string'}, + 'deleted': {'type': 'boolean'}, + 'lastModifiedTimestamp': {'type': 'string'}, + 'recentTimestamp': {'type': 'string'}, + 'clientId': {'type': 'string'}, + 'seed': sj_station_seed, + 'id': {'type': 'string'}, + 'description': {'type': 'string', 'required': False}, + 'tracks': {'type': 'array', 'required': False, 'items': sj_track}, + } +} + class McCall(Call): """Abstract base for mobile client calls.""" @@ -284,7 +316,6 @@ def validate(cls, response, msg): class ListPlaylists(McCall): - """List tracks in the library.""" static_method = 'POST' static_url = sj_url + 'playlistfeed' static_headers = {'Content-Type': 'application/json'} @@ -324,6 +355,63 @@ def filter_response(msg): return filtered +class ListStations(McCall): + static_method = 'POST' + static_url = sj_url + 'radio/station' + static_headers = {'Content-Type': 'application/json'} + static_params = {'alt': 'json'} + + _res_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'nextPageToken': {'type': 'string', 'required': False}, + 'data': {'type': 'object', + 'items': {'type': 'array', 'items': sj_station}, + }, + }, + } + + @staticmethod + def dynamic_params(updated_after=None, start_token=None, max_results=None): + """ + :param updated_after: datetime.datetime; defaults to epoch + """ + + if updated_after is None: + microseconds = 0 + else: + microseconds = utils.datetime_to_microseconds(updated_after) + + return {'updated-min': microseconds} + + @staticmethod + def dynamic_data(updated_after=None, start_token=None, max_results=None): + """ + :param updated_after: ignored + :param start_token: nextPageToken from a previous response + :param max_results: a positive int; if not provided, server defaults to 1000 + + args/kwargs are ignored. + """ + data = {} + + if start_token is not None: + data['start-token'] = start_token + + if max_results is not None: + data['max-results'] = str(max_results) + + return json.dumps(data) + + @staticmethod + def filter_response(msg): + filtered = copy.deepcopy(msg) + filtered['data']['items'] = ["<%s stations>" % len(filtered['data'].get('items', []))] + return filtered + + #TODO below here class GetArtist(McCall): static_method = 'GET' diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index a4542a41..08b34ed3 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -183,6 +183,16 @@ def get_registered_devices(self): # MC/AA tests #--------- + #TODO clean all this up + + @mc_test + def list_stations_mc(self): + lib_chunk_gen = self.mc.get_all_stations(incremental=True) + assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) + + assert_equal([p for chunk in lib_chunk_gen for p in chunk], + self.mc.get_all_stations(incremental=False)) + @mc_test def list_playlists_mc(self): lib_chunk_gen = self.mc.get_all_playlists(incremental=True) diff --git a/gmusicapi/utils/utils.py b/gmusicapi/utils/utils.py index 3b776d08..95c95b5d 100644 --- a/gmusicapi/utils/utils.py +++ b/gmusicapi/utils/utils.py @@ -106,6 +106,15 @@ def __getattr__(self, name): log = DynamicClientLogger(__name__) +def datetime_to_microseconds(dt): + """Return microseconds since epoch, as an int. + + :param dt: a datetime.datetime + + """ + return int(time.mktime(dt.timetuple()) * 1000000) + + def is_valid_mac(mac_string): """Return True if mac_string is of form eg '00:11:22:33:AA:BB'. From c7f8511550ec6a34c5bd9b07c25bea96a3640989 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sat, 13 Jul 2013 23:17:48 -0400 Subject: [PATCH 53/72] factor out to McListCall --- gmusicapi/clients/mobileclient.py | 2 +- gmusicapi/protocol/mobileclient.py | 188 +++++++++++------------------ 2 files changed, 73 insertions(+), 117 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index debcfe6b..73c6bfc2 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -91,7 +91,7 @@ def get_all_songs(self, incremental=False): } """ - return self._get_all_items(mobileclient.GetLibraryTracks, incremental) + return self._get_all_items(mobileclient.ListTracks, incremental) def get_stream_url(self, song_id, device_id): """Returns a url that will point to an mp3 file. diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 47c73e54..0aca6cd3 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -194,30 +194,14 @@ def parse_response(cls, response): return cls._parse_json(response.text) -class Search(McCall): - """Search for All Access tracks.""" - static_method = 'GET' - static_url = sj_url + 'query' - - _res_schema = { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'kind': {'type': 'string'}, - 'entries': {'type': 'array', 'items': sj_result} - }, - } - - @staticmethod - def dynamic_params(query, max_results): - return {'q': query, 'max-results': max_results} - +class McListCall(McCall): + """Abc for calls that list a resource.""" + # concrete classes provide: + item_schema = utils.NotImplementedField + filter_text = utils.NotImplementedField -class GetLibraryTracks(McCall): - """List tracks in the library.""" - static_method = 'POST' - static_url = sj_url + 'trackfeed' static_headers = {'Content-Type': 'application/json'} + static_params = {'alt': 'json'} _res_schema = { 'type': 'object', @@ -226,14 +210,29 @@ class GetLibraryTracks(McCall): 'kind': {'type': 'string'}, 'nextPageToken': {'type': 'string', 'required': False}, 'data': {'type': 'object', - 'items': {'type': 'array', 'items': sj_track}, + 'items': {'type': 'array', 'items': item_schema}, + 'required': False, }, }, } - @staticmethod - def dynamic_data(start_token=None, max_results=None): + @classmethod + def dynamic_params(cls, updated_after=None, start_token=None, max_results=None): + """ + :param updated_after: datetime.datetime; defaults to epoch + """ + + if updated_after is None: + microseconds = 0 + else: + microseconds = utils.datetime_to_microseconds(updated_after) + + return {'updated-min': microseconds} + + @classmethod + def dynamic_data(cls, updated_after=None, start_token=None, max_results=None): """ + :param updated_after: ignored :param start_token: nextPageToken from a previous response :param max_results: a positive int; if not provided, server defaults to 1000 """ @@ -247,13 +246,51 @@ def dynamic_data(start_token=None, max_results=None): return json.dumps(data) - @staticmethod - def filter_response(msg): + @classmethod + def parse_response(cls, response): + # empty results don't include the data key + # make sure it's always there + res = cls._parse_json(response.text) + if 'data' not in res: + res['data'] = {'items': []} + + return res + + @classmethod + def filter_response(cls, msg): filtered = copy.deepcopy(msg) - filtered['data']['items'] = ["<%s songs>" % len(filtered['data'].get('items', []))] + filtered['data']['items'] = ["<%s %s>" % (len(filtered['data']['items']), + cls.filter_text)] return filtered +class Search(McCall): + """Search for All Access tracks.""" + static_method = 'GET' + static_url = sj_url + 'query' + + _res_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'entries': {'type': 'array', 'items': sj_result} + }, + } + + @staticmethod + def dynamic_params(query, max_results): + return {'q': query, 'max-results': max_results} + + +class ListTracks(McListCall): + item_schema = sj_track + filter_text = 'tracks' + + static_method = 'POST' + static_url = sj_url + 'trackfeed' + + class GetStreamUrl(McCall): static_method = 'GET' static_url = 'https://android.clients.google.com/music/mplay' @@ -315,101 +352,20 @@ def validate(cls, response, msg): pass -class ListPlaylists(McCall): +class ListPlaylists(McListCall): + item_schema = sj_playlist + filter_text = 'playlists' + static_method = 'POST' static_url = sj_url + 'playlistfeed' - static_headers = {'Content-Type': 'application/json'} - _res_schema = { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'kind': {'type': 'string'}, - 'nextPageToken': {'type': 'string', 'required': False}, - 'data': {'type': 'object', - 'items': {'type': 'array', 'items': sj_playlist}, - }, - }, - } - - @staticmethod - def dynamic_data(start_token=None, max_results=None): - """ - :param start_token: nextPageToken from a previous response - :param max_results: a positive int; if not provided, server defaults to 1000 - """ - data = {} - - if start_token is not None: - data['start-token'] = start_token - - if max_results is not None: - data['max-results'] = str(max_results) - return json.dumps(data) +class ListStations(McListCall): + item_schema = sj_station + filter_text = 'stations' - @staticmethod - def filter_response(msg): - filtered = copy.deepcopy(msg) - filtered['data']['items'] = ["<%s playlists>" % len(filtered['data'].get('items', []))] - return filtered - - -class ListStations(McCall): static_method = 'POST' static_url = sj_url + 'radio/station' - static_headers = {'Content-Type': 'application/json'} - static_params = {'alt': 'json'} - - _res_schema = { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'kind': {'type': 'string'}, - 'nextPageToken': {'type': 'string', 'required': False}, - 'data': {'type': 'object', - 'items': {'type': 'array', 'items': sj_station}, - }, - }, - } - - @staticmethod - def dynamic_params(updated_after=None, start_token=None, max_results=None): - """ - :param updated_after: datetime.datetime; defaults to epoch - """ - - if updated_after is None: - microseconds = 0 - else: - microseconds = utils.datetime_to_microseconds(updated_after) - - return {'updated-min': microseconds} - - @staticmethod - def dynamic_data(updated_after=None, start_token=None, max_results=None): - """ - :param updated_after: ignored - :param start_token: nextPageToken from a previous response - :param max_results: a positive int; if not provided, server defaults to 1000 - - args/kwargs are ignored. - """ - data = {} - - if start_token is not None: - data['start-token'] = start_token - - if max_results is not None: - data['max-results'] = str(max_results) - - return json.dumps(data) - - @staticmethod - def filter_response(msg): - filtered = copy.deepcopy(msg) - filtered['data']['items'] = ["<%s stations>" % len(filtered['data'].get('items', []))] - return filtered #TODO below here From 43c94396efed303f2f09ff0c74499b341256678d Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sun, 14 Jul 2013 16:54:15 -0400 Subject: [PATCH 54/72] mc.{add_aa_song, delete_songs} tested --- gmusicapi/clients/mobileclient.py | 45 ++- gmusicapi/protocol/mobileclient.py | 111 ++++- gmusicapi/protocol/shared.py | 4 +- gmusicapi/test/server_tests.py | 630 ++++++++++++++++------------- gmusicapi/test/utils.py | 3 + 5 files changed, 488 insertions(+), 305 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 73c6bfc2..fb75434f 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -1,6 +1,7 @@ +from gmusicapi import session from gmusicapi.clients.shared import _Base from gmusicapi.protocol import mobileclient -from gmusicapi import session +from gmusicapi.utils import utils class Mobileclient(_Base): @@ -36,7 +37,7 @@ def login(self, email, password): return True - #TODO expose max-results for get_all_* + #TODO expose max-results, updated_after, etc for list operations def get_all_songs(self, incremental=False): """Returns a list of dictionaries that each represent a song. @@ -93,6 +94,39 @@ def get_all_songs(self, incremental=False): return self._get_all_items(mobileclient.ListTracks, incremental) + def add_aa_track(self, aa_song_id): + """Adds an All Access track to the library, + returning the library track id. + + :param aa_song_id: All Access song id + """ + #TODO is there a way to do this on multiple tracks at once? + # problem is with gathering aa track info + + aa_track_info = self.get_track(aa_song_id) + + mutate_call = mobileclient.BatchMutateTracks + add_mutation = mutate_call.build_track_add(aa_track_info) + res = self._make_call(mutate_call, [add_mutation]) + + return res['mutate_response'][0]['id'] + + @utils.accept_singleton(basestring) + @utils.empty_arg_shortcircuit + @utils.enforce_ids_param + def delete_songs(self, library_song_ids): + """Deletes songs from the library. + Returns a list of deleted song ids. + + :param song_ids: a list of song ids, or a single song id. + """ + + mutate_call = mobileclient.BatchMutateTracks + del_mutations = mutate_call.build_track_deletes(library_song_ids) + res = self._make_call(mutate_call, del_mutations) + + return [d['id'] for d in res['mutate_response']] + def get_stream_url(self, song_id, device_id): """Returns a url that will point to an mp3 file. @@ -419,6 +453,9 @@ def get_album(self, albumid, tracks=True): return res def get_track(self, trackid): - """Retrieve artist data""" - res = self._make_call(mobileclient.GetTrack, trackid) + """Retrieve information about a store track. + + TODO does this work on library tracks? + """ + res = self._make_call(mobileclient.GetStoreTrack, trackid) return res diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 0aca6cd3..9588e084 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -264,6 +264,49 @@ def filter_response(cls, msg): return filtered +class McBatchMutateCall(McCall): + """Abc for batch mutation calls.""" + + static_headers = {'Content-Type': 'application/json'} + static_params = {'alt': 'json'} + + _res_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'mutate_response': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'id': {'type': 'string'}, + 'client_id': {'type': 'string', 'blank': True, + 'required': False}, + 'response_code': {'type': 'string'}, + }, + }, + } + }, + } + + @staticmethod + def dynamic_data(mutations): + """ + :param mutations: list of mutation dictionaries + """ + + return json.dumps({'mutations': mutations}) + + @staticmethod + def check_success(response, msg): + if not all([d.get('response_code', None) == 'OK' + for d in msg.get('mutate_response', [])]): + raise ValidationException + + if 'error' in msg: + raise ValidationException + + class Search(McCall): """Search for All Access tracks.""" static_method = 'GET' @@ -368,6 +411,63 @@ class ListStations(McListCall): static_url = sj_url + 'radio/station' +class BatchMutateTracks(McBatchMutateCall): + static_method = 'POST' + static_url = sj_url + 'trackbatch' + + # utility functions to build the mutation dicts + @staticmethod + def build_track_deletes(track_ids): + """ + :param track_id + """ + return [{'delete': id} for id in track_ids] + + @staticmethod + def build_track_add(store_track_info): + """ + :param store_track_info: sj_track + """ + track_dict = copy.deepcopy(store_track_info) + for key in ('kind', 'trackAvailableForPurchase', + 'albumAvailableForPurchase', 'albumArtRef', + 'artistId', + ): + del track_dict[key] + + for key, default in { + 'playCount': 0, + 'rating': '0', + 'genre': '', + 'lastModifiedTimestamp': '0', + 'deleted': False, + 'beatsPerMinute': -1, + 'composer': '', + 'creationTimestamp': '-1', + 'totalDiscCount': 0, + }.items(): + track_dict.setdefault(key, default) + + # TODO unsure about this + track_dict['trackType'] = 8 + + return {'create': track_dict} + + +class GetStoreTrack(McCall): + #TODO I think this should accept library ids, too + static_method = 'GET' + static_url = sj_url + 'fetchtrack' + static_headers = {'Content-Type': 'application/json'} + static_params = {'alt': 'json'} + + _res_schema = sj_track + + @staticmethod + def dynamic_params(track_id): + return {'nid': track_id} + + #TODO below here class GetArtist(McCall): static_method = 'GET' @@ -395,14 +495,3 @@ def dynamic_url(albumid, tracks=True): ret += '&nid=%s' % albumid ret += '&include-tracks=%r' % tracks return ret - - -class GetTrack(McCall): - static_method = 'GET' - _res_schema = sj_track - - @staticmethod - def dynamic_url(trackid): - ret = sj_url + 'fetchtrack?alt=json' - ret += '&nid=%s' % trackid - return ret diff --git a/gmusicapi/protocol/shared.py b/gmusicapi/protocol/shared.py index 6e7c8414..888a2d19 100644 --- a/gmusicapi/protocol/shared.py +++ b/gmusicapi/protocol/shared.py @@ -260,8 +260,8 @@ def perform(cls, session, validate, *args, **kwargs): " If you can recreate this error with the most recent code" " please [create an issue](http://goo.gl/qbAW8) that includes" " the above ValidationException" - " and the following raw response:\n%r\n" - "\nA traceback follows:\n") % raw_response + " and the following request/response:\n%r\n\n%r\n" + "\nA traceback follows:\n") % (req_kwargs, raw_response) log.exception(err_msg) diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 08b34ed3..176b878a 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -13,6 +13,7 @@ import re import types +from decorator import decorator from proboscis.asserts import ( assert_true, assert_equal, assert_is_not_none, assert_not_equal, Check, assert_raises @@ -31,12 +32,28 @@ TestSong = namedtuple('TestSong', 'sid title artist album') +def test_all_access_features(): + return 'GM_A' in os.environ + + +@decorator +def all_access(f, *args, **kwargs): + """Declare a test to only be run if All Access testing is enabled.""" + if test_all_access_features(): + return f(*args, **kwargs) + else: + raise SkipTest('All Access testing disabled') + + @test(groups=['server']) class UpauthTests(object): - #These are set on the instance in create_song/playlist. + # set on the instance in login wc = None # webclient mm = None # musicmanager - song = None + mc = None # mobileclient + + #These are set on the instance in create_song/playlist. + songs = None # [TestSong] playlist_id = None @before_class @@ -82,42 +99,62 @@ def logout(self): # Singleton groups are used to ease code ordering restraints. # Suggestions to improve any of this are welcome! + @retry + def assert_songs_state(self, sids, present): + """ + Assert presence/absence of sids and return a list of + TestSongs found. + + :param sids: list of song ids + :param present: if True verify songs are present; False the opposite + """ + + library = self.wc.get_all_songs() + + found = [s for s in library if s['id'] in sids] + + expected_len = len(sids) + if not present: + expected_len = 0 + + assert_equal(len(found), expected_len) + + return [TestSong(s['id'], s['title'], s['artist'], s['album']) + for s in found] + @test def song_create(self): + # This can create more than one song: one through uploading, one through + # adding an AA track to the library. + fname = test_utils.small_mp3 uploaded, matched, not_uploaded = self.mm.upload(fname) + sids = [] + if len(not_uploaded) == 1 and 'ALREADY_EXISTS' in not_uploaded[fname]: # If a previous test went wrong, the track might be there already. #TODO This build will fail because of the warning - is that what we want? assert_equal(matched, {}) assert_equal(uploaded, {}) - sid = re.search(r'\(.*\)', not_uploaded[fname]).group().strip('()') + sids.append(re.search(r'\(.*\)', not_uploaded[fname]).group().strip('()')) else: # Otherwise, it should have been uploaded normally. assert_equal(not_uploaded, {}) assert_equal(matched, {}) assert_equal(uploaded.keys(), [fname]) - sid = uploaded[fname] + sids.append(uploaded[fname]) + + if test_all_access_features(): + sids.append(self.mc.add_aa_track(test_utils.aa_song_id)) # we test get_all_songs here so that we can assume the existance # of the song for future tests (the servers take time to sync an upload) - @retry - def assert_song_exists(sid): - songs = self.wc.get_all_songs() - - found = [s for s in songs if s['id'] == sid] or None - - assert_is_not_none(found) - assert_equal(len(found), 1) - s = found[0] - return TestSong(s['id'], s['title'], s['artist'], s['album']) - - self.song = assert_song_exists(sid) + self.songs = self.assert_songs_state(sids, present=True) @test(depends_on=[song_create], runs_after_groups=['song.exists']) def playlist_create(self): @@ -155,362 +192,379 @@ def playlist_delete(self): runs_after_groups=["song.exists"], always_run=True) def song_delete(self): - if self.song is None: - raise SkipTest('did not store self.song') - - res = self.wc.delete_songs(self.song.sid) - - assert_equal(res, [self.song.sid]) - - # These decorators just prevent setting groups and depends_on over and over. - # They won't work right with additional settings; if that's needed this - # pattern should be factored out. - - #TODO it'd be nice to have per-client test groups + if self.songs is None: + raise SkipTest('did not store self.songs') + + # split deletion between wc and mc + # mc is the only to run if AA testing not enabled + with Check() as check: + for i, testsong in enumerate(self.songs): + if i % 2 == 0: + res = self.mc.delete_songs(testsong.sid) + else: + res = self.wc.delete_songs(testsong.sid) + check.equal(res, [testsong.sid]) + + self.assert_songs_state([s.sid for s in self.songs], present=False) + + ## These decorators just prevent setting groups and depends_on over and over. + ## They won't work right with additional settings; if that's needed this + ## pattern should be factored out. + + ##TODO it'd be nice to have per-client test groups song_test = test(groups=['song', 'song.exists'], depends_on=[song_create]) playlist_test = test(groups=['playlist', 'playlist.exists'], depends_on=[playlist_create]) - mc_test = test(groups=['mobile']) + #mc_test = test(groups=['mobile']) - # Non-wonky tests resume down here. + @song_test + def st(self): + raise SkipTest('remove this stub test') - @test - def get_registered_devices(self): - # no logic; just checking schema - self.wc.get_registered_devices() - - #--------- - # MC/AA tests - #--------- - - #TODO clean all this up - - @mc_test - def list_stations_mc(self): - lib_chunk_gen = self.mc.get_all_stations(incremental=True) - assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) - - assert_equal([p for chunk in lib_chunk_gen for p in chunk], - self.mc.get_all_stations(incremental=False)) - - @mc_test - def list_playlists_mc(self): - lib_chunk_gen = self.mc.get_all_playlists(incremental=True) - assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) - - assert_equal([p for chunk in lib_chunk_gen for p in chunk], - self.mc.get_all_playlists(incremental=False)) - - @mc_test - def mc_search_aa(self): - if 'GM_A' in os.environ: - res = self.mc.search_all_access('amorphis') - with Check() as check: - for hits in res.values(): - check.true(len(hits) > 0) - else: - assert_raises(CallFailure, self.mc.search_all_access, 'amorphis') + @playlist_test + def pt(self): + raise SkipTest('remove this stub test') + + + ## Non-wonky tests resume down here. + + #@test + #def get_registered_devices(self): + # # no logic; just checking schema + # self.wc.get_registered_devices() + + ##--------- + ## MC/AA tests + ##--------- + + ##TODO clean all this up + + #@mc_test + #def list_stations_mc(self): + # lib_chunk_gen = self.mc.get_all_stations(incremental=True) + # assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) - @mc_test - def mc_search_aa_with_limit(self): - if 'GM_A' in os.environ: - res_unlimited = self.mc.search_all_access('cat empire') - res_5 = self.mc.search_all_access('cat empire', max_results=5) + # assert_equal([p for chunk in lib_chunk_gen for p in chunk], + # self.mc.get_all_stations(incremental=False)) - assert_equal(len(res_5['song_hits']), 5) - assert_true(len(res_unlimited['song_hits']) > len(res_5['song_hits'])) + #@mc_test + #def list_playlists_mc(self): + # lib_chunk_gen = self.mc.get_all_playlists(incremental=True) + # assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) - else: - raise SkipTest('AA testing not enabled') + # assert_equal([p for chunk in lib_chunk_gen for p in chunk], + # self.mc.get_all_playlists(incremental=False)) - @mc_test - def mc_artist_info(self): - if 'GM_A' in os.environ: - aid = 'Apoecs6off3y6k4h5nvqqos4b5e' # amorphis - optional_keys = set(('albums', 'topTracks', 'related_artists')) + #@mc_test + #def mc_search_aa(self): + # if 'GM_A' in os.environ: + # res = self.mc.search_all_access('amorphis') + # with Check() as check: + # for hits in res.values(): + # check.true(len(hits) > 0) + # else: + # assert_raises(CallFailure, self.mc.search_all_access, 'amorphis') - include_all_res = self.mc.get_artist_info(aid, include_albums=True, - max_top_tracks=1, max_rel_artist=1) + #@mc_test + #def mc_search_aa_with_limit(self): + # if 'GM_A' in os.environ: + # res_unlimited = self.mc.search_all_access('cat empire') + # res_5 = self.mc.search_all_access('cat empire', max_results=5) - no_albums_res = self.mc.get_artist_info(aid, include_albums=False) - no_rel_res = self.mc.get_artist_info(aid, max_rel_artist=0) - no_tracks_res = self.mc.get_artist_info(aid, max_top_tracks=0) + # assert_equal(len(res_5['song_hits']), 5) + # assert_true(len(res_unlimited['song_hits']) > len(res_5['song_hits'])) - with Check() as check: - check.true(set(include_all_res.keys()) & optional_keys == optional_keys) + # else: + # raise SkipTest('AA testing not enabled') - check.true(set(no_albums_res.keys()) & optional_keys == - optional_keys - set(['albums'])) - check.true(set(no_rel_res.keys()) & optional_keys == - optional_keys - set(['related_artists'])) - check.true(set(no_tracks_res.keys()) & optional_keys == - optional_keys - set(['topTracks'])) + #@mc_test + #def mc_artist_info(self): + # if 'GM_A' in os.environ: + # aid = 'Apoecs6off3y6k4h5nvqqos4b5e' # amorphis + # optional_keys = set(('albums', 'topTracks', 'related_artists')) - else: - assert_raises(CallFailure, self.mc.search_all_access, 'amorphis') + # include_all_res = self.mc.get_artist_info(aid, include_albums=True, + # max_top_tracks=1, max_rel_artist=1) - @test - def get_aa_stream_urls(self): - if 'GM_A' in os.environ: - # that dumb little intro track on Conspiracy of One - urls = self.wc.get_stream_urls('Tqqufr34tuqojlvkolsrwdwx7pe') + # no_albums_res = self.mc.get_artist_info(aid, include_albums=False) + # no_rel_res = self.mc.get_artist_info(aid, max_rel_artist=0) + # no_tracks_res = self.mc.get_artist_info(aid, max_top_tracks=0) - assert_true(len(urls) > 1) - #TODO test getting the stream - else: - raise SkipTest('AA testing not enabled') + # with Check() as check: + # check.true(set(include_all_res.keys()) & optional_keys == optional_keys) - @test - def stream_aa_track(self): - if 'GM_A' in os.environ: - # that dumb little intro track on Conspiracy of One - audio = self.wc.get_stream_audio('Tqqufr34tuqojlvkolsrwdwx7pe') - assert_is_not_none(audio) - else: - raise SkipTest('AA testing not enabled') + # check.true(set(no_albums_res.keys()) & optional_keys == + # optional_keys - set(['albums'])) + # check.true(set(no_rel_res.keys()) & optional_keys == + # optional_keys - set(['related_artists'])) + # check.true(set(no_tracks_res.keys()) & optional_keys == + # optional_keys - set(['topTracks'])) - #----------- - # Song tests - #----------- + # else: + # assert_raises(CallFailure, self.mc.search_all_access, 'amorphis') - #TODO album art + #@test + #def get_aa_stream_urls(self): + # if 'GM_A' in os.environ: + # # that dumb little intro track on Conspiracy of One + # urls = self.wc.get_stream_urls('Tqqufr34tuqojlvkolsrwdwx7pe') - def _assert_get_song(self, sid, client=None): - """Return the song dictionary with this sid. + # assert_true(len(urls) > 1) + # #TODO test getting the stream + # else: + # raise SkipTest('AA testing not enabled') - (GM has no native get for songs, just list). + #@test + #def stream_aa_track(self): + # if 'GM_A' in os.environ: + # # that dumb little intro track on Conspiracy of One + # audio = self.wc.get_stream_audio('Tqqufr34tuqojlvkolsrwdwx7pe') + # assert_is_not_none(audio) + # else: + # raise SkipTest('AA testing not enabled') - :param client: a Webclient or Musicmanager - """ - if client is None: - client = self.wc + ##----------- + ## Song tests + ##----------- - songs = client.get_all_songs() + ##TODO album art + + #def _assert_get_song(self, sid, client=None): + # """Return the song dictionary with this sid. - found = [s for s in songs if s['id'] == sid] or None + # (GM has no native get for songs, just list). - assert_is_not_none(found) - assert_equal(len(found), 1) + # :param client: a Webclient or Musicmanager + # """ + # if client is None: + # client = self.wc - return found[0] + # songs = client.get_all_songs() + + # found = [s for s in songs if s['id'] == sid] or None - @song_test - def list_songs_wc(self): - self._assert_get_song(self.song.sid, self.wc) + # assert_is_not_none(found) + # assert_equal(len(found), 1) - @song_test - def list_songs_mm(self): - self._assert_get_song(self.song.sid, self.mm) + # return found[0] - @song_test - def list_songs_mc(self): - self._assert_get_song(self.song.sid, self.mc) + #@song_test + #def list_songs_wc(self): + # self._assert_get_song(self.song.sid, self.wc) - @staticmethod - def _list_songs_incrementally(client): - lib_chunk_gen = client.get_all_songs(incremental=True) - assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) + #@song_test + #def list_songs_mm(self): + # self._assert_get_song(self.song.sid, self.mm) - assert_equal([s for chunk in lib_chunk_gen for s in chunk], - client.get_all_songs(incremental=False)) + #@song_test + #def list_songs_mc(self): + # self._assert_get_song(self.song.sid, self.mc) - @song_test - def list_songs_incrementally_wc(self): - self._list_songs_incrementally(self.wc) + #@staticmethod + #def _list_songs_incrementally(client): + # lib_chunk_gen = client.get_all_songs(incremental=True) + # assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) - @song_test - def list_songs_incrementally_mm(self): - self._list_songs_incrementally(self.mm) + # assert_equal([s for chunk in lib_chunk_gen for s in chunk], + # client.get_all_songs(incremental=False)) - @mc_test - def list_songs_incrementally_mc(self): - self._list_songs_incrementally(self.mc) + #@song_test + #def list_songs_incrementally_wc(self): + # self._list_songs_incrementally(self.wc) - @song_test - def change_metadata(self): - orig_md = self._assert_get_song(self.song.sid) + #@song_test + #def list_songs_incrementally_mm(self): + # self._list_songs_incrementally(self.mm) - # Change all mutable entries. + #@mc_test + #def list_songs_incrementally_mc(self): + # self._list_songs_incrementally(self.mc) - new_md = copy(orig_md) + #@song_test + #def change_metadata(self): + # orig_md = self._assert_get_song(self.song.sid) - for name, expt in md_expectations.items(): - if name in orig_md and expt.mutable: - old_val = orig_md[name] - new_val = test_utils.modify_md(name, old_val) + # # Change all mutable entries. - assert_not_equal(new_val, old_val) - new_md[name] = new_val + # new_md = copy(orig_md) - #TODO check into attempting to mutate non mutables - self.wc.change_song_metadata(new_md) + # for name, expt in md_expectations.items(): + # if name in orig_md and expt.mutable: + # old_val = orig_md[name] + # new_val = test_utils.modify_md(name, old_val) - #Recreate the dependent md to what they should be (based on how orig_md was changed) - correct_dependent_md = {} - for name, expt in md_expectations.items(): - if expt.depends_on and name in orig_md: - master_name = expt.depends_on - correct_dependent_md[name] = expt.dependent_transformation(new_md[master_name]) + # assert_not_equal(new_val, old_val) + # new_md[name] = new_val - @retry - def assert_metadata_is(sid, orig_md, correct_dependent_md): - result_md = self._assert_get_song(sid) + # #TODO check into attempting to mutate non mutables + # self.wc.change_song_metadata(new_md) - with Check() as check: - for name, expt in md_expectations.items(): - if name in orig_md: - #TODO really need to factor out to test_utils? + # #Recreate the dependent md to what they should be (based on how orig_md was changed) + # correct_dependent_md = {} + # for name, expt in md_expectations.items(): + # if expt.depends_on and name in orig_md: + # master_name = expt.depends_on + # correct_dependent_md[name] = expt.dependent_transformation(new_md[master_name]) - #Check mutability if it's not volatile or dependent. - if not expt.volatile and expt.depends_on is None: - same, message = test_utils.md_entry_same(name, orig_md, result_md) - check.equal(not expt.mutable, same, - "metadata mutability incorrect: " + message) + # @retry + # def assert_metadata_is(sid, orig_md, correct_dependent_md): + # result_md = self._assert_get_song(sid) - #Check dependent md. - if expt.depends_on is not None: - same, message = test_utils.md_entry_same( - name, correct_dependent_md, result_md - ) - check.true(same, "dependent metadata incorrect: " + message) + # with Check() as check: + # for name, expt in md_expectations.items(): + # if name in orig_md: + # #TODO really need to factor out to test_utils? - assert_metadata_is(self.song.sid, orig_md, correct_dependent_md) + # #Check mutability if it's not volatile or dependent. + # if not expt.volatile and expt.depends_on is None: + # same, message = test_utils.md_entry_same(name, orig_md, result_md) + # check.equal(not expt.mutable, same, + # "metadata mutability incorrect: " + message) - #Revert the metadata. - self.wc.change_song_metadata(orig_md) + # #Check dependent md. + # if expt.depends_on is not None: + # same, message = test_utils.md_entry_same( + # name, correct_dependent_md, result_md + # ) + # check.true(same, "dependent metadata incorrect: " + message) - @retry - def assert_metadata_reverted(sid, orig_md): - result_md = self._assert_get_song(sid) + # assert_metadata_is(self.song.sid, orig_md, correct_dependent_md) - with Check() as check: - for name in orig_md: - #If it's not volatile, it should be back to what it was. - if not md_expectations[name].volatile: - same, message = test_utils.md_entry_same(name, orig_md, result_md) - check.true(same, "failed to revert: " + message) - assert_metadata_reverted(self.song.sid, orig_md) + # #Revert the metadata. + # self.wc.change_song_metadata(orig_md) - #TODO verify these better? + # @retry + # def assert_metadata_reverted(sid, orig_md): + # result_md = self._assert_get_song(sid) - @song_test - def get_download_info(self): - url, download_count = self.wc.get_song_download_info(self.song.sid) + # with Check() as check: + # for name in orig_md: + # #If it's not volatile, it should be back to what it was. + # if not md_expectations[name].volatile: + # same, message = test_utils.md_entry_same(name, orig_md, result_md) + # check.true(same, "failed to revert: " + message) + # assert_metadata_reverted(self.song.sid, orig_md) - assert_is_not_none(url) + ##TODO verify these better? - @song_test - def download_song_mm(self): + #@song_test + #def get_download_info(self): + # url, download_count = self.wc.get_song_download_info(self.song.sid) - @retry - def assert_download(sid=self.song.sid): - filename, audio = self.mm.download_song(sid) + # assert_is_not_none(url) - # there's some kind of a weird race happening here with CI; - # usually one will succeed and one will fail + #@song_test + #def download_song_mm(self): - #TODO could use original filename to verify this - # but, when manually checking, got modified title occasionally - assert_true(filename.endswith('.mp3')) # depends on specific file - assert_is_not_none(audio) - assert_download() + # @retry + # def assert_download(sid=self.song.sid): + # filename, audio = self.mm.download_song(sid) - @song_test - def get_uploaded_stream_urls(self): - urls = self.wc.get_stream_urls(self.song.sid) + # # there's some kind of a weird race happening here with CI; + # # usually one will succeed and one will fail - assert_equal(len(urls), 1) + # #TODO could use original filename to verify this + # # but, when manually checking, got modified title occasionally + # assert_true(filename.endswith('.mp3')) # depends on specific file + # assert_is_not_none(audio) + # assert_download() - url = urls[0] + #@song_test + #def get_uploaded_stream_urls(self): + # urls = self.wc.get_stream_urls(self.song.sid) - assert_is_not_none(url) - assert_equal(url[:7], 'http://') + # assert_equal(len(urls), 1) - @song_test - def upload_album_art(self): - orig_md = self._assert_get_song(self.song.sid) + # url = urls[0] - self.wc.upload_album_art(self.song.sid, test_utils.image_filename) + # assert_is_not_none(url) + # assert_equal(url[:7], 'http://') - self.wc.change_song_metadata(orig_md) - #TODO redownload and verify against original? + #@song_test + #def upload_album_art(self): + # orig_md = self._assert_get_song(self.song.sid) - # these search tests are all skipped: see - # https://github.com/simon-weber/Unofficial-Google-Music-API/issues/114 + # self.wc.upload_album_art(self.song.sid, test_utils.image_filename) - @staticmethod - def _assert_search_hit(res, hit_type, hit_key, val): - """Assert that the result (returned from wc.search) has - ``hit[hit_type][hit_key] == val`` for only one result in hit_type.""" + # self.wc.change_song_metadata(orig_md) + # #TODO redownload and verify against original? - raise SkipTest('search is unpredictable (#114)') + ## these search tests are all skipped: see + ## https://github.com/simon-weber/Unofficial-Google-Music-API/issues/114 - #assert_equal(sorted(res.keys()), ['album_hits', 'artist_hits', 'song_hits']) - #assert_not_equal(res[hit_type], []) + #@staticmethod + #def _assert_search_hit(res, hit_type, hit_key, val): + # """Assert that the result (returned from wc.search) has + # ``hit[hit_type][hit_key] == val`` for only one result in hit_type.""" - #hitmap = (hit[hit_key] == val for hit in res[hit_type]) - #assert_equal(sum(hitmap), 1) # eg sum(True, False, True) == 2 + # raise SkipTest('search is unpredictable (#114)') - #@song_test - #def search_title(self): - # res = self.wc.search(self.song.title) + # #assert_equal(sorted(res.keys()), ['album_hits', 'artist_hits', 'song_hits']) + # #assert_not_equal(res[hit_type], []) - # self._assert_search_hit(res, 'song_hits', 'id', self.song.sid) + # #hitmap = (hit[hit_key] == val for hit in res[hit_type]) + # #assert_equal(sum(hitmap), 1) # eg sum(True, False, True) == 2 - #@song_test - #def search_artist(self): - # res = self.wc.search(self.song.artist) + ##@song_test + ##def search_title(self): + ## res = self.wc.search(self.song.title) - # self._assert_search_hit(res, 'artist_hits', 'id', self.song.sid) + ## self._assert_search_hit(res, 'song_hits', 'id', self.song.sid) - #@song_test - #def search_album(self): - # res = self.wc.search(self.song.album) + ##@song_test + ##def search_artist(self): + ## res = self.wc.search(self.song.artist) - # self._assert_search_hit(res, 'album_hits', 'albumName', self.song.album) + ## self._assert_search_hit(res, 'artist_hits', 'id', self.song.sid) - #--------------- - # Playlist tests - #--------------- + ##@song_test + ##def search_album(self): + ## res = self.wc.search(self.song.album) - #TODO copy, change (need two songs?) + ## self._assert_search_hit(res, 'album_hits', 'albumName', self.song.album) - @playlist_test - def change_name(self): - new_name = TEST_PLAYLIST_NAME + '_mod' - self.wc.change_playlist_name(self.playlist_id, new_name) + ##--------------- + ## Playlist tests + ##--------------- - @retry # change takes time to propogate - def assert_name_equal(plid, name): - playlists = self.wc.get_all_playlist_ids() + ##TODO copy, change (need two songs?) - found = playlists['user'].get(name, None) + #@playlist_test + #def change_name(self): + # new_name = TEST_PLAYLIST_NAME + '_mod' + # self.wc.change_playlist_name(self.playlist_id, new_name) - assert_is_not_none(found) - assert_equal(found[-1], self.playlist_id) + # @retry # change takes time to propogate + # def assert_name_equal(plid, name): + # playlists = self.wc.get_all_playlist_ids() - assert_name_equal(self.playlist_id, new_name) + # found = playlists['user'].get(name, None) - # revert - self.wc.change_playlist_name(self.playlist_id, TEST_PLAYLIST_NAME) - assert_name_equal(self.playlist_id, TEST_PLAYLIST_NAME) + # assert_is_not_none(found) + # assert_equal(found[-1], self.playlist_id) - @playlist_test - def add_remove(self): - @retry - def assert_song_order(plid, order): - songs = self.wc.get_playlist_songs(plid) - server_order = [s['id'] for s in songs] + # assert_name_equal(self.playlist_id, new_name) + + # # revert + # self.wc.change_playlist_name(self.playlist_id, TEST_PLAYLIST_NAME) + # assert_name_equal(self.playlist_id, TEST_PLAYLIST_NAME) + + #@playlist_test + #def add_remove(self): + # @retry + # def assert_song_order(plid, order): + # songs = self.wc.get_playlist_songs(plid) + # server_order = [s['id'] for s in songs] - assert_equal(server_order, order) + # assert_equal(server_order, order) - # initially empty - assert_song_order(self.playlist_id, []) + # # initially empty + # assert_song_order(self.playlist_id, []) - # add two copies - self.wc.add_songs_to_playlist(self.playlist_id, [self.song.sid] * 2) - assert_song_order(self.playlist_id, [self.song.sid] * 2) + # # add two copies + # self.wc.add_songs_to_playlist(self.playlist_id, [self.song.sid] * 2) + # assert_song_order(self.playlist_id, [self.song.sid] * 2) - # remove all copies - self.wc.remove_songs_from_playlist(self.playlist_id, self.song.sid) - assert_song_order(self.playlist_id, []) + # # remove all copies + # self.wc.remove_songs_from_playlist(self.playlist_id, self.song.sid) + # assert_song_order(self.playlist_id, []) diff --git a/gmusicapi/test/utils.py b/gmusicapi/test/utils.py index 65283946..db1ebabf 100644 --- a/gmusicapi/test/utils.py +++ b/gmusicapi/test/utils.py @@ -30,6 +30,9 @@ small_mp3 = os.path.join(test_file_dir, u'audiotest_small.mp3') image_filename = os.path.join(test_file_dir, u'imagetest_10x10_check.png') +# that dumb intro track on conspiracy of one +aa_song_id = 'Tqqufr34tuqojlvkolsrwdwx7pe' + class NoticeLogging(logging.Handler): """A log handler that, if asked to emit, will set From 932ed05074f2640d3a6779f2723f0e50af8353b5 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sun, 14 Jul 2013 17:49:34 -0400 Subject: [PATCH 55/72] better add/list/delete testing --- gmusicapi/clients/mobileclient.py | 11 +++++- gmusicapi/test/server_tests.py | 62 +++++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index fb75434f..5749fe2e 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -39,13 +39,16 @@ def login(self, email, password): #TODO expose max-results, updated_after, etc for list operations - def get_all_songs(self, incremental=False): + def get_all_songs(self, incremental=False, include_deleted=False): """Returns a list of dictionaries that each represent a song. :param incremental: if True, return a generator that yields lists of at most 1000 tracks as they are retrieved from the server. This can be useful for presenting a loading bar to a user. + :param include_delete: if True, include tracks that have been deleted + in the past. If False, ``t['deleted'] is False`` will hold for all + tracks. Here is an example song dictionary:: { @@ -92,7 +95,11 @@ def get_all_songs(self, incremental=False): } """ - return self._get_all_items(mobileclient.ListTracks, incremental) + tracks = self._get_all_items(mobileclient.ListTracks, incremental) + if not include_deleted: + tracks = [t for t in tracks if not t['deleted']] + + return tracks def add_aa_track(self, aa_song_id): """Adds an All Access track to the library, diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 176b878a..55a14a72 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -109,7 +109,7 @@ def assert_songs_state(self, sids, present): :param present: if True verify songs are present; False the opposite """ - library = self.wc.get_all_songs() + library = self.mc.get_all_songs() found = [s for s in library if s['id'] in sids] @@ -122,6 +122,38 @@ def assert_songs_state(self, sids, present): return [TestSong(s['id'], s['title'], s['artist'], s['album']) for s in found] + @staticmethod + @retry + def assert_list_inc_equivalence(method): + """ + Assert that some listing method returns the same + contents for incremental=True/False. + + :param method: eg self.mc.get_all_songs + """ + lib_chunk_gen = method(incremental=True) + assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) + + assert_equal([p for chunk in lib_chunk_gen for p in chunk], + method(incremental=False)) + + @staticmethod + @retry + def assert_list_with_deleted(method): + """ + Assert that some listing method includes deleted tracks + when requested. + + :param method: eg self.mc.get_all_songs + """ + lib = method(incremental=False, include_deleted=True) + + # how long do deleted tracks get returned for? + # will this return tracks I've deleted since...ever? + + num_deleted = [t for t in lib if t['deleted']] + assert_true(num_deleted > 0) + @test def song_create(self): # This can create more than one song: one through uploading, one through @@ -206,6 +238,7 @@ def song_delete(self): check.equal(res, [testsong.sid]) self.assert_songs_state([s.sid for s in self.songs], present=False) + self.assert_list_with_deleted(self.mc.get_all_songs) ## These decorators just prevent setting groups and depends_on over and over. ## They won't work right with additional settings; if that's needed this @@ -215,8 +248,8 @@ def song_delete(self): song_test = test(groups=['song', 'song.exists'], depends_on=[song_create]) playlist_test = test(groups=['playlist', 'playlist.exists'], depends_on=[playlist_create]) - #mc_test = test(groups=['mobile']) + # just to make song_test/playlist_test exist for now @song_test def st(self): raise SkipTest('remove this stub test') @@ -225,27 +258,24 @@ def st(self): def pt(self): raise SkipTest('remove this stub test') - ## Non-wonky tests resume down here. - #@test - #def get_registered_devices(self): - # # no logic; just checking schema - # self.wc.get_registered_devices() - ##--------- - ## MC/AA tests + ## WC tests ##--------- - ##TODO clean all this up + @test + def wc_get_registered_devices(self): + # no logic; just checking schema + self.wc.get_registered_devices() - #@mc_test - #def list_stations_mc(self): - # lib_chunk_gen = self.mc.get_all_stations(incremental=True) - # assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) + ##--------- + ## MC tests + ##--------- - # assert_equal([p for chunk in lib_chunk_gen for p in chunk], - # self.mc.get_all_stations(incremental=False)) + @test + def mc_list_stations_inc_equal(self): + self.assert_list_inc_equivalence(self.mc.get_all_stations) #@mc_test #def list_playlists_mc(self): From 4ad8cc1c5cc6dd2e35bc57f7aad6c26fe5c5e3d2 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 17 Jul 2013 12:50:00 -0400 Subject: [PATCH 56/72] playlist add/delete/list --- gmusicapi/clients/mobileclient.py | 51 ++++++++++++++++++++++++++-- gmusicapi/protocol/mobileclient.py | 27 +++++++++++++++ gmusicapi/test/server_tests.py | 54 ++++++++++++++++++++---------- 3 files changed, 112 insertions(+), 20 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 5749fe2e..eb6220d8 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -96,8 +96,14 @@ def get_all_songs(self, incremental=False, include_deleted=False): """ tracks = self._get_all_items(mobileclient.ListTracks, incremental) + + # this should probably go into _get_all_items if not include_deleted: - tracks = [t for t in tracks if not t['deleted']] + if not incremental: + tracks = [t for t in tracks if not t['deleted']] + else: + tracks = ([t for t in chunk if not t['deleted']] + for chunk in tracks) return tracks @@ -166,13 +172,16 @@ def get_stream_url(self, song_id, device_id): return self._make_call(mobileclient.GetStreamUrl, song_id, device_id) - def get_all_playlists(self, incremental=False): + def get_all_playlists(self, incremental=False, include_deleted=False): """Returns a list of dictionaries that each represent a playlist. :param incremental: if True, return a generator that yields lists of at most 1000 playlists as they are retrieved from the server. This can be useful for presenting a loading bar to a user. + :param include_delete: if True, include playlists that have been deleted + in the past. If False, ``p['deleted'] is False`` will hold for all + playlists. Here is an example playlist dictionary:: { @@ -191,7 +200,43 @@ def get_all_playlists(self, incremental=False): } """ - return self._get_all_items(mobileclient.ListPlaylists, incremental) + playlists = self._get_all_items(mobileclient.ListPlaylists, incremental) + + if not include_deleted: + if not incremental: + playlists = [t for t in playlists if not t['deleted']] + else: + playlists = ([t for t in chunk if not t['deleted']] + for chunk in playlists) + + return playlists + + # these could trivially support multiple creation/deletion, but + # I chose to match the old webclient interface (at least for now). + def create_playlist(self, name): + """Creates a new empty playlist and returns its id. + + :param name: the desired title. + Creating multiple playlists with the same name is allowed. + """ + + mutate_call = mobileclient.BatchMutatePlaylists + add_mutations = mutate_call.build_playlist_adds([name]) + res = self._make_call(mutate_call, add_mutations) + + return res['mutate_response'][0]['id'] + + def delete_playlist(self, playlist_id): + """Deletes a playlist and returns its id. + + :param playlist_id: the id to delete. + """ + + mutate_call = mobileclient.BatchMutatePlaylists + del_mutations = mutate_call.build_playlist_deletes([playlist_id]) + res = self._make_call(mutate_call, del_mutations) + + return res['mutate_response'][0]['id'] def get_all_stations(self, updated_after=None, incremental=False): """Returns a list of dictionaries that each represent a radio station. diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 9588e084..682294c7 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -403,6 +403,33 @@ class ListPlaylists(McListCall): static_url = sj_url + 'playlistfeed' +class BatchMutatePlaylists(McBatchMutateCall): + static_method = 'POST' + static_url = sj_url + 'playlistbatch' + + @staticmethod + def build_playlist_deletes(playlist_ids): + #TODO can probably pull this up one + """ + :param playlist_ids + """ + return [{'delete': id} for id in playlist_ids] + + @staticmethod + def build_playlist_adds(names): + """ + :param names + """ + + return [{'create': { + 'creationTimestamp': '-1', + 'deleted': False, + 'lastModifiedTimestamp': '0', + 'name': name, + 'type': 'USER_GENERATED' + }} for name in names] + + class ListStations(McListCall): item_schema = sj_station filter_text = 'stations' diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 55a14a72..3be23b00 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -124,25 +124,26 @@ def assert_songs_state(self, sids, present): @staticmethod @retry - def assert_list_inc_equivalence(method): + def assert_list_inc_equivalence(method, **kwargs): """ Assert that some listing method returns the same contents for incremental=True/False. :param method: eg self.mc.get_all_songs + :param **kwargs: passed to method """ - lib_chunk_gen = method(incremental=True) + + lib_chunk_gen = method(incremental=True, **kwargs) assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) - assert_equal([p for chunk in lib_chunk_gen for p in chunk], - method(incremental=False)) + assert_equal([e for chunk in lib_chunk_gen for e in chunk], + method(incremental=False, **kwargs)) @staticmethod @retry def assert_list_with_deleted(method): """ - Assert that some listing method includes deleted tracks - when requested. + Assert that some listing method includes deleted tracks. :param method: eg self.mc.get_all_songs """ @@ -188,24 +189,21 @@ def song_create(self): self.songs = self.assert_songs_state(sids, present=True) + #TODO pull this out to include song state checking @test(depends_on=[song_create], runs_after_groups=['song.exists']) def playlist_create(self): - raise SkipTest('playlist create broken') - - self.playlist_id = self.wc.create_playlist(TEST_PLAYLIST_NAME) + playlist_id = self.mc.create_playlist(TEST_PLAYLIST_NAME) # like song_create, retry until the playlist appears - @retry def assert_playlist_exists(plid): - playlists = self.wc.get_all_playlist_ids(auto=False, user=True) - - found = playlists['user'].get(TEST_PLAYLIST_NAME, None) + found = [p for p in self.mc.get_all_playlists() + if p['id'] == plid] - assert_is_not_none(found) - assert_equal(found[-1], self.playlist_id) + assert_equal(len(found), 1) - assert_playlist_exists(self.playlist_id) + assert_playlist_exists(playlist_id) + self.playlist_id = playlist_id #TODO consider listing/searching if the id isn't there # to ensure cleanup. @@ -216,9 +214,19 @@ def playlist_delete(self): if self.playlist_id is None: raise SkipTest('did not store self.playlist_id') - res = self.wc.delete_playlist(self.playlist_id) + res = self.mc.delete_playlist(self.playlist_id) assert_equal(res, self.playlist_id) + @retry + def assert_playlist_does_not_exist(plid): + found = [p for p in self.mc.get_all_playlists(include_deleted=False) + if p['id'] == plid] + + assert_equal(len(found), 0) + + assert_playlist_does_not_exist(self.playlist_id) + self.assert_list_with_deleted(self.mc.get_all_playlists) + @test(groups=['song'], depends_on=[song_create], runs_after=[playlist_delete], runs_after_groups=["song.exists"], @@ -277,6 +285,18 @@ def wc_get_registered_devices(self): def mc_list_stations_inc_equal(self): self.assert_list_inc_equivalence(self.mc.get_all_stations) + @test + def mc_list_songs_inc_equal(self): + self.assert_list_inc_equivalence(self.mc.get_all_songs) + + @test + def mc_list_songs_inc_equal_with_deleted(self): + self.assert_list_inc_equivalence(self.mc.get_all_songs, include_deleted=True) + + @test + def mc_list_playlists_inc_equal(self): + self.assert_list_inc_equivalence(self.mc.get_all_playlists) + #@mc_test #def list_playlists_mc(self): # lib_chunk_gen = self.mc.get_all_playlists(incremental=True) From 0f890704ba5735365c225d722ae5d817ba2a8a2d Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 17 Jul 2013 13:11:48 -0400 Subject: [PATCH 57/72] include_deleted factored down --- gmusicapi/clients/mobileclient.py | 65 +++++++++++++++---------------- gmusicapi/test/server_tests.py | 8 ++++ 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index eb6220d8..3b1aad7f 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -46,11 +46,10 @@ def get_all_songs(self, incremental=False, include_deleted=False): of at most 1000 tracks as they are retrieved from the server. This can be useful for presenting a loading bar to a user. - :param include_delete: if True, include tracks that have been deleted - in the past. If False, ``t['deleted'] is False`` will hold for all - tracks. + :param include_deleted: if True, include tracks that have been deleted + in the past. - Here is an example song dictionary:: + Here is an example song dictionary:: { u'comment':u'', u'rating':u'0', @@ -95,15 +94,7 @@ def get_all_songs(self, incremental=False, include_deleted=False): } """ - tracks = self._get_all_items(mobileclient.ListTracks, incremental) - - # this should probably go into _get_all_items - if not include_deleted: - if not incremental: - tracks = [t for t in tracks if not t['deleted']] - else: - tracks = ([t for t in chunk if not t['deleted']] - for chunk in tracks) + tracks = self._get_all_items(mobileclient.ListTracks, incremental, include_deleted) return tracks @@ -179,11 +170,10 @@ def get_all_playlists(self, incremental=False, include_deleted=False): of at most 1000 playlists as they are retrieved from the server. This can be useful for presenting a loading bar to a user. - :param include_delete: if True, include playlists that have been deleted - in the past. If False, ``p['deleted'] is False`` will hold for all - playlists. + :param include_deleted: if True, include playlists that have been deleted + in the past. - Here is an example playlist dictionary:: + Here is an example playlist dictionary:: { u 'kind': u 'sj#playlist', u 'name': u 'Something Mix', @@ -200,14 +190,7 @@ def get_all_playlists(self, incremental=False, include_deleted=False): } """ - playlists = self._get_all_items(mobileclient.ListPlaylists, incremental) - - if not include_deleted: - if not incremental: - playlists = [t for t in playlists if not t['deleted']] - else: - playlists = ([t for t in chunk if not t['deleted']] - for chunk in playlists) + playlists = self._get_all_items(mobileclient.ListPlaylists, incremental, include_deleted) return playlists @@ -238,16 +221,18 @@ def delete_playlist(self, playlist_id): return res['mutate_response'][0]['id'] - def get_all_stations(self, updated_after=None, incremental=False): + def get_all_stations(self, incremental=False, include_deleted=False, updated_after=None): """Returns a list of dictionaries that each represent a radio station. - :param updated_after: a datetime.datetime; defaults to epoch :param incremental: if True, return a generator that yields lists of at most 1000 stations as they are retrieved from the server. This can be useful for presenting a loading bar to a user. + :param include_deleted: if True, include stations that have been deleted + in the past. + :param updated_after: a datetime.datetime; defaults to epoch - Here is an example station dictionary:: + Here is an example station dictionary:: { u 'imageUrl': u 'http://lh6.ggpht.com/...', u 'kind': u 'sj#radioStation', @@ -266,7 +251,7 @@ def get_all_stations(self, updated_after=None, incremental=False): u 'id': u '69f1bfce-308a-313e-9ed2-e50abe33a25d' }, """ - return self._get_all_items(mobileclient.ListStations, incremental, + return self._get_all_items(mobileclient.ListStations, incremental, include_deleted, updated_after=updated_after) def search_all_access(self, query, max_results=50): @@ -466,22 +451,29 @@ def get_artist_info(self, artist_id, include_albums=True, max_top_tracks=5, max_ artist_id, include_albums, max_top_tracks, max_rel_artist) return res - def _get_all_items(self, call, incremental, **kwargs): + def _get_all_items(self, call, incremental, include_deleted, **kwargs): """ :param call: protocol.McCall :param incremental: bool + :param include_deleted: bool kwargs are passed to the call. """ if not incremental: # slight optimization; can get all items at once res = self._make_call(call, max_results=20000, **kwargs) - return res['data']['items'] + + items = res['data']['items'] + + if not include_deleted: + items = [t for t in items if not t['deleted']] + + return items # otherwise, return a generator - return self._get_all_items_incremental(call, **kwargs) + return self._get_all_items_incremental(call, include_deleted, **kwargs) - def _get_all_items_incremental(self, call, **kwargs): + def _get_all_items_incremental(self, call, include_deleted, **kwargs): """Return a generator of lists of tracks. kwargs are passed to the call.""" @@ -494,7 +486,12 @@ def _get_all_items_incremental(self, call, **kwargs): start_token=lib_chunk['nextPageToken'], **kwargs) - yield lib_chunk['data']['items'] # list of songs of the chunk + items = lib_chunk['data']['items'] + + if not include_deleted: + items = [item for item in items if not item['deleted']] + + yield items get_next_chunk = 'nextPageToken' in lib_chunk diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 3be23b00..482b9af4 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -285,6 +285,10 @@ def wc_get_registered_devices(self): def mc_list_stations_inc_equal(self): self.assert_list_inc_equivalence(self.mc.get_all_stations) + @test + def mc_list_stations_inc_equal_with_deleted(self): + self.assert_list_inc_equivalence(self.mc.get_all_stations, include_deleted=True) + @test def mc_list_songs_inc_equal(self): self.assert_list_inc_equivalence(self.mc.get_all_songs) @@ -297,6 +301,10 @@ def mc_list_songs_inc_equal_with_deleted(self): def mc_list_playlists_inc_equal(self): self.assert_list_inc_equivalence(self.mc.get_all_playlists) + @test + def mc_list_playlists_inc_equal_with_deleted(self): + self.assert_list_inc_equivalence(self.mc.get_all_playlists, include_deleted=True) + #@mc_test #def list_playlists_mc(self): # lib_chunk_gen = self.mc.get_all_playlists(incremental=True) From 2f60b77280f915f818b17cfb8b39e3bdb911ab2f Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 17 Jul 2013 14:09:47 -0400 Subject: [PATCH 58/72] test search, fix schema --- gmusicapi/protocol/mobileclient.py | 2 ++ gmusicapi/test/server_tests.py | 24 +++++++----------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 682294c7..96b41601 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -116,6 +116,8 @@ 'score': {'type': 'number'}, 'type': {'type': 'string'}, 'best_result': {'type': 'boolean', 'required': False}, + 'navigational_result': {'type': 'boolean', 'required': False}, + 'navigational_confidence': {'type': 'number', 'required': False}, 'artist': sj_artist.copy(), 'album': sj_album.copy(), 'track': sj_track.copy(), diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 482b9af4..04fc7392 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -305,23 +305,13 @@ def mc_list_playlists_inc_equal(self): def mc_list_playlists_inc_equal_with_deleted(self): self.assert_list_inc_equivalence(self.mc.get_all_playlists, include_deleted=True) - #@mc_test - #def list_playlists_mc(self): - # lib_chunk_gen = self.mc.get_all_playlists(incremental=True) - # assert_true(isinstance(lib_chunk_gen, types.GeneratorType)) - - # assert_equal([p for chunk in lib_chunk_gen for p in chunk], - # self.mc.get_all_playlists(incremental=False)) - - #@mc_test - #def mc_search_aa(self): - # if 'GM_A' in os.environ: - # res = self.mc.search_all_access('amorphis') - # with Check() as check: - # for hits in res.values(): - # check.true(len(hits) > 0) - # else: - # assert_raises(CallFailure, self.mc.search_all_access, 'amorphis') + @test + @all_access + def mc_search_aa(self): + res = self.mc.search_all_access('amorphis') + with Check() as check: + for hits in res.values(): + check.true(len(hits) > 0) #@mc_test #def mc_search_aa_with_limit(self): From 94af3e2edef5a09f23f7f2c93c4c8a5107e17bc0 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Wed, 17 Jul 2013 14:20:49 -0400 Subject: [PATCH 59/72] don't output auth headers --- gmusicapi/protocol/shared.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/gmusicapi/protocol/shared.py b/gmusicapi/protocol/shared.py index 888a2d19..30cfa4a4 100644 --- a/gmusicapi/protocol/shared.py +++ b/gmusicapi/protocol/shared.py @@ -89,7 +89,6 @@ def build_request(cls, *args, **kwargs): req_kwargs[key] = val return req_kwargs - #return Request(**req_kwargs) return build_request new_cls.build_request = classmethod(req_closure()) @@ -209,6 +208,10 @@ def perform(cls, session, validate, *args, **kwargs): response = session.send(req_kwargs, cls.required_auth) #TODO trim the logged response if it's huge? + safe_req_kwargs = req_kwargs.copy() + if safe_req_kwargs.get('headers', {}).get('Authorization', None) is not None: + safe_req_kwargs['headers']['Authorization'] = '' + # check response code try: response.raise_for_status() @@ -216,7 +219,7 @@ def perform(cls, session, validate, *args, **kwargs): err_msg = str(e) if cls.gets_logged: - err_msg += "\n(requests kwargs: %r)" % (req_kwargs) + err_msg += "\n(requests kwargs: %r)" % (safe_req_kwargs) err_msg += "\n(response was: %r)" % response.content raise CallFailure(err_msg, call_name) @@ -227,7 +230,7 @@ def perform(cls, session, validate, *args, **kwargs): err_msg = ("the server's response could not be understood." " The call may still have succeeded, but it's unlikely.") if cls.gets_logged: - err_msg += "\n(requests kwargs: %r)" % (req_kwargs) + err_msg += "\n(requests kwargs: %r)" % (safe_req_kwargs) err_msg += "\n(response was: %r)" % response.content log.exception("could not parse %s response: %r", call_name, response.content) else: @@ -261,7 +264,7 @@ def perform(cls, session, validate, *args, **kwargs): " please [create an issue](http://goo.gl/qbAW8) that includes" " the above ValidationException" " and the following request/response:\n%r\n\n%r\n" - "\nA traceback follows:\n") % (req_kwargs, raw_response) + "\nA traceback follows:\n") % (safe_req_kwargs, raw_response) log.exception(err_msg) From 2612124764243f9dc11bb5540c9843d837368a85 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sat, 20 Jul 2013 11:36:16 -0400 Subject: [PATCH 60/72] add hasFreeTrial GetSettings key --- gmusicapi/protocol/webclient.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 2ffe0ec0..840f3039 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -686,6 +686,7 @@ class GetSettings(WcCall): }, 'isSubscription': {'type': 'boolean', 'required': False}, 'isTrial': {'type': 'boolean', 'required': False}, + 'hasFreeTrial': {'type': 'boolean'}, }, }, }, From e15012d02ba18df3e134163f9025e224db966849 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sat, 20 Jul 2013 12:00:43 -0400 Subject: [PATCH 61/72] disable playlist tests (#144) --- gmusicapi/test/server_tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 9547dc1a..79b9c6e3 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -114,6 +114,8 @@ def assert_song_exists(sid): @test(depends_on=[song_create], runs_after_groups=['song.exists']) def playlist_create(self): + raise SkipTest('playlist create broken') + self.playlist_id = self.wc.create_playlist(TEST_PLAYLIST_NAME) # like song_create, retry until the playlist appears @@ -377,6 +379,8 @@ def _assert_search_hit(res, hit_type, hit_key, val): @playlist_test def change_name(self): + raise SkipTest('playlist create broken') + new_name = TEST_PLAYLIST_NAME + '_mod' self.wc.change_playlist_name(self.playlist_id, new_name) @@ -397,6 +401,8 @@ def assert_name_equal(plid, name): @playlist_test def add_remove(self): + raise SkipTest('playlist create broken') + @retry def assert_song_order(plid, order): songs = self.wc.get_playlist_songs(plid) From 9eb6b05546bf727c82c12f6d1e673e475ce413e6 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Sat, 20 Jul 2013 12:03:46 -0400 Subject: [PATCH 62/72] cherrypick hasFreeTrial change --- gmusicapi/protocol/webclient.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 54588089..4ffe336d 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -684,6 +684,7 @@ class GetSettings(WcCall): }, 'isSubscription': {'type': 'boolean', 'required': False}, 'isTrial': {'type': 'boolean', 'required': False}, + 'hasFreeTrial': {'type': 'boolean'}, }, }, }, From 46900ae50a28190ee9ff0c135a3dae9e58066c5c Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Mon, 22 Jul 2013 12:25:18 -0400 Subject: [PATCH 63/72] add get_playlist_songs; begin work on playlist_mutation --- gmusicapi/clients/mobileclient.py | 46 +++++++- gmusicapi/protocol/mobileclient.py | 163 +++++++++++++++++++++++++++++ gmusicapi/test/server_tests.py | 30 ++---- 3 files changed, 219 insertions(+), 20 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 3b1aad7f..04dd2a08 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -221,6 +221,50 @@ def delete_playlist(self, playlist_id): return res['mutate_response'][0]['id'] + def get_playlist_songs(self, playlist_id, incremental=False, include_deleted=False, + updated_after=None): + """ + Returns a list of dictionaries representing playlist entries. + + :param playlist_id: the id of the playlist to read. + :param incremental: if True, return a generator that yields lists + of at most 1000 entries + as they are retrieved from the server. This can be useful for + presenting a loading bar to a user. + :param include_deleted: if True, include entries that have been deleted + in the past. + :param updated_after: a datetime.datetime; defaults to unix epoch + + Here is an example playlist entry:: + { + u'kind': u'sj#playlistEntry', + u'deleted': False, + u'trackId': u'2bb0ab1c-ce1a-3c0f-9217-a06da207b7a7', + u'lastModifiedTimestamp': u'1325285553655027', + u'playlistId': u'3d72c9b5-baad-4ff7-815d-cdef717e5d61', + u'absolutePosition': u'01729382256910287871', # ?? + u'source': u'1', # ?? + u'creationTimestamp': u'1325285553655027', + u'id': u'c9f1aff5-f93d-4b98-b13a-429cc7972fea' + } + """ + return self._get_all_items(mobileclient.ListPlaylistEntries, + incremental, include_deleted, + updated_after=updated_after) + + @utils.accept_singleton(basestring, 2) + @utils.empty_arg_shortcircuit(position=2) + @utils.enforce_ids_param(position=2) + def add_songs_to_playlist(self, playlist_id, song_ids): + """Appends songs to the end of a playlist. + Returns a list of (song id, playlistEntryId) pairs that were added. + + :param playlist_id: the id of the playlist to add to. + :param song_ids: a list of song ids, or a single song id. + """ + + pass # TODO + def get_all_stations(self, incremental=False, include_deleted=False, updated_after=None): """Returns a list of dictionaries that each represent a radio station. @@ -230,7 +274,7 @@ def get_all_stations(self, incremental=False, include_deleted=False, updated_aft presenting a loading bar to a user. :param include_deleted: if True, include stations that have been deleted in the past. - :param updated_after: a datetime.datetime; defaults to epoch + :param updated_after: a datetime.datetime; defaults to unix epoch Here is an example station dictionary:: { diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index 96b41601..bbb6477f 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -72,6 +72,23 @@ } } +sj_plentry = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'kind': {'type': 'string'}, + 'id': {'type': 'string'}, + 'clientId': {'type': 'string'}, + 'playlistId': {'type': 'string'}, + 'absolutePosition': {'type': 'string'}, + 'trackId': {'type': 'string'}, + 'creationTimestamp': {'type': 'string'}, + 'lastModifiedTimestamp': {'type': 'string'}, + 'deleted': {'type': 'boolean'}, + 'source': {'type': 'string'}, + }, +} + sj_album = { 'type': 'object', 'additionalProperties': False, @@ -405,10 +422,20 @@ class ListPlaylists(McListCall): static_url = sj_url + 'playlistfeed' +class ListPlaylistEntries(McListCall): + item_schema = sj_plentry + filter_text = 'plentries' + + static_method = 'POST' + static_url = sj_url + 'plentryfeed' + + class BatchMutatePlaylists(McBatchMutateCall): static_method = 'POST' static_url = sj_url + 'playlistbatch' + #TODO is it possible to mutate name through this? + @staticmethod def build_playlist_deletes(playlist_ids): #TODO can probably pull this up one @@ -432,6 +459,39 @@ def build_playlist_adds(names): }} for name in names] +class BatchMutatePlaylistEntries(McBatchMutateCall): + filter_text = 'plentries' + item_schema = sj_plentry + + static_method = 'POST' + static_url = sj_url + 'plentriesbatch' + + @staticmethod + def build_plentry_deletes(entry_ids): + """ + :param entry_ids + """ + return [{'delete': id} for id in entry_ids] + + @staticmethod + def build_plentry_adds(playlist_id, song_ids): + """ + :param playlist_id + :param song_ids + """ + + return [{'create': { + 'clientId': '', # ?? + 'creationTimestamp': '-1', + 'deleted': False, + 'lastModifiedTimestamp': '0', + 'playlistId': playlist_id, + # 'precedingEntryId': '', # optional + 'source': 1, + 'trackId': song_id, + }} for song_id in song_ids] + + class ListStations(McListCall): item_schema = sj_station filter_text = 'stations' @@ -440,6 +500,109 @@ class ListStations(McListCall): static_url = sj_url + 'radio/station' +class BatchMutateStations(McBatchMutateCall): + static_method = 'POST' + static_url = sj_url + 'radio/editstation' + + @staticmethod + def build_deletes(station_ids): + """ + :param station_ids + """ + return [{'delete': id, 'includeFeed': False, 'numEntries': 0} + for id in station_ids] + + @staticmethod + def build_adds(names): + """ + :param names + """ + + #TODO + # this has a clientId; need to figure out where that comes from + pass + + +#TODO +class ListStationTracks(McListCall): + pass + #static_headers = {'Content-Type': 'application/json'} + #static_params = {'alt': 'json'} + + #_res_schema = { + # 'type': 'object', + # 'additionalProperties': False, + # 'properties': { + # 'kind': {'type': 'string'}, + # 'nextPageToken': {'type': 'string', 'required': False}, + # 'data': {'type': 'object', + # 'items': {'type': 'array', 'items': item_schema}, + # 'required': False, + # }, + # }, + #} + + #@classmethod + #def dynamic_params(cls, updated_after=None, start_token=None, max_results=None): + # """ + # :param updated_after: datetime.datetime; defaults to epoch + # """ + + # if updated_after is None: + # microseconds = 0 + # else: + # microseconds = utils.datetime_to_microseconds(updated_after) + + # return {'updated-min': microseconds} + + #@classmethod + #def dynamic_data(cls, updated_after=None, start_token=None, max_results=None): + # """ + # :param updated_after: ignored + # :param start_token: nextPageToken from a previous response + # :param max_results: a positive int; if not provided, server defaults to 1000 + # """ + # data = {} + + # if start_token is not None: + # data['start-token'] = start_token + + # if max_results is not None: + # data['max-results'] = str(max_results) + + # return json.dumps(data) + + #@classmethod + #def parse_response(cls, response): + # # empty results don't include the data key + # # make sure it's always there + # res = cls._parse_json(response.text) + # if 'data' not in res: + # res['data'] = {'items': []} + + # return res + + #@classmethod + #def filter_response(cls, msg): + # filtered = copy.deepcopy(msg) + # filtered['data']['items'] = ["<%s %s>" % (len(filtered['data']['items']), + # cls.filter_text)] + # return filtered + + #item_schema = { + # 'type': 'object', + # 'additionalProperties': False, + # 'properties': { + # 'radioId': {'type': 'string'}, + # 'tracks': {'type': 'array', 'items': sj_track}, + # } + #} + #filter_text = 'tracks' + + #static_method = 'POST' + #static_url = sj_url + 'radio/stationfeed' + + class BatchMutateTracks(McBatchMutateCall): static_method = 'POST' static_url = sj_url + 'trackbatch' diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index 04fc7392..fc0630d8 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -7,7 +7,6 @@ an extra test playlist or song may result. """ -from copy import copy from collections import namedtuple import os import re @@ -15,14 +14,12 @@ from decorator import decorator from proboscis.asserts import ( - assert_true, assert_equal, assert_is_not_none, - assert_not_equal, Check, assert_raises + assert_true, assert_equal, + Check ) from proboscis import test, before_class, after_class, SkipTest -from gmusicapi import Webclient, Musicmanager, Mobileclient, CallFailure -#from gmusicapi.exceptions import NotLoggedIn -from gmusicapi.protocol.metadata import md_expectations +from gmusicapi import Webclient, Musicmanager, Mobileclient from gmusicapi.utils.utils import retry import gmusicapi.test.utils as test_utils @@ -257,15 +254,6 @@ def song_delete(self): playlist_test = test(groups=['playlist', 'playlist.exists'], depends_on=[playlist_create]) - # just to make song_test/playlist_test exist for now - @song_test - def st(self): - raise SkipTest('remove this stub test') - - @playlist_test - def pt(self): - raise SkipTest('remove this stub test') - ## Non-wonky tests resume down here. ##--------- @@ -289,22 +277,26 @@ def mc_list_stations_inc_equal(self): def mc_list_stations_inc_equal_with_deleted(self): self.assert_list_inc_equivalence(self.mc.get_all_stations, include_deleted=True) - @test + @song_test def mc_list_songs_inc_equal(self): self.assert_list_inc_equivalence(self.mc.get_all_songs) - @test + @song_test def mc_list_songs_inc_equal_with_deleted(self): self.assert_list_inc_equivalence(self.mc.get_all_songs, include_deleted=True) - @test + @playlist_test def mc_list_playlists_inc_equal(self): self.assert_list_inc_equivalence(self.mc.get_all_playlists) - @test + @playlist_test def mc_list_playlists_inc_equal_with_deleted(self): self.assert_list_inc_equivalence(self.mc.get_all_playlists, include_deleted=True) + @playlist_test + def mc_list_plentries_inc_equal(self): + self.assert_list_inc_equivalence(self.mc.get_playlist_songs, playlist_id=self.playlist_id) + @test @all_access def mc_search_aa(self): From 3b86688eb3e325fa5be3e9f58846d1e0fd5e8008 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 23 Jul 2013 12:43:19 -0400 Subject: [PATCH 64/72] fix up playlist mutation + tests --- gmusicapi/clients/mobileclient.py | 61 ++++++++++++++------ gmusicapi/protocol/mobileclient.py | 40 +++++++++---- gmusicapi/test/server_tests.py | 92 ++++++++++++++++++++++++------ 3 files changed, 148 insertions(+), 45 deletions(-) diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 04dd2a08..5093bd3c 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -1,3 +1,5 @@ +from operator import itemgetter + from gmusicapi import session from gmusicapi.clients.shared import _Base from gmusicapi.protocol import mobileclient @@ -221,19 +223,16 @@ def delete_playlist(self, playlist_id): return res['mutate_response'][0]['id'] - def get_playlist_songs(self, playlist_id, incremental=False, include_deleted=False, - updated_after=None): + def get_all_playlist_contents(self): """ - Returns a list of dictionaries representing playlist entries. + Retrieves the contents of *all* playlists + -- the Mobileclient does not support retrieving + only the contents of one + playlist. - :param playlist_id: the id of the playlist to read. - :param incremental: if True, return a generator that yields lists - of at most 1000 entries - as they are retrieved from the server. This can be useful for - presenting a loading bar to a user. - :param include_deleted: if True, include entries that have been deleted - in the past. - :param updated_after: a datetime.datetime; defaults to unix epoch + Returns the same structure as get_all_playlists, + with the addition of a ``'tracks'`` key in each dict + set to a list of properly-ordered playlist entry dicts. Here is an example playlist entry:: { @@ -248,22 +247,52 @@ def get_playlist_songs(self, playlist_id, incremental=False, include_deleted=Fal u'id': u'c9f1aff5-f93d-4b98-b13a-429cc7972fea' } """ - return self._get_all_items(mobileclient.ListPlaylistEntries, - incremental, include_deleted, - updated_after=updated_after) + + playlists = self.get_all_playlists() + + all_entries = self._get_all_items(mobileclient.ListPlaylistEntries, + incremental=False, include_deleted=False, + updated_after=None) + + for playlist in playlists: + entries = [e for e in all_entries + if e['playlistId'] == playlist['id']] + entries.sort(key=itemgetter('absolutePosition')) + + playlist['tracks'] = entries + + return playlists @utils.accept_singleton(basestring, 2) @utils.empty_arg_shortcircuit(position=2) @utils.enforce_ids_param(position=2) def add_songs_to_playlist(self, playlist_id, song_ids): """Appends songs to the end of a playlist. - Returns a list of (song id, playlistEntryId) pairs that were added. + Returns a list of playlistEntryIds that were added. :param playlist_id: the id of the playlist to add to. :param song_ids: a list of song ids, or a single song id. """ + mutate_call = mobileclient.BatchMutatePlaylistEntries + add_mutations = mutate_call.build_plentry_adds(playlist_id, song_ids) + res = self._make_call(mutate_call, add_mutations) + + return [e['id'] for e in res['mutate_response']] + + @utils.accept_singleton(basestring, 1) + @utils.empty_arg_shortcircuit(position=1) + @utils.enforce_ids_param(position=1) + def remove_entries_from_playlist(self, entry_ids): + """Remove specific entries from a playlist. + Returns a list of entry ids that were removed. + + :param entry_ids: a list of entry ids, or a single entry id. + """ + mutate_call = mobileclient.BatchMutatePlaylistEntries + del_mutations = mutate_call.build_plentry_deletes(entry_ids) + res = self._make_call(mutate_call, del_mutations) - pass # TODO + return [e['id'] for e in res['mutate_response']] def get_all_stations(self, incremental=False, include_deleted=False, updated_after=None): """Returns a list of dictionaries that each represent a radio station. diff --git a/gmusicapi/protocol/mobileclient.py b/gmusicapi/protocol/mobileclient.py index bbb6477f..49a1319a 100644 --- a/gmusicapi/protocol/mobileclient.py +++ b/gmusicapi/protocol/mobileclient.py @@ -9,6 +9,7 @@ import hmac import sys import time +from uuid import uuid1 import validictory @@ -220,7 +221,7 @@ class McListCall(McCall): filter_text = utils.NotImplementedField static_headers = {'Content-Type': 'application/json'} - static_params = {'alt': 'json'} + static_params = {'alt': 'json', 'include-tracks': 'true'} _res_schema = { 'type': 'object', @@ -480,16 +481,33 @@ def build_plentry_adds(playlist_id, song_ids): :param song_ids """ - return [{'create': { - 'clientId': '', # ?? - 'creationTimestamp': '-1', - 'deleted': False, - 'lastModifiedTimestamp': '0', - 'playlistId': playlist_id, - # 'precedingEntryId': '', # optional - 'source': 1, - 'trackId': song_id, - }} for song_id in song_ids] + mutations = [] + + prev_id, cur_id, next_id = None, str(uuid1()), str(uuid1()) + + for i, song_id in enumerate(song_ids): + m_details = { + 'clientId': cur_id, + 'creationTimestamp': '-1', + 'deleted': False, + 'lastModifiedTimestamp': '0', + 'playlistId': playlist_id, + 'source': 1, + 'trackId': song_id, + } + + if song_id.startswith('T'): + m_details['source'] = 2 # AA track + + if i > 0: + m_details['precedingEntryId'] = prev_id + if i < len(song_ids) - 1: + m_details['followingEntryId'] = next_id + + mutations.append({'create': m_details}) + prev_id, cur_id, next_id = cur_id, next_id, str(uuid1()) + + return mutations class ListStations(McListCall): diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index fc0630d8..d5be76c9 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -52,6 +52,16 @@ class UpauthTests(object): #These are set on the instance in create_song/playlist. songs = None # [TestSong] playlist_id = None + plentry_ids = None + + def mc_get_playlist_songs(self, plid): + """For convenience, since mc can only get all playlists at once.""" + all_contents = self.mc.get_all_playlist_contents() + found = [p for p in all_contents if p['id'] == plid] + + assert_true(len(found), 1) + + return found[0]['tracks'] @before_class def login(self): @@ -78,20 +88,21 @@ def logout(self): raise SkipTest('did not create mc') assert_true(self.mc.logout()) - # This next section is a bit odd: it nests playlist tests inside song tests. + # This next section is a bit odd: it orders tests that create + # required resources. - # The intuitition: starting from an empty library, you need to have - # a song before you can modify a playlist. + # The intuitition: starting from an empty library, you need to create + # a song before you can add it to a playlist. # If x --> y means x runs after y, then the graph looks like: - # song_create <-- playlist_create - # ^ ^ - # | | - # song_test playlist_test - # ^ ^ - # | | - # song_delete playlist_delete + # song_create <-- plentry_create --> playlist_create + # ^ ^ ^ + # | | | + # song_test plentry_test playlist_test + # ^ ^ ^ + # | | | + # song_delete plentry_delete playlist_delete # Singleton groups are used to ease code ordering restraints. # Suggestions to improve any of this are welcome! @@ -186,8 +197,7 @@ def song_create(self): self.songs = self.assert_songs_state(sids, present=True) - #TODO pull this out to include song state checking - @test(depends_on=[song_create], runs_after_groups=['song.exists']) + @test def playlist_create(self): playlist_id = self.mc.create_playlist(TEST_PLAYLIST_NAME) @@ -202,9 +212,49 @@ def assert_playlist_exists(plid): assert_playlist_exists(playlist_id) self.playlist_id = playlist_id - #TODO consider listing/searching if the id isn't there - # to ensure cleanup. + @test(depends_on=[playlist_create, song_create], + runs_after_groups=['playlist.exists', 'song.exists']) + def plentry_create(self): + # create 3 entries: the uploaded track and two of the all access track + # 3 songs is the minimum to fully test reordering, and also includes the + # duplicate song_id case + + song_ids = [self.songs[0].sid] + [test_utils.aa_song_id] * 2 + plentry_ids = self.mc.add_songs_to_playlist(self.playlist_id, song_ids) + + @retry(tries=2) + def assert_plentries_exist(plid, plentry_ids): + songs = self.mc_get_playlist_songs(plid) + found = [e for e in songs + if e['id'] in plentry_ids] + + assert_equal(len(found), len(plentry_ids)) + + assert_plentries_exist(self.playlist_id, plentry_ids) + self.plentry_ids = plentry_ids + + @test(groups=['plentry'], depends_on=[plentry_create], + runs_after_groups=['plentry.exists'], + always_run=True) + def plentry_delete(self): + if self.plentry_ids is None: + raise SkipTest('did not store self.plentry_ids') + + res = self.mc.remove_entries_from_playlist(self.plentry_ids) + assert_equal(res, self.plentry_ids) + + @retry + def assert_plentries_removed(plid, entry_ids): + found = [e for e in self.mc_get_playlist_songs(plid) + if e['id'] in entry_ids] + + assert_equal(len(found), 0) + + assert_plentries_removed(self.playlist_id, self.plentry_ids) + #self.assert_list_with_deleted(self.mc_get_playlist_songs) + @test(groups=['playlist'], depends_on=[playlist_create], + runs_after=[plentry_delete], runs_after_groups=['playlist.exists'], always_run=True) def playlist_delete(self): @@ -225,7 +275,7 @@ def assert_playlist_does_not_exist(plid): self.assert_list_with_deleted(self.mc.get_all_playlists) @test(groups=['song'], depends_on=[song_create], - runs_after=[playlist_delete], + runs_after=[plentry_delete], runs_after_groups=["song.exists"], always_run=True) def song_delete(self): @@ -253,6 +303,8 @@ def song_delete(self): song_test = test(groups=['song', 'song.exists'], depends_on=[song_create]) playlist_test = test(groups=['playlist', 'playlist.exists'], depends_on=[playlist_create]) + plentry_test = test(groups=['plentry', 'plentry.exists'], + depends_on=[plentry_create]) ## Non-wonky tests resume down here. @@ -293,9 +345,13 @@ def mc_list_playlists_inc_equal(self): def mc_list_playlists_inc_equal_with_deleted(self): self.assert_list_inc_equivalence(self.mc.get_all_playlists, include_deleted=True) - @playlist_test - def mc_list_plentries_inc_equal(self): - self.assert_list_inc_equivalence(self.mc.get_playlist_songs, playlist_id=self.playlist_id) + @plentry_test + def pt(self): + raise SkipTest('remove this') + + #@plentry_test + #def mc_list_plentries_inc_equal(self): + # self.assert_list_inc_equivalence(self.mc_get_playlist_songs, playlist_id=self.playlist_id) @test @all_access From 6dbe9ecc4675bab6e016ba97399c85ced73f617e Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 23 Jul 2013 12:56:29 -0400 Subject: [PATCH 65/72] don't use AA track without AA account --- gmusicapi/test/server_tests.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/gmusicapi/test/server_tests.py b/gmusicapi/test/server_tests.py index d5be76c9..cc1b1b47 100644 --- a/gmusicapi/test/server_tests.py +++ b/gmusicapi/test/server_tests.py @@ -215,11 +215,18 @@ def assert_playlist_exists(plid): @test(depends_on=[playlist_create, song_create], runs_after_groups=['playlist.exists', 'song.exists']) def plentry_create(self): - # create 3 entries: the uploaded track and two of the all access track + + song_ids = [self.songs[0].sid] + + # create 3 entries # 3 songs is the minimum to fully test reordering, and also includes the # duplicate song_id case + double_id = self.songs[0].sid + if test_all_access_features(): + double_id = test_utils.aa_song_id + + song_ids += [double_id] * 2 - song_ids = [self.songs[0].sid] + [test_utils.aa_song_id] * 2 plentry_ids = self.mc.add_songs_to_playlist(self.playlist_id, song_ids) @retry(tries=2) From 0671e5673e3278b1d8a62092a0aa6adb98418a17 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 23 Jul 2013 14:15:40 -0400 Subject: [PATCH 66/72] mobileclient doc updates --- docs/source/index.rst | 18 ++++---- docs/source/reference/api.rst | 14 ++++-- docs/source/reference/mobileclient.rst | 55 +++++++++++++++++++++++ docs/source/usage.rst | 10 ++--- gmusicapi/clients/mobileclient.py | 61 +++++++++++++++----------- 5 files changed, 114 insertions(+), 44 deletions(-) create mode 100644 docs/source/reference/mobileclient.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 2b34a9a6..433ba111 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,17 +6,17 @@ This library allows control of .. code-block:: python - from gmusicapi import Webclient - - api = Webclient() + from gmusicapi import Mobileclient + + api = Mobileclient() api.login('user@gmail.com', 'my-password') # => True - + library = api.get_all_songs() sweet_tracks = [track for track in library if track['artist'] == 'The Cat Empire'] - + playlist_id = api.create_playlist('Rad muzak') - api.change_playlist(playlist_id, sweet_tracks) + api.add_songs_to_playlist(playlist_id, sweet_tracks) **This project is not supported nor endorsed by Google.** @@ -30,11 +30,11 @@ All major functionality is supported: - Library management: list, create, delete, and modify songs and playlists -- Webclient streaming and single-song downloading +- Streaming and single-song downloading - Music Manager uploading/scan-and-match and library downloading -Support for Google Play Music All Access features is currently under consideration. +Support for Google Play Music All Access features is in progress. See `the changelog `__ @@ -52,7 +52,7 @@ Using gmusicapi Getting started +++++++++++++++ -The :ref:`usage section ` has information on installing +The :ref:`usage section ` has installation instructions and some simple examples. Api and data reference diff --git a/docs/source/reference/api.rst b/docs/source/reference/api.rst index 0eae8796..819b2379 100644 --- a/docs/source/reference/api.rst +++ b/docs/source/reference/api.rst @@ -4,12 +4,17 @@ Client Interfaces ================= -gmusicapi has two main interfaces: one for the music.google.com webclient, and +gmusicapi currently has three main interfaces: +one for the music.google.com webclient, +one for the Android App, and one for the Music Manager. The big differences are: -* :py:class:`Musicmanager` is used only for uploading, while :py:class:`Webclient` - supports everything but uploading. -* :py:class:`Webclient` requires a plaintext email and password to login, while +* :py:class:`Webclient` development has ceased, and the :py:class:`Mobileclient` + will take its place +* :py:class:`Musicmanager` is used only for uploading, while + :py:class:`Webclient`/:py:class:`Mobileclient` + support everything but uploading. +* :py:class:`Webclient`/:py:class:`Mobileclient` require a plaintext email and password to login, while :py:class:`Musicmanager` uses `OAuth2 `__. @@ -17,4 +22,5 @@ one for the Music Manager. The big differences are: :maxdepth: 2 webclient + mobileclient musicmanager diff --git a/docs/source/reference/mobileclient.rst b/docs/source/reference/mobileclient.rst new file mode 100644 index 00000000..3b139df3 --- /dev/null +++ b/docs/source/reference/mobileclient.rst @@ -0,0 +1,55 @@ +.. _mobileclient: +.. currentmodule:: gmusicapi.clients + +Mobileclient Interface +====================== + +.. autoclass:: Mobileclient + +Setup and login +--------------- +.. automethod:: Mobileclient.__init__ +.. automethod:: Mobileclient.login +.. automethod:: Mobileclient.logout + +Songs +----- +Songs are uniquely referred to within a library +with a 'song id' or 'track id' uuid. + +.. automethod:: Mobileclient.get_all_songs +.. automethod:: Mobileclient.get_stream_url +.. automethod:: Mobileclient.delete_songs + +Playlists +--------- +Like songs, playlists have unique ids within a library. +However, their names do not need to be unique. + +The tracks making up a playlist are referred to as +'playlist entries', and have unique entry ids within the +entire library (not just their containing playlist). + +.. automethod:: Mobileclient.get_all_playlists +.. automethod:: Mobileclient.get_all_playlist_contents +.. automethod:: Mobileclient.create_playlist +.. automethod:: Mobileclient.delete_playlist +.. automethod:: Mobileclient.add_songs_to_playlist +.. automethod:: Mobileclient.remove_entries_from_playlist + +All Access features +------------------- +All Access/store tracks also have track ids, but they are in a different +form from normal track ids. +``store_id.beginswith('T')`` always holds for these ids (and will not +for library track ids). + +Adding a store track to a library will yield a normal song id. + +All Access track ids can be used in most places that normal song ids can +(e.g. when for playlist addition or streaming). + +.. automethod:: Mobileclient.search_all_access +.. automethod:: Mobileclient.add_aa_track +.. automethod:: Mobileclient.get_all_stations +.. automethod:: Mobileclient.get_artist_info diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 3b8e7955..ba2f2603 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -30,14 +30,14 @@ If you need to install avconv from source, be sure to use Quickstart ---------- -If you're not going to be uploading music, use the :py:class:`Webclient`. +If you're not going to be uploading music, use the :py:class:`Mobileclient`. This requires plaintext auth, so your code might look something like: .. code-block:: python - from gmusicapi import Webclient + from gmusicapi import Mobileclient - api = Webclient() + api = Mobileclient() logged_in = api.login('user@gmail.com', 'my-password') # logged_in is True if login was successful @@ -49,7 +49,7 @@ It uses `OAuth2 does not require plaintext credentials. Instead, you'll need to authorize your account *once* before logging in. -The easiest way is to follow the prompts from: +The easiest way is to run: .. code-block:: python @@ -59,7 +59,7 @@ The easiest way is to follow the prompts from: mm.perform_oauth() If successful, this will save your credentials to disk. -Then, future runs will start with: +Then, future runs can start with: .. code-block:: python diff --git a/gmusicapi/clients/mobileclient.py b/gmusicapi/clients/mobileclient.py index 5093bd3c..32698dbb 100644 --- a/gmusicapi/clients/mobileclient.py +++ b/gmusicapi/clients/mobileclient.py @@ -48,11 +48,13 @@ def get_all_songs(self, incremental=False, include_deleted=False): of at most 1000 tracks as they are retrieved from the server. This can be useful for presenting a loading bar to a user. + :param include_deleted: if True, include tracks that have been deleted in the past. Here is an example song dictionary:: - { + + { u'comment':u'', u'rating':u'0', u'albumArtRef':[ @@ -94,6 +96,7 @@ def get_all_songs(self, incremental=False, include_deleted=False): u'clientId':u'+eGFGTbiyMktbPuvB5MfsA', u'durationMillis':u'418000' } + """ tracks = self._get_all_items(mobileclient.ListTracks, incremental, include_deleted) @@ -176,19 +179,20 @@ def get_all_playlists(self, incremental=False, include_deleted=False): in the past. Here is an example playlist dictionary:: + { - u 'kind': u 'sj#playlist', - u 'name': u 'Something Mix', - u 'deleted': False, - u 'type': u 'MAGIC', # if not present, playlist is user-created - u 'lastModifiedTimestamp': u '1325458766483033', - u 'recentTimestamp': u '1325458766479000', - u 'shareToken': u '', - u 'ownerProfilePhotoUrl': u 'http://lh3.googleusercontent.com/...', - u 'ownerName': u 'Simon Weber', - u 'accessControlled': False, # something to do with shared playlists? - u 'creationTimestamp': u '1325285553626172', - u 'id': u '3d72c9b5-baad-4ff7-815d-cdef717e5d61' + u'kind': u'sj#playlist', + u'name': u'Something Mix', + u'deleted': False, + u'type': u'MAGIC', # if not present, playlist is user-created + u'lastModifiedTimestamp': u'1325458766483033', + u'recentTimestamp': u'1325458766479000', + u'shareToken': u'', + u'ownerProfilePhotoUrl': u'http://lh3.googleusercontent.com/...', + u'ownerName': u'Simon Weber', + u'accessControlled': False, # something to do with shared playlists? + u'creationTimestamp': u'1325285553626172', + u'id': u'3d72c9b5-baad-4ff7-815d-cdef717e5d61' } """ @@ -230,11 +234,12 @@ def get_all_playlist_contents(self): only the contents of one playlist. - Returns the same structure as get_all_playlists, + Returns the same structure as :func:`get_all_playlists`, with the addition of a ``'tracks'`` key in each dict set to a list of properly-ordered playlist entry dicts. Here is an example playlist entry:: + { u'kind': u'sj#playlistEntry', u'deleted': False, @@ -306,22 +311,23 @@ def get_all_stations(self, incremental=False, include_deleted=False, updated_aft :param updated_after: a datetime.datetime; defaults to unix epoch Here is an example station dictionary:: + { - u 'imageUrl': u 'http://lh6.ggpht.com/...', - u 'kind': u 'sj#radioStation', - u 'name': u 'station', - u 'deleted': False, - u 'lastModifiedTimestamp': u '1370796487455005', - u 'recentTimestamp': u '1370796487454000', - u 'clientId': u 'c2639bf4-af24-4e4f-ab37-855fc89d15a1', - u 'seed': + u'imageUrl': u'http://lh6.ggpht.com/...', + u'kind': u'sj#radioStation', + u'name': u'station', + u'deleted': False, + u'lastModifiedTimestamp': u'1370796487455005', + u'recentTimestamp': u'1370796487454000', + u'clientId': u'c2639bf4-af24-4e4f-ab37-855fc89d15a1', + u'seed': { - u 'kind': u 'sj#radioSeed', - u 'trackLockerId': u '7df3aadd-9a18-3dc1-b92e-a7cf7619da7e' + u'kind': u'sj#radioSeed', + u'trackLockerId': u'7df3aadd-9a18-3dc1-b92e-a7cf7619da7e' # possible keys: # albumId, artistId, genreId, trackId, trackLockerId }, - u 'id': u '69f1bfce-308a-313e-9ed2-e50abe33a25d' + u'id': u'69f1bfce-308a-313e-9ed2-e50abe33a25d' }, """ return self._get_all_items(mobileclient.ListStations, incremental, include_deleted, @@ -336,7 +342,9 @@ def search_all_access(self, query, max_results=50): :param query: a string keyword to search with. Capitalization and punctuation are ignored. :param max_results: Maximum number of items to be retrieved - The results are returned in a dictionary, arranged by how they were found, eg:: + The results are returned in a dictionary, arranged by how they were found. + Here are example results for a search on ``'Amorphis'``:: + { 'album_hits':[ { @@ -461,6 +469,7 @@ def get_artist_info(self, artist_id, include_albums=True, max_top_tracks=5, max_ CallFailure being raised. Returns a dict, eg:: + { u'albums':[ # only if include_albums is True { From 55856159da44fd88b714388cdd2b612b91a71564 Mon Sep 17 00:00:00 2001 From: Simon Weber Date: Tue, 23 Jul 2013 14:38:47 -0400 Subject: [PATCH 67/72] webclient cleanup, doc updates (close #144 #145) --- HISTORY.rst | 2 + README.rst | 18 +-- docs/source/reference/webclient.rst | 5 - example.py | 36 ++--- gmusicapi/clients/webclient.py | 238 +--------------------------- gmusicapi/protocol/webclient.py | 89 ----------- protocol_info | 161 ------------------- 7 files changed, 30 insertions(+), 519 deletions(-) delete mode 100644 protocol_info diff --git a/HISTORY.rst b/HISTORY.rst index 145c6bbe..91d993a5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ As of 1.0.0, `semantic versioning `__ is used. +++++++++ released 2013-XX-XX +- add Mobileclient +- remove broken Webclient.{create_playlist, change_playlist, copy_playlist, search, change_playlist_name} - add support for streaming All Access songs - add Webclient.get_registered_devices - add a toggle to turn off validation per client diff --git a/README.rst b/README.rst index afea4b16..78b14e9d 100644 --- a/README.rst +++ b/README.rst @@ -4,20 +4,19 @@ gmusicapi: an unofficial API for Google Play Music gmusicapi allows control of `Google Music `__ with Python. - .. code-block:: python - from gmusicapi import Webclient + from gmusicapi import Mobileclient - api = Webclient() + api = Mobileclient() api.login('user@gmail.com', 'my-password') # => True - + library = api.get_all_songs() sweet_tracks = [track for track in library if track['artist'] == 'The Cat Empire'] - + playlist_id = api.create_playlist('Rad muzak') - api.change_playlist(playlist_id, sweet_tracks) + api.add_songs_to_playlist(playlist_id, sweet_tracks) **gmusicapi is not supported nor endorsed by Google.** @@ -49,10 +48,11 @@ Status and updates .. image:: https://travis-ci.org/simon-weber/Unofficial-Google-Music-API.png?branch=develop :target: https://travis-ci.org/simon-weber/Unofficial-Google-Music-API -The Webclient interface has gotten horrible to maintain lately, so I'm currently working on +The project is in the middle of a major change at the moment: the Webclient interface has +gotten horrible to maintain, so I'm working on switching the the Android app api. This will provide easy All Access support and easier -maintainability going forward. Expect this release before August -- you can follow along -`here `__. +maintainability going forward. At this point, prefer the Mobileclient to the Webclient +whenever possible. Version 1.2.0 fixes a bug that fixes uploader_id formatting from a mac address. This change may cause another machine to be registered - you can safely remove the diff --git a/docs/source/reference/webclient.rst b/docs/source/reference/webclient.rst index f18edd13..3af7d41f 100644 --- a/docs/source/reference/webclient.rst +++ b/docs/source/reference/webclient.rst @@ -17,7 +17,6 @@ Getting songs and playlists .. automethod:: Webclient.get_all_songs .. automethod:: Webclient.get_all_playlist_ids .. automethod:: Webclient.get_playlist_songs -.. automethod:: Webclient.search Song downloading and streaming ------------------------------ @@ -34,14 +33,10 @@ Song manipulation Playlist manipulation --------------------- -.. automethod:: Webclient.create_playlist -.. automethod:: Webclient.change_playlist_name -.. automethod:: Webclient.copy_playlist .. automethod:: Webclient.delete_playlist Playlist content manipulation ----------------------------- -.. automethod:: Webclient.change_playlist .. automethod:: Webclient.add_songs_to_playlist .. automethod:: Webclient.remove_songs_from_playlist diff --git a/example.py b/example.py index 887c45b2..fc18eda4 100644 --- a/example.py +++ b/example.py @@ -3,7 +3,7 @@ from getpass import getpass -from gmusicapi import Webclient +from gmusicapi import Mobileclient def ask_for_credentials(): @@ -11,14 +11,14 @@ def ask_for_credentials(): Return the authenticated api. """ - # We're not going to upload anything, so the webclient is what we want. - api = Webclient() + # We're not going to upload anything, so the Mobileclient is what we want. + api = Mobileclient() logged_in = False attempts = 0 while not logged_in and attempts < 3: - email = raw_input("Email: ") + email = raw_input('Email: ') password = getpass() logged_in = api.login(email, password) @@ -36,56 +36,56 @@ def demonstrate(): print "Sorry, those credentials weren't accepted." return - print "Successfully logged in." + print 'Successfully logged in.' print # Get all of the users songs. # library is a big list of dictionaries, each of which contains a single song. - print "Loading library...", + print 'Loading library...', library = api.get_all_songs() - print "done." + print 'done.' - print len(library), "tracks detected." + print len(library), 'tracks detected.' print # Show some info about a song. There is no guaranteed order; # this is essentially a random song. first_song = library[0] print "The first song I see is '{}' by '{}'.".format( - first_song["name"].encode('utf-8'), - first_song["artist"].encode('utf-8')) + first_song['title'].encode('utf-8'), + first_song['artist'].encode('utf-8')) # We're going to create a new playlist and add a song to it. # Songs are uniquely identified by 'song ids', so let's get the id: - song_id = first_song["id"] + song_id = first_song['id'] print "I'm going to make a new playlist and add that song to it." print "I'll delete it when we're finished." print - playlist_name = raw_input("Enter a name for the playlist: ") + playlist_name = raw_input('Enter a name for the playlist: ') # Like songs, playlists have unique ids. # Google Music allows more than one playlist of the same name; # these ids are necessary. playlist_id = api.create_playlist(playlist_name) - print "Made the playlist." + print 'Made the playlist.' print # Now let's add the song to the playlist, using their ids: api.add_songs_to_playlist(playlist_id, song_id) - print "Added the song to the playlist." + print 'Added the song to the playlist.' print # We're all done! The user can now go and see that the playlist is there. # The web client syncs our changes in real time. - raw_input("You can now check on Google Music that the playlist exists.\n" - "When done, press enter to delete the playlist:") + raw_input('You can now check on Google Music that the playlist exists.\n' + 'When done, press enter to delete the playlist:') api.delete_playlist(playlist_id) - print "Deleted the playlist." + print 'Deleted the playlist.' # It's good practice to logout when finished. api.logout() - print "All done!" + print 'All done!' if __name__ == '__main__': demonstrate() diff --git a/gmusicapi/clients/webclient.py b/gmusicapi/clients/webclient.py index 3fa40f70..98d2b989 100644 --- a/gmusicapi/clients/webclient.py +++ b/gmusicapi/clients/webclient.py @@ -1,12 +1,9 @@ # -*- coding: utf-8 -*- -import copy from urlparse import urlparse, parse_qsl import gmusicapi from gmusicapi.clients.shared import _Base -from gmusicapi.gmtools import tools -from gmusicapi.exceptions import CallFailure from gmusicapi.protocol import webclient from gmusicapi.utils import utils import gmusicapi.session @@ -74,17 +71,6 @@ def get_registered_devices(self): res = self._make_call(webclient.GetSettings, '') return res['settings']['devices'] - def change_playlist_name(self, playlist_id, new_name): - """Changes the name of a playlist. Returns the changed id. - - :param playlist_id: id of the playlist to rename. - :param new_title: desired title. - """ - - self._make_call(webclient.ChangePlaylistName, playlist_id, new_name) - - return playlist_id # the call actually doesn't return anything. - @utils.accept_singleton(dict) @utils.empty_arg_shortcircuit def change_song_metadata(self, songs): @@ -115,24 +101,6 @@ def change_song_metadata(self, songs): return [s['id'] for s in res['songs']] - def create_playlist(self, name): - """Creates a new playlist. Returns the new playlist id. - - :param title: the title of the playlist to create. - """ - - return self._make_call(webclient.AddPlaylist, name)['id'] - - def delete_playlist(self, playlist_id): - """Deletes a playlist. Returns the deleted id. - - :param playlist_id: id of the playlist to delete. - """ - - res = self._make_call(webclient.DeletePlaylist, playlist_id) - - return res['deleteId'] - @utils.accept_singleton(basestring) @utils.empty_arg_shortcircuit @utils.enforce_ids_param @@ -345,146 +313,6 @@ def get_stream_audio(self, song_id): return ''.join(stream_pieces) - def copy_playlist(self, playlist_id, copy_name): - """Copies the contents of a playlist to a new playlist. Returns the id of the new playlist. - - :param playlist_id: id of the playlist to be copied. - :param copy_name: the name of the new copied playlist. - - This is useful for making backups of playlists before modifications. - """ - - orig_tracks = self.get_playlist_songs(playlist_id) - - new_id = self.create_playlist(copy_name) - self.add_songs_to_playlist(new_id, [t["id"] for t in orig_tracks]) - - return new_id - - def change_playlist(self, playlist_id, desired_playlist, safe=True): - """Changes the order and contents of an existing playlist. - Returns the id of the playlist when finished - - this may be the same as the argument in the case of a failure and recovery. - - :param playlist_id: the id of the playlist being modified. - :param desired_playlist: the desired contents and order as a list of - :ref:`song dictionaries `, like is returned - from :func:`get_playlist_songs`. - - :param safe: if ``True``, ensure playlists will not be lost if a problem occurs. - This may slow down updates. - - The server only provides 3 basic playlist mutations: addition, deletion, and reordering. - This function will use these to automagically apply the desired changes. - - However, this might involve multiple calls to the server, and if a call fails, - the playlist will be left in an inconsistent state. - The ``safe`` option makes a backup of the playlist before doing anything, so it can be - rolled back if a problem occurs. This is enabled by default. - This might slow down updates of very large playlists. - - There will always be a warning logged if a problem occurs, even if ``safe`` is ``False``. - """ - - #We'll be modifying the entries in the playlist, and need to copy it. - #Copying ensures two things: - # 1. the user won't see our changes - # 2. changing a key for one entry won't change it for another - which would be the case - # if the user appended the same song twice, for example. - desired_playlist = [copy.deepcopy(t) for t in desired_playlist] - server_tracks = self.get_playlist_songs(playlist_id) - - if safe: - #Make a backup. - #The backup is stored on the server as a new playlist with "_gmusicapi_backup" - # appended to the backed up name. - names_to_ids = self.get_all_playlist_ids()['user'] - playlist_name = (ni_pair[0] - for ni_pair in names_to_ids.iteritems() - if playlist_id in ni_pair[1]).next() - - backup_id = self.copy_playlist(playlist_id, playlist_name + u"_gmusicapi_backup") - - try: - #Counter, Counter, and set of id pairs to delete, add, and keep. - to_del, to_add, to_keep = \ - tools.find_playlist_changes(server_tracks, desired_playlist) - - ##Delete unwanted entries. - to_del_eids = [pair[1] for pair in to_del.elements()] - if to_del_eids: - self._remove_entries_from_playlist(playlist_id, to_del_eids) - - ##Add new entries. - to_add_sids = [pair[0] for pair in to_add.elements()] - if to_add_sids: - new_pairs = self.add_songs_to_playlist(playlist_id, to_add_sids) - - ##Update desired tracks with added tracks server-given eids. - #Map new sid -> [eids] - new_sid_to_eids = {} - for sid, eid in new_pairs: - if not sid in new_sid_to_eids: - new_sid_to_eids[sid] = [] - new_sid_to_eids[sid].append(eid) - - for d_t in desired_playlist: - if d_t["id"] in new_sid_to_eids: - #Found a matching sid. - match = d_t - sid = match["id"] - eid = match.get("playlistEntryId") - pair = (sid, eid) - - if pair in to_keep: - to_keep.remove(pair) # only keep one of the to_keep eids. - else: - match["playlistEntryId"] = new_sid_to_eids[sid].pop() - if len(new_sid_to_eids[sid]) == 0: - del new_sid_to_eids[sid] - - ##Now, the right eids are in the playlist. - ##Set the order of the tracks: - - #The web client has no way to dictate the order without block insertion, - # but the api actually supports setting the order to a given list. - #For whatever reason, though, it needs to be set backwards; might be - # able to get around this by messing with afterEntry and beforeEntry parameters. - if desired_playlist: - #can't *-unpack an empty list - sids, eids = zip(*tools.get_id_pairs(desired_playlist[::-1])) - - if sids: - self._make_call(webclient.ChangePlaylistOrder, playlist_id, sids, eids) - - ##Clean up the backup. - if safe: - self.delete_playlist(backup_id) - - except CallFailure: - self.logger.info("a subcall of change_playlist failed - " - "playlist %s is in an inconsistent state", playlist_id) - - if not safe: - raise # there's nothing we can do - else: # try to revert to the backup - self.logger.info("attempting to revert changes from playlist " - "'%s_gmusicapi_backup'", playlist_name) - - try: - self.delete_playlist(playlist_id) - self.change_playlist_name(backup_id, playlist_name) - except CallFailure: - self.logger.warning("failed to revert failed change_playlist call on '%s'", - playlist_name) - raise - else: - self.logger.info("reverted changes safely; playlist id of '%s' is now '%s'", - playlist_name, backup_id) - playlist_id = backup_id - - return playlist_id - @utils.accept_singleton(basestring, 2) @utils.empty_arg_shortcircuit(position=2) @utils.enforce_ids_param(position=2) @@ -511,9 +339,7 @@ def remove_songs_from_playlist(self, playlist_id, sids_to_match): :param sids_to_match: a list of song ids to match, or a single song id. This does *not always* the inverse of a call to :func:`add_songs_to_playlist`, - since multiple copies of the same song are removed. For more control in this case, - get the playlist tracks with :func:`get_playlist_songs`, modify the list of tracks, - then use :func:`change_playlist` to push changes to the server. + since multiple copies of the same song are removed. """ playlist_tracks = self.get_playlist_songs(playlist_id) @@ -560,68 +386,6 @@ def _remove_entries_from_playlist(self, playlist_id, entry_ids_to_remove): return res['deleteIds'] - def search(self, query): - """Queries the server for songs and albums. - - **WARNING**: Google no longer uses this endpoint in their client; - it may stop working or be removed from gmusicapi without warning. - In addition, it is known to occasionally return unexpected results. - See `#114 - `__ - for more information. - - Instead of using this call, retrieve all tracks with :func:`get_all_songs` - and search them locally. `This gist - `__ has some examples of - simple linear-time searches. - - :param query: a string keyword to search with. Capitalization and punctuation are ignored. - - The results are returned in a dictionary, arranged by how they were found. - ``artist_hits`` and ``song_hits`` return a list of - :ref:`song dictionaries `, while ``album_hits`` entries - have a different structure. - - For example, a search on ``'cat'`` could return:: - - { - "album_hits": [ - { - "albumArtist": "The Cat Empire", - "albumName": "Cities: The Cat Empire Project", - "artistName": "The Cat Empire", - "imageUrl": "//ssl.gstatic.com/music/fe/[...].png" - # no more entries - }, - ], - "artist_hits": [ - { - "album": "Cinema", - "artist": "The Cat Empire", - "id": "c9214fc1-91fa-3bd2-b25d-693727a5f978", - "title": "Waiting" - # ... normal song dictionary - }, - ], - "song_hits": [ - { - "album": "Mandala", - "artist": "RX Bandits", - "id": "a7781438-8ec3-37ab-9c67-0ddb4115f60a", - "title": "Breakfast Cat", - # ... normal song dictionary - }, - ] - } - - """ - - res = self._make_call(webclient.Search, query)['results'] - - return {"album_hits": res["albums"], - "artist_hits": res["artists"], - "song_hits": res["songs"]} - @utils.accept_singleton(basestring) @utils.empty_arg_shortcircuit @utils.enforce_id_param diff --git a/gmusicapi/protocol/webclient.py b/gmusicapi/protocol/webclient.py index 4ffe336d..0a55fa3f 100644 --- a/gmusicapi/protocol/webclient.py +++ b/gmusicapi/protocol/webclient.py @@ -118,32 +118,6 @@ def parse_response(cls, response): return cls._parse_json(response.text) -class AddPlaylist(WcCall): - """Creates a new playlist.""" - - static_method = 'POST' - static_url = service_url + 'addplaylist' - - _res_schema = { - "type": "object", - "properties": { - "id": {"type": "string"}, - "title": {"type": "string"}, - "success": {"type": "boolean"}, - "timestamp": {"type": "integer"}, - "token": {"type": "string", "blank": True}, - }, - "additionalProperties": False - } - - @staticmethod - def dynamic_data(title): - """ - :param title: the title of the playlist to create. - """ - return {'json': json.dumps({"title": title})} - - class AddToPlaylist(WcCall): """Adds songs to a playlist.""" static_method = 'POST' @@ -189,31 +163,6 @@ def filter_response(msg): return filtered -class ChangePlaylistName(WcCall): - """Changes the name of a playlist.""" - - static_method = 'POST' - static_url = service_url + 'modifyplaylist' - - _res_schema = { - "type": "object", - "properties": {}, - "additionalProperties": False - } - - @staticmethod - def dynamic_data(playlist_id, new_name): - """ - :param playlist_id: id of the playlist to rename. - :param new_title: desired title. - """ - return { - 'json': json.dumps( - {"playlistId": playlist_id, "playlistName": new_name} - ) - } - - class ChangePlaylistOrder(WcCall): """Reorder existing tracks in a playlist.""" @@ -545,44 +494,6 @@ def dynamic_params(song_id): return params -class Search(WcCall): - """Fuzzily search for songs, artists and albums. - Not needed for most use-cases; local search is usually faster and more flexible""" - - static_method = 'POST' - static_url = service_url + 'search' - - _res_schema = { - "type": "object", - "properties": { - "results": { - "type": "object", - "properties": { - "artists": song_array, # hits on artists - "songs": song_array, # hits on tracks - "albums": { # hits on albums; no track info returned - "type": "array", - "items": { - "type": "object", - "properties": { - "artistName": {"type": "string", "blank": True}, - "imageUrl": {"type": "string", "required": False}, - "albumArtist": {"type": "string", "blank": True}, - "albumName": {"type": "string"}, - } - } - } - } - } - }, - "additionalProperties": False - } - - @staticmethod - def dynamic_data(query): - return {'json': json.dumps({'q': query})} - - class ReportBadSongMatch(WcCall): """Request to signal the uploader to reupload a matched track.""" diff --git a/protocol_info b/protocol_info deleted file mode 100644 index 065c93d4..00000000 --- a/protocol_info +++ /dev/null @@ -1,161 +0,0 @@ -Protocol Information: - - #-------------------------------------------------# - This file is obsolete, and might be out of date. - Check protocol.py for current information. - #-------------------------------------------------# - - - Calls are made to music.google.com/music/services/ - - Calls also need to send 'u' (I've only seen it =0) and 'xt' (=the same as the cookie) in the url. A sample url: - - https://music.google.com/music/services/search?u=0&xt=AM-WbXjYA_Y1LgZx_znahg9rOeIg3aDtWg: - - - == Implemented Calls == - - req - request - res - response on a success - - addplaylist: - req: {"title": ""} - res: {"id":"","title":"","success":true} - - addtoplaylist: - req: {"playlistId":"","songIds":[""]} - res: {"playlistId":"","songIds":[{"playlistEntryId":"","songId":""}]} - - deleteplaylist: - req: {"id": ""} - res: {"deleteID": ""} - - deletesong: - delete from library: - req: {"songIds": ["", ""], "entryIds":[""], "listId": "all"} - res: {"listId":"all","deleteIds":[""]} - - delete from playlist: - same as from library, but given entryIds (from loadplaylist) and a listId. - - loadalltracks: - Libraries can be big, so GM sends the tracks down in 'chunks'. - Requests that don't complete the library have 'continuation tokens' required to get the next chunk. - - req: - first: {} - continuations: {"continuationToken":""} - res: - {"continuation": , - "continuationToken": "", - "differentialUpdate": , - "playlist": [ {}, {}... ], - "playlistID": "all", - "requestTime": - } - - loadplaylist: - Loads the songs from a playlist. - req: {"id": ""} - res: {"continuation": False, //never seen true, but my playlists are small. likely works like loadalltracks - "playlist": [{}, {}], //songs also include an entryId - "playlistId": "", - "unavailableTrackCount": } - - modifyentries: - Edits song metadata. - - There are a lot of things to be careful of when editing metadata. - Everything known is in protocol.py.WC_Protocol under Metadata Expectations. - You should be safe if you stick to changing these keys: - - rating to one of: - 0 - no thumb - 1 - down thumb - 5 - up thumb - - name - _don't_ use title. title is reset to whatever name is. - - album - albumArtist - artist - composer - disc - genre - playCount - totalDiscs - totalTracks - track - year - - Also note that the server response is _not_ to be trusted. - Reload the entire library, then re-read tracks to see updates. - - req: {"entries":[ {}, ...]} - res: {"songs":[ {}, ...], - "success":true} - - modifyplaylist: - Changes the title of a playlist. - - req: {"playlistId": "", "playlistName": ""} - res: {} - note the lack of success entry here; this is normal. - - multidownload: - Gets download links. Only tested with 1 id, but it likely supports multiple. - - req: {"songIds":[<id>"]} - res: {"downloadCounts": {"<id>":<times it has been downloaded>}, - "url":"<download url>"} - - play: - Gets a url that holds a streamable copy of this song. Authentication is _not_ needed to stream this url. - This call is unique in that it does not use /services. Its complete url is music.google.com/music/play - In addition, it has an empty request body. The songid is passed in the querystring, along with u=0 and pt=e (not sure what pt signifies). - - req: <completely empty> - res: {"url":"<url that holds audio file>"} - - search: - req: {"q": "<query>"} - res: {"results":{"artists":[<hits>],"albums":[<hits>],"songs":[<hits>]}} - - - == Song Metadata Example == - (see protocol.WC_Protocol metadata expectations for more information) - - Song metadata is sent in dictionaries. - - All songs in my library have either 27 or 28 keys. Here's an example: - - {'comment': '' - 'rating': 0 - 'lastPlayed': 1324954872637533L - 'disc': 1 - 'composer': '' - 'year': 2009 - 'id': '305a7b83-32fa-3a71-9a77-498dfce74aad' - 'album': 'Live on Earth' - 'title': 'The Car Song' - 'deleted': False - 'albumArtist': 'The Cat Empire' - 'type': 2 - 'titleNorm': 'the car song' - 'track': 2 - 'albumArtistNorm': 'the cat empire' - 'totalTracks': 0 - 'beatsPerMinute': 0 - 'genre': 'Alternative' - 'playCount': 0 - 'creationDate': 1324614519429366L - 'name': 'The Car Song' - 'albumNorm': 'live on earth' - 'artist': 'The Cat Empire' - 'url': '' - 'totalDiscs': 2 - 'durationMillis': 562000 - 'artistNorm': 'the cat empire', - - (optional entry; exists if there is album art) - 'albumArtUrl': '//lh6.googleusercontent.com/<long identifier>' - } \ No newline at end of file From 3fde1a159cfc5ecb2297dd6c7e876d6fe44756d6 Mon Sep 17 00:00:00 2001 From: Simon Weber <simon@simonmweber.com> Date: Wed, 24 Jul 2013 16:20:03 -0400 Subject: [PATCH 68/72] matchedId is optional metadata (#148) --- gmusicapi/protocol/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gmusicapi/protocol/metadata.py b/gmusicapi/protocol/metadata.py index 7a9b03e6..7b7f424a 100644 --- a/gmusicapi/protocol/metadata.py +++ b/gmusicapi/protocol/metadata.py @@ -132,7 +132,6 @@ def get_schema(self): "the server does not calculate this - it's just what was in track metadata"), ('subjectToCuration', 'boolean', 'meaning unknown.'), - ('matchedId', 'string', 'meaning unknown; related to scan and match?'), ('curatedByUser', 'boolean', 'meaning unknown'), ('curationSuggested', 'boolean', 'meaning unknown'), ) @@ -154,6 +153,7 @@ def get_schema(self): ('artistImageBaseUrl', 'string', 'like albumArtUrl, but for the artist. May be blank.'), ('recentTimestamp', 'integer', 'UTC/microsecond timestamp: meaning unknown.'), ('deleted', 'boolean', ''), + ('matchedId', 'string', 'meaning unknown; related to scan and match?'), ) ] + [ Expectation(name + 'Norm', 'string', mutable=False, optional=False, From 58d1e2d8bb197fcfb2f98cbc85d6ca5d3fe2e3c3 Mon Sep 17 00:00:00 2001 From: Simon Weber <simon@simonmweber.com> Date: Thu, 1 Aug 2013 14:19:52 -0400 Subject: [PATCH 69/72] clarify addition of mobileclient --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 91d993a5..f8c9c100 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,8 +9,8 @@ As of 1.0.0, `semantic versioning <http://semver.org/>`__ is used. +++++++++ released 2013-XX-XX -- add Mobileclient - remove broken Webclient.{create_playlist, change_playlist, copy_playlist, search, change_playlist_name} +- add Mobileclient; this will slowly replace most of the Webclient, so prefer it when possible - add support for streaming All Access songs - add Webclient.get_registered_devices - add a toggle to turn off validation per client From 675af1b4570bbf6e55d0014326b7b3e1020aec97 Mon Sep 17 00:00:00 2001 From: Simon Weber <simon@simonmweber.com> Date: Thu, 1 Aug 2013 14:35:07 -0400 Subject: [PATCH 70/72] update version --- HISTORY.rst | 4 ++-- gmusicapi/_version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f8c9c100..c3e39f23 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,9 +5,9 @@ History As of 1.0.0, `semantic versioning <http://semver.org/>`__ is used. -1.2.0-dev +2.0.0 +++++++++ -released 2013-XX-XX +released 2013-08-01 - remove broken Webclient.{create_playlist, change_playlist, copy_playlist, search, change_playlist_name} - add Mobileclient; this will slowly replace most of the Webclient, so prefer it when possible diff --git a/gmusicapi/_version.py b/gmusicapi/_version.py index ca5babe0..8c0d5d5b 100644 --- a/gmusicapi/_version.py +++ b/gmusicapi/_version.py @@ -1 +1 @@ -__version__ = "1.2.0-dev" +__version__ = "2.0.0" From b1dd7ee7d75526a78d9077c89da069bb327beff1 Mon Sep 17 00:00:00 2001 From: Simon Weber <simon@simonmweber.com> Date: Thu, 1 Aug 2013 14:44:31 -0400 Subject: [PATCH 71/72] update docs with project status --- README.rst | 4 ---- docs/source/index.rst | 7 ++++++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 78b14e9d..541896a0 100644 --- a/README.rst +++ b/README.rst @@ -54,10 +54,6 @@ switching the the Android app api. This will provide easy All Access support and maintainability going forward. At this point, prefer the Mobileclient to the Webclient whenever possible. -Version 1.2.0 fixes a bug that fixes uploader_id formatting from a mac address. -This change may cause another machine to be registered - you can safely remove the -old machine (it's the one without the version in the name). - For development updates, follow me on Twitter: `@simonmweber <https://twitter.com/simonmweber>`__. diff --git a/docs/source/index.rst b/docs/source/index.rst index 433ba111..b10409dd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -34,7 +34,12 @@ All major functionality is supported: - Music Manager uploading/scan-and-match and library downloading -Support for Google Play Music All Access features is in progress. +Some features may be temporarily unavailable while the project +transitions from the Webclient interface to the Mobileclient. +New code should prefer the Mobileclient when possible, as Webclient +calls may break unexpectedly. + +All Access support is in progress. See `the changelog <https://github.com/simon-weber/Unofficial-Google-Music-API/blob/develop/HISTORY.rst>`__ From d9df4bb08f375e66d5d4448f062f5ab439a48806 Mon Sep 17 00:00:00 2001 From: Simon Weber <simon@simonmweber.com> Date: Thu, 1 Aug 2013 14:55:24 -0400 Subject: [PATCH 72/72] clientlogin can 403 --- gmusicapi/session.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gmusicapi/session.py b/gmusicapi/session.py index d9fe743e..c403ee64 100644 --- a/gmusicapi/session.py +++ b/gmusicapi/session.py @@ -83,7 +83,11 @@ def login(self, email, password, *args, **kwargs): super(Webclient, self).login() - res = ClientLogin.perform(self, True, email, password) + try: + res = ClientLogin.perform(self, True, email, password) + except CallFailure: + self.logout() + return self.is_authenticated if 'SID' not in res or 'Auth' not in res: return False