Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 314 lines (272 sloc) 9.428 kB
088f8be @palfrey Initial commit
authored
1 from mutagen import File
2 from mutagen.asf import ASF, ASFTags
3 from mutagen.apev2 import APEv2File
4 from mutagen.flac import FLAC
5 from mutagen.id3 import ID3FileType
6 from mutagen.easyid3 import EasyID3
7 from mutagen.mp3 import MP3
8 from mutagen.oggflac import OggFLAC
9 from mutagen.oggspeex import OggSpeex
10 from mutagen.oggtheora import OggTheora
11 from mutagen.oggvorbis import OggVorbis
12 from mutagen.trueaudio import TrueAudio
13 from mutagen.wavpack import WavPack
14 from mutagen.mp4 import MP4, MP4Tags
15 from mutagen.musepack import Musepack
16 from mutagen.monkeysaudio import MonkeysAudio
17 from mutagen.optimfrog import OptimFROG
18
3d93ba3 @palfrey Add Amazon searching
authored
19 import amazon
20
088f8be @palfrey Initial commit
authored
21 import logging
22 import musicbrainz2.webservice as ws
23 import musicbrainz2.model as m
24
25 import sqlite3
26 from os import walk
27 from os.path import splitext,join, abspath
28 import sys
849213b @palfrey First cut of "can't find album" functionality
authored
29 from time import sleep, strptime
30 from types import IntType
088f8be @palfrey Initial commit
authored
31
32 from optparse import OptionParser
33
34 parser = OptionParser()
35 parser.add_option("-m","--music-dir",dest="directory",default=".",help="Pick music files directory. Default is current directory")
36 parser.add_option("-d","--database",dest="db", default="songs.db",help="Songs database file")
69ea7bc @palfrey Add overrides support
authored
37 parser.add_option("--overrides", dest="overrides", default="None", help="Overrides info file")
088f8be @palfrey Initial commit
authored
38 parser.add_option("--no-walk",dest="walk",default="True",action="store_false",help="Don't re-read music directory")
39 (opts,args) = parser.parse_args()
40
69ea7bc @palfrey Add overrides support
authored
41 overrides = {"artist": {}}
42
43 if opts.overrides!=None:
44 for line in open(opts.overrides).readlines():
45 key, value = [x.strip() for x in line.split("=",1)]
46 if key == "artist":
47 name, alt = value.split(";")
48 overrides["artist"][name] = alt
49 else:
50 raise Exception, (key, value)
51
088f8be @palfrey Initial commit
authored
52 class EasyMP3(MP3):
53 def __init__(self, *args, **kwargs):
54 kwargs['ID3'] = EasyID3
55 MP3.__init__(self,*args, **kwargs)
56
57 class EasierTags:
58 def __getitem__(self, key):
59 if key in self.simpler.keys():
60 return self._parent.__getitem__(self,self.simpler[key])
61 else:
62 return self._parent.__getitem__(self, key)
63
64 class EasyMP4Tags(MP4Tags, EasierTags):
65 simpler = {"title":"\xa9nam","artist":"\xa9ART","album":"\xa9alb"}
66
67 class EasyMP4(MP4):
68 def load(self, filename):
69 MP4.load(self,filename)
70 self.tags.__class__ = EasyMP4Tags
71
72 class EasyASFTags(EasierTags, ASFTags):
73 _parent = ASFTags
74 simpler = {"title":"Title"}
75
76 class EasyASF(ASF):
77 def load(self, filename):
78 ASF.load(self,filename)
79 self.tags.__class__ = EasyASFTags
80
81 options = [EasyMP3, TrueAudio, OggTheora, OggSpeex, OggVorbis, OggFLAC,
82 FLAC, APEv2File, EasyMP4, ID3FileType, WavPack, Musepack,
83 MonkeysAudio, OptimFROG, EasyASF]
84
85 doregen = True
86 con = sqlite3.connect(opts.db)
87 cur = con.cursor()
88 cur.execute("select name from sqlite_master where type='table' and name='songs'")
89 if len(cur.fetchall())==0:
90 cur.execute("create table songs (fullpath text(300) primary key, artist text(100),album text(100),title text(100),duration integer);")
91 con.commit()
92
93 if opts.walk:
94 for path, dirs, files in walk(opts.directory):
95 for f in files:
96 if f[0] == ".":
97 continue # ignore hidden files
98 try:
99 fp = unicode(abspath(join(path,f)),"utf_8","ignore")
100 except UnicodeDecodeError:
101 print type(join(path,f)),path,f
102 raise
103 cur.execute("select artist,album,title,duration from songs where fullpath=?", (fp,))
104 d = cur.fetchall()
105 if d==[]:
f77fde9 @palfrey Handle dodgy files better
authored
106 try:
107 data = File(fp, options=options)
8e296af @palfrey Lots of updates
authored
108 except IOError,e:
109 print e
110 print "rebuilding song db"
f77fde9 @palfrey Handle dodgy files better
authored
111 data = None
088f8be @palfrey Initial commit
authored
112 if data == None:
113 cur.execute("insert into songs (fullpath,duration) values(?,?)",(fp, -1))
114 con.commit()
115 continue
116 try:
117 try:
118 artist = data["artist"][0].strip()
119 except KeyError:
120 artist = unicode("")
121 try:
122 album = data["album"][0].strip()
123 except KeyError:
124 album = unicode("")
125 try:
126 title = data["title"][0].strip()
127 except KeyError:
128 title = unicode("")
129 duration = int(data.info.length)
130 print (fp, artist, album, title, duration)
131 cur.execute("insert into songs values(?,?,?,?,?)",(fp, artist, album, title, duration))
132 con.commit()
133 except KeyError:
134 print fp,data.keys()
135 raise
136
137 cur.execute("select artist,album, count(title) from songs group by artist,album having count(title)>2 and artist!=\"\"")
138 artists = {}
139 d = cur.fetchall()
140 #print d
141 for (artist, album,title) in d:
142 if artist not in artists:
143 artists[artist] = {}
144 artists[artist][album] = title
8e296af @palfrey Lots of updates
authored
145 #print artists.keys()
088f8be @palfrey Initial commit
authored
146
147 logging.basicConfig()
148 logger = logging.getLogger()
a2a93b1 @palfrey Remove debug logging
authored
149 #logger.setLevel(logging.DEBUG)
088f8be @palfrey Initial commit
authored
150
151 def getAlbums(artist):
c6d5a3b @palfrey Don't add EPs to get list, and strip multiple versions down
authored
152 cur.execute("select album, asin, date, ep from musicbrainz where artist=?", (artist,))
849213b @palfrey First cut of "can't find album" functionality
authored
153 d = cur.fetchall()
154 if d == []:
155 q = ws.Query()
156
157 f = ws.ArtistFilter(name=artist, limit=5)
a203f36 @palfrey Give a bit more output on web service errors
authored
158 while True:
159 try:
160 artistResults = q.getArtists(f)
161 break
162 except ws.WebServiceError, e:
9ceea2a @palfrey Improve exception support
authored
163 print "problem during artist name", e.msg, e.reason
164 sleep(5)
849213b @palfrey First cut of "can't find album" functionality
authored
165
8e296af @palfrey Lots of updates
authored
166 print "name", artistResults[0].artist.name
849213b @palfrey First cut of "can't find album" functionality
authored
167 artist_id = artistResults[0].artist.id
168
169 release_ids = []
170
171 for kind in (m.Release.TYPE_ALBUM, m.Release.TYPE_EP):
c6d5a3b @palfrey Don't add EPs to get list, and strip multiple versions down
authored
172 while True:
173 try:
174 # The result should include all official albums.
175 #
176 inc = ws.ArtistIncludes(
177 releases=(m.Release.TYPE_OFFICIAL, kind),
178 tags=True)
179 release_ids.extend([(x.id,kind) for x in q.getArtistById(artist_id, inc).getReleases()])
8e296af @palfrey Lots of updates
authored
180 break
c6d5a3b @palfrey Don't add EPs to get list, and strip multiple versions down
authored
181 except ws.WebServiceError, e:
9ceea2a @palfrey Improve exception support
authored
182 print "problem during releases", e.msg, e.reason
183 sleep(5)
849213b @palfrey First cut of "can't find album" functionality
authored
184
185 if release_ids == []:
8e296af @palfrey Lots of updates
authored
186 print "No releases found for %s"%artist
9ceea2a @palfrey Improve exception support
authored
187 raise Exception
8e296af @palfrey Lots of updates
authored
188 return {}
189
190 print "release ids", release_ids
849213b @palfrey First cut of "can't find album" functionality
authored
191
192 ret = {}
c6d5a3b @palfrey Don't add EPs to get list, and strip multiple versions down
authored
193 for (id,kind) in release_ids:
849213b @palfrey First cut of "can't find album" functionality
authored
194 inc = ws.ReleaseIncludes(artist=True, releaseEvents=True)
195 while True:
196 try:
197 release = q.getReleaseById(id, inc)
198 break
a203f36 @palfrey Give a bit more output on web service errors
authored
199 except ws.WebServiceError, e:
9ceea2a @palfrey Improve exception support
authored
200 print "problem during release", e.msg, e.reason
201 sleep(5)
849213b @palfrey First cut of "can't find album" functionality
authored
202 if release.asin == None: # ignore these
203 continue
204 title = release.title
205 if title.find("(disc ")!=-1:
206 title = title[:title.find("(disc ")].strip()
3d93ba3 @palfrey Add Amazon searching
authored
207
208 #assert title not in ret.keys(),(title, release)
c6d5a3b @palfrey Don't add EPs to get list, and strip multiple versions down
authored
209 ret[title] = {"when":release.getEarliestReleaseDate(), "asin":release.asin, "ep": kind == m.Release.TYPE_EP}
849213b @palfrey First cut of "can't find album" functionality
authored
210 print "got", title
211
212 for title in ret:
c6d5a3b @palfrey Don't add EPs to get list, and strip multiple versions down
authored
213 cur.execute("insert into musicbrainz values(?, ?, ?, ?, ?)", (artist, title, ret[title]["asin"], ret[title]["when"], ret[title]["ep"]))
849213b @palfrey First cut of "can't find album" functionality
authored
214 con.commit()
215 else:
216 ret = {}
c6d5a3b @palfrey Don't add EPs to get list, and strip multiple versions down
authored
217 for (album, asin, when, ep) in d:
218 ret[album] = {"asin":asin, "when":when, "ep": ep}
219
220 keys = ret.keys()
221
222 for title in keys:
223 if title.find("(")!=-1:
224 stripped = title[:title.find("(")].strip()
225 if stripped[-1] == ".":
226 stripped = stripped[:-1]
227 if stripped in ret.keys():
228 print "removed", title, stripped
229 del ret[title]
230 continue
231
849213b @palfrey First cut of "can't find album" functionality
authored
232 try:
233 ret[title]["when"] = strptime(ret[title]["when"], "%Y-%m-%d")
234 except ValueError:
235 if ret[title]["when"].find("-")!=-1:
236 ret[title]["when"] = int(ret[title]["when"][:ret[title]["when"].find("-")])
237 else:
238 ret[title]["when"] = int(ret[title]["when"])
239 except TypeError:
240 if type(ret[title]["when"]) == IntType:
241 ret[title]["when"] = (ret[title]["when"], 0,0,0,0,0,0,0,0)
8e296af @palfrey Lots of updates
authored
242 elif ret[title]["when"] == None:
243 pass
849213b @palfrey First cut of "can't find album" functionality
authored
244 else:
245 raise
088f8be @palfrey Initial commit
authored
246 return ret
247
849213b @palfrey First cut of "can't find album" functionality
authored
248 most_tracks = sorted(artists.keys(), lambda x,y:cmp(sum(artists[y].values()), sum(artists[x].values())))
249 print most_tracks
250
251 cur.execute("select name from sqlite_master where type='table' and name='musicbrainz'")
252 if len(cur.fetchall())==0:
c6d5a3b @palfrey Don't add EPs to get list, and strip multiple versions down
authored
253 cur.execute("create table musicbrainz (artist text(100), album text(100), asin text(20), date integer, ep boolean, primary key(artist, album));")
849213b @palfrey First cut of "can't find album" functionality
authored
254 con.commit()
255
3d93ba3 @palfrey Add Amazon searching
authored
256 cur.execute("select name from sqlite_master where type='table' and name='amazon'")
257 if len(cur.fetchall())==0:
8e296af @palfrey Lots of updates
authored
258 cur.execute("create table amazon (artist text(100), album text(100), url text(500), image text(500), amazon_new integer, primary key(artist, album));")
3d93ba3 @palfrey Add Amazon searching
authored
259 con.commit()
260
849213b @palfrey First cut of "can't find album" functionality
authored
261 def compact(inp):
262 inp = inp.lower()
263 return inp.replace("'","")
264
265 for artist in most_tracks:
69ea7bc @palfrey Add overrides support
authored
266 if artist in overrides["artist"]:
267 newartist = overrides["artist"][artist]
268 artists[newartist] = artists[artist]
269 del artists[artist]
270 artist = newartist
271
088f8be @palfrey Initial commit
authored
272 print "artist",artist
273 albums = getAlbums(artist)
849213b @palfrey First cut of "can't find album" functionality
authored
274 print artist, albums.keys(), artists[artist]
275
276 newest = None
277 for got_a in artists[artist].keys():
278 use_a = None
279 if got_a in albums.keys():
280 use_a = got_a
281 else:
282 items = [x for x in compact(got_a).split() if x not in ("(ep)",)]
283 for k in albums.keys():
284 for i in items:
285 if i not in compact(k):
286 break
287 else:
8e296af @palfrey Lots of updates
authored
288 #print "found all bits", items, k
849213b @palfrey First cut of "can't find album" functionality
authored
289 use_a = k
290 break
291 if use_a != None:
292 if newest == None or newest < albums[use_a]['when']:
293 newest = albums[use_a]['when']
294 else:
295 print "Can't find '%s'"%got_a, albums.keys()
296
297 for a in albums.keys():
c6d5a3b @palfrey Don't add EPs to get list, and strip multiple versions down
authored
298 if albums[a]['when'] > newest and not albums[a]['ep']:
3d93ba3 @palfrey Add Amazon searching
authored
299 print "don't have",a, albums[a]['asin'], artist
8e296af @palfrey Lots of updates
authored
300 cur.execute("select url, image, amazon_new from amazon where artist=? and album=?",(artist, a))
3d93ba3 @palfrey Add Amazon searching
authored
301 d = cur.fetchall()
302 if d == []:
303 results = amazon.searchByTitle(artist, a)
8e296af @palfrey Lots of updates
authored
304 cur.execute("insert into amazon values(?, ?, ?, ?, ?)",(artist, a, unicode(results["url"]), unicode(results["image"]), results["amazon_new"]))
3d93ba3 @palfrey Add Amazon searching
authored
305 con.commit()
306 else:
307 d = d[0]
8e296af @palfrey Lots of updates
authored
308 results = {"title":a, "url":d[0], "image":d[1], "amazon_new":d[2]}
3d93ba3 @palfrey Add Amazon searching
authored
309 print results
310 #raise Exception,albums[a]
849213b @palfrey First cut of "can't find album" functionality
authored
311 #break
088f8be @palfrey Initial commit
authored
312
3d93ba3 @palfrey Add Amazon searching
authored
313
Something went wrong with that request. Please try again.