Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 311 lines (272 sloc) 9.428 kb
088f8be0 »
2010-07-30 Initial commit
1 from mutagen import File
2 from mutagen.asf import ASF, ASFTags
3 from mutagen.apev2 import APEv2File
4 from mutagen.flac import FLAC
5 from mutagen.id3 import ID3FileType
6 from mutagen.easyid3 import EasyID3
7 from mutagen.mp3 import MP3
8 from mutagen.oggflac import OggFLAC
9 from mutagen.oggspeex import OggSpeex
10 from mutagen.oggtheora import OggTheora
11 from mutagen.oggvorbis import OggVorbis
12 from mutagen.trueaudio import TrueAudio
13 from mutagen.wavpack import WavPack
14 from mutagen.mp4 import MP4, MP4Tags
15 from mutagen.musepack import Musepack
16 from mutagen.monkeysaudio import MonkeysAudio
17 from mutagen.optimfrog import OptimFROG
18
3d93ba38 »
2010-08-02 Add Amazon searching
19 import amazon
20
088f8be0 »
2010-07-30 Initial commit
21 import logging
22 import musicbrainz2.webservice as ws
23 import musicbrainz2.model as m
24
25 import sqlite3
26 from os import walk
27 from os.path import splitext,join, abspath
28 import sys
849213b4 »
2010-07-31 First cut of "can't find album" functionality
29 from time import sleep, strptime
30 from types import IntType
088f8be0 »
2010-07-30 Initial commit
31
32 from optparse import OptionParser
33
34 parser = OptionParser()
35 parser.add_option("-m","--music-dir",dest="directory",default=".",help="Pick music files directory. Default is current directory")
36 parser.add_option("-d","--database",dest="db", default="songs.db",help="Songs database file")
69ea7bc1 »
2011-12-24 Add overrides support
37 parser.add_option("--overrides", dest="overrides", default="None", help="Overrides info file")
088f8be0 »
2010-07-30 Initial commit
38 parser.add_option("--no-walk",dest="walk",default="True",action="store_false",help="Don't re-read music directory")
39 (opts,args) = parser.parse_args()
40
69ea7bc1 »
2011-12-24 Add overrides support
41 overrides = {"artist": {}}
42
43 if opts.overrides!=None:
44 for line in open(opts.overrides).readlines():
45 key, value = [x.strip() for x in line.split("=",1)]
46 if key == "artist":
47 name, alt = value.split(";")
48 overrides["artist"][name] = alt
49 else:
50 raise Exception, (key, value)
51
088f8be0 »
2010-07-30 Initial commit
52 class EasyMP3(MP3):
53 def __init__(self, *args, **kwargs):
54 kwargs['ID3'] = EasyID3
55 MP3.__init__(self,*args, **kwargs)
56
57 class EasierTags:
58 def __getitem__(self, key):
59 if key in self.simpler.keys():
60 return self._parent.__getitem__(self,self.simpler[key])
61 else:
62 return self._parent.__getitem__(self, key)
63
64 class EasyMP4Tags(MP4Tags, EasierTags):
65 simpler = {"title":"\xa9nam","artist":"\xa9ART","album":"\xa9alb"}
66
67 class EasyMP4(MP4):
68 def load(self, filename):
69 MP4.load(self,filename)
70 self.tags.__class__ = EasyMP4Tags
71
72 class EasyASFTags(EasierTags, ASFTags):
73 _parent = ASFTags
74 simpler = {"title":"Title"}
75
76 class EasyASF(ASF):
77 def load(self, filename):
78 ASF.load(self,filename)
79 self.tags.__class__ = EasyASFTags
80
81 options = [EasyMP3, TrueAudio, OggTheora, OggSpeex, OggVorbis, OggFLAC,
82 FLAC, APEv2File, EasyMP4, ID3FileType, WavPack, Musepack,
83 MonkeysAudio, OptimFROG, EasyASF]
84
85 doregen = True
86 con = sqlite3.connect(opts.db)
87 cur = con.cursor()
88 cur.execute("select name from sqlite_master where type='table' and name='songs'")
89 if len(cur.fetchall())==0:
90 cur.execute("create table songs (fullpath text(300) primary key, artist text(100),album text(100),title text(100),duration integer);")
91 con.commit()
92
93 if opts.walk:
94 for path, dirs, files in walk(opts.directory):
95 for f in files:
96 if f[0] == ".":
97 continue # ignore hidden files
98 try:
99 fp = unicode(abspath(join(path,f)),"utf_8","ignore")
100 except UnicodeDecodeError:
101 print type(join(path,f)),path,f
102 raise
103 cur.execute("select artist,album,title,duration from songs where fullpath=?", (fp,))
104 d = cur.fetchall()
105 if d==[]:
f77fde90 »
2010-07-31 Handle dodgy files better
106 try:
107 data = File(fp, options=options)
8e296af0 »
2011-12-20 Lots of updates
108 except IOError,e:
109 print e
110 print "rebuilding song db"
f77fde90 »
2010-07-31 Handle dodgy files better
111 data = None
088f8be0 »
2010-07-30 Initial commit
112 if data == None:
113 cur.execute("insert into songs (fullpath,duration) values(?,?)",(fp, -1))
114 con.commit()
115 continue
116 try:
117 try:
118 artist = data["artist"][0].strip()
119 except KeyError:
120 artist = unicode("")
121 try:
122 album = data["album"][0].strip()
123 except KeyError:
124 album = unicode("")
125 try:
126 title = data["title"][0].strip()
127 except KeyError:
128 title = unicode("")
129 duration = int(data.info.length)
130 print (fp, artist, album, title, duration)
131 cur.execute("insert into songs values(?,?,?,?,?)",(fp, artist, album, title, duration))
132 con.commit()
133 except KeyError:
134 print fp,data.keys()
135 raise
136
137 cur.execute("select artist,album, count(title) from songs group by artist,album having count(title)>2 and artist!=\"\"")
138 artists = {}
139 d = cur.fetchall()
140 #print d
141 for (artist, album,title) in d:
142 if artist not in artists:
143 artists[artist] = {}
144 artists[artist][album] = title
8e296af0 »
2011-12-20 Lots of updates
145 #print artists.keys()
088f8be0 »
2010-07-30 Initial commit
146
147 logging.basicConfig()
148 logger = logging.getLogger()
a2a93b15 »
2010-08-02 Remove debug logging
149 #logger.setLevel(logging.DEBUG)
088f8be0 »
2010-07-30 Initial commit
150
151 def getAlbums(artist):
c6d5a3b5 »
2010-08-03 Don't add EPs to get list, and strip multiple versions down
152 cur.execute("select album, asin, date, ep from musicbrainz where artist=?", (artist,))
849213b4 »
2010-07-31 First cut of "can't find album" functionality
153 d = cur.fetchall()
154 if d == []:
155 q = ws.Query()
156
157 f = ws.ArtistFilter(name=artist, limit=5)
a203f365 »
2010-08-03 Give a bit more output on web service errors
158 while True:
159 try:
160 artistResults = q.getArtists(f)
161 break
162 except ws.WebServiceError, e:
9ceea2ab »
2011-12-24 Improve exception support
163 print "problem during artist name", e.msg, e.reason
164 sleep(5)
849213b4 »
2010-07-31 First cut of "can't find album" functionality
165
8e296af0 »
2011-12-20 Lots of updates
166 print "name", artistResults[0].artist.name
849213b4 »
2010-07-31 First cut of "can't find album" functionality
167 artist_id = artistResults[0].artist.id
168
169 release_ids = []
170
171 for kind in (m.Release.TYPE_ALBUM, m.Release.TYPE_EP):
c6d5a3b5 »
2010-08-03 Don't add EPs to get list, and strip multiple versions down
172 while True:
173 try:
174 # The result should include all official albums.
175 #
176 inc = ws.ArtistIncludes(
177 releases=(m.Release.TYPE_OFFICIAL, kind),
178 tags=True)
179 release_ids.extend([(x.id,kind) for x in q.getArtistById(artist_id, inc).getReleases()])
8e296af0 »
2011-12-20 Lots of updates
180 break
c6d5a3b5 »
2010-08-03 Don't add EPs to get list, and strip multiple versions down
181 except ws.WebServiceError, e:
9ceea2ab »
2011-12-24 Improve exception support
182 print "problem during releases", e.msg, e.reason
183 sleep(5)
849213b4 »
2010-07-31 First cut of "can't find album" functionality
184
185 if release_ids == []:
8e296af0 »
2011-12-20 Lots of updates
186 print "No releases found for %s"%artist
9ceea2ab »
2011-12-24 Improve exception support
187 raise Exception
8e296af0 »
2011-12-20 Lots of updates
188 return {}
189
190 print "release ids", release_ids
849213b4 »
2010-07-31 First cut of "can't find album" functionality
191
192 ret = {}
c6d5a3b5 »
2010-08-03 Don't add EPs to get list, and strip multiple versions down
193 for (id,kind) in release_ids:
849213b4 »
2010-07-31 First cut of "can't find album" functionality
194 inc = ws.ReleaseIncludes(artist=True, releaseEvents=True)
195 while True:
196 try:
197 release = q.getReleaseById(id, inc)
198 break
a203f365 »
2010-08-03 Give a bit more output on web service errors
199 except ws.WebServiceError, e:
9ceea2ab »
2011-12-24 Improve exception support
200 print "problem during release", e.msg, e.reason
201 sleep(5)
849213b4 »
2010-07-31 First cut of "can't find album" functionality
202 if release.asin == None: # ignore these
203 continue
204 title = release.title
205 if title.find("(disc ")!=-1:
206 title = title[:title.find("(disc ")].strip()
3d93ba38 »
2010-08-02 Add Amazon searching
207
208 #assert title not in ret.keys(),(title, release)
c6d5a3b5 »
2010-08-03 Don't add EPs to get list, and strip multiple versions down
209 ret[title] = {"when":release.getEarliestReleaseDate(), "asin":release.asin, "ep": kind == m.Release.TYPE_EP}
849213b4 »
2010-07-31 First cut of "can't find album" functionality
210 print "got", title
211
212 for title in ret:
c6d5a3b5 »
2010-08-03 Don't add EPs to get list, and strip multiple versions down
213 cur.execute("insert into musicbrainz values(?, ?, ?, ?, ?)", (artist, title, ret[title]["asin"], ret[title]["when"], ret[title]["ep"]))
849213b4 »
2010-07-31 First cut of "can't find album" functionality
214 con.commit()
215 else:
216 ret = {}
c6d5a3b5 »
2010-08-03 Don't add EPs to get list, and strip multiple versions down
217 for (album, asin, when, ep) in d:
218 ret[album] = {"asin":asin, "when":when, "ep": ep}
219
220 keys = ret.keys()
221
222 for title in keys:
223 if title.find("(")!=-1:
224 stripped = title[:title.find("(")].strip()
225 if stripped[-1] == ".":
226 stripped = stripped[:-1]
227 if stripped in ret.keys():
228 print "removed", title, stripped
229 del ret[title]
230 continue
231
849213b4 »
2010-07-31 First cut of "can't find album" functionality
232 try:
233 ret[title]["when"] = strptime(ret[title]["when"], "%Y-%m-%d")
234 except ValueError:
235 if ret[title]["when"].find("-")!=-1:
236 ret[title]["when"] = int(ret[title]["when"][:ret[title]["when"].find("-")])
237 else:
238 ret[title]["when"] = int(ret[title]["when"])
239 except TypeError:
240 if type(ret[title]["when"]) == IntType:
241 ret[title]["when"] = (ret[title]["when"], 0,0,0,0,0,0,0,0)
8e296af0 »
2011-12-20 Lots of updates
242 elif ret[title]["when"] == None:
243 pass
849213b4 »
2010-07-31 First cut of "can't find album" functionality
244 else:
245 raise
088f8be0 »
2010-07-30 Initial commit
246 return ret
247
849213b4 »
2010-07-31 First cut of "can't find album" functionality
248 most_tracks = sorted(artists.keys(), lambda x,y:cmp(sum(artists[y].values()), sum(artists[x].values())))
249 print most_tracks
250
251 cur.execute("select name from sqlite_master where type='table' and name='musicbrainz'")
252 if len(cur.fetchall())==0:
c6d5a3b5 »
2010-08-03 Don't add EPs to get list, and strip multiple versions down
253 cur.execute("create table musicbrainz (artist text(100), album text(100), asin text(20), date integer, ep boolean, primary key(artist, album));")
849213b4 »
2010-07-31 First cut of "can't find album" functionality
254 con.commit()
255
3d93ba38 »
2010-08-02 Add Amazon searching
256 cur.execute("select name from sqlite_master where type='table' and name='amazon'")
257 if len(cur.fetchall())==0:
8e296af0 »
2011-12-20 Lots of updates
258 cur.execute("create table amazon (artist text(100), album text(100), url text(500), image text(500), amazon_new integer, primary key(artist, album));")
3d93ba38 »
2010-08-02 Add Amazon searching
259 con.commit()
260
849213b4 »
2010-07-31 First cut of "can't find album" functionality
261 def compact(inp):
262 inp = inp.lower()
263 return inp.replace("'","")
264
265 for artist in most_tracks:
69ea7bc1 »
2011-12-24 Add overrides support
266 if artist in overrides["artist"]:
267 newartist = overrides["artist"][artist]
268 artists[newartist] = artists[artist]
269 del artists[artist]
270 artist = newartist
271
088f8be0 »
2010-07-30 Initial commit
272 print "artist",artist
273 albums = getAlbums(artist)
849213b4 »
2010-07-31 First cut of "can't find album" functionality
274 print artist, albums.keys(), artists[artist]
275
276 newest = None
277 for got_a in artists[artist].keys():
278 use_a = None
279 if got_a in albums.keys():
280 use_a = got_a
281 else:
282 items = [x for x in compact(got_a).split() if x not in ("(ep)",)]
283 for k in albums.keys():
284 for i in items:
285 if i not in compact(k):
286 break
287 else:
8e296af0 »
2011-12-20 Lots of updates
288 #print "found all bits", items, k
849213b4 »
2010-07-31 First cut of "can't find album" functionality
289 use_a = k
290 break
291 if use_a != None:
292 if newest == None or newest < albums[use_a]['when']:
293 newest = albums[use_a]['when']
294 else:
295 print "Can't find '%s'"%got_a, albums.keys()
296
297 for a in albums.keys():
c6d5a3b5 »
2010-08-03 Don't add EPs to get list, and strip multiple versions down
298 if albums[a]['when'] > newest and not albums[a]['ep']:
3d93ba38 »
2010-08-02 Add Amazon searching
299 print "don't have",a, albums[a]['asin'], artist
8e296af0 »
2011-12-20 Lots of updates
300 cur.execute("select url, image, amazon_new from amazon where artist=? and album=?",(artist, a))
3d93ba38 »
2010-08-02 Add Amazon searching
301 d = cur.fetchall()
302 if d == []:
303 results = amazon.searchByTitle(artist, a)
8e296af0 »
2011-12-20 Lots of updates
304 cur.execute("insert into amazon values(?, ?, ?, ?, ?)",(artist, a, unicode(results["url"]), unicode(results["image"]), results["amazon_new"]))
3d93ba38 »
2010-08-02 Add Amazon searching
305 con.commit()
306 else:
307 d = d[0]
8e296af0 »
2011-12-20 Lots of updates
308 results = {"title":a, "url":d[0], "image":d[1], "amazon_new":d[2]}
3d93ba38 »
2010-08-02 Add Amazon searching
309 print results
310 #raise Exception,albums[a]
849213b4 »
2010-07-31 First cut of "can't find album" functionality
311 #break
088f8be0 »
2010-07-30 Initial commit
312
3d93ba38 »
2010-08-02 Add Amazon searching
313
Something went wrong with that request. Please try again.