Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 542 lines (473 sloc) 21.6 KB
#!/usr/bin/python
#
# appdailysales.py
#
# iTune Connect Daily Sales Reports Downloader
# Copyright 2008-2011 Kirby Turner
#
# Version 2.9.3
#
# Latest version and additional information available at:
# http://appdailysales.googlecode.com/
#
#
# This script will download yesterday's daily sales report from
# the iTunes Connect web site. The downloaded file is stored
# in the same directory containing the script file. Note: if
# the download file already exists then it will be overwritten.
#
# The iTunes Connect web site has dynamic urls and form field
# names. In other words, these values change from session to
# session. So to get to the download file we must navigate
# the site and webscrape the pages. Joy, joy.
#
#
# Contributors:
# Leon Ho
# Rogue Amoeba Software, LLC
# Keith Simmons
# Andrew de los Reyes
# Maarten Billemont
# Daniel Dickison
# Mike Kasprzak
# Shintaro TAKEMURA
# aaarrrggh (Paul)
#
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
# -- Change the following to match your credentials --
# -- or use the command line options. --
appleId = 'Your Apple Id'
password = 'Your Password'
outputDirectory = ''
unzipFile = False
verbose = False
daysToDownload = 1
dateToDownload = None
outputFormat = None
overWriteFiles = True
proxy = ''
debug = False
# ----------------------------------------------------
import urllib
import urllib2
import cookielib
import datetime
import re
import getopt
import sys
import os
import gzip
import StringIO
import traceback
import getpass
class ITCException(Exception):
def __init__(self,value):
self.value = value
def __str__(self):
return repr(self.value);
# The class ReportOptions defines a structure for passing
# report options to the download routine. The expected
# data attributes are:
# appleId
# password
# outputDirectory
# unzipFile
# verbose
# daysToDownload
# dateToDownload
# outputFormat
# overWriteFiles
# debug
# Note that the class attributes will default to the global
# variable value equivalent.
class ReportOptions:
def __getattr__(self, attrname):
if attrname == 'appleId':
return appleId
elif attrname == 'password':
return password
elif attrname == 'outputDirectory':
return outputDirectory
elif attrname == 'unzipFile':
return unzipFile
elif attrname == 'verbose':
return verbose
elif attrname == 'daysToDownload':
return daysToDownload
elif attrname == 'dateToDownload':
return dateToDownload
elif attrname == 'outputFormat':
return outputFormat
elif attrname == 'overWriteFiles':
return overWriteFiles
elif attrname == 'proxy':
return proxy
elif attrname == 'debug':
return debug
else:
raise AttributeError, attrname
def usage():
print '''usage: %s [options]
Options and arguments:
-h : print this help message and exit (also --help)
-a uid : your apple id (also --appleId)
-p pwd : your password (also --password)
-P : read the password from stdin (also --passwordStdin)
-o dir : directory where download file is stored, default is the current working directory (also --outputDirectory)
-v : verbose output, default is off (also --verbose)
-u : unzip download file, default is off (also --unzip)
-d num : number of days to download, default is 1 (also --days)
-D mm/dd/yyyy : report date to download, -d option is ignored when -D is used (also --date)
-f format : output file name format (see strftime; also --format)
-n : used with -f, skips downloading of report files that already exist (also --noOverWriteFiles)
--proxy : URL of the proxy
--debug : debug output, default is off''' % sys.argv[0]
def processCmdArgs():
global appleId
global password
global outputDirectory
global unzipFile
global verbose
global daysToDownload
global dateToDownload
global outputFormat
global overWriteFiles
global proxy
global debug
# Check for command line options. The command line options
# override the globals set above if present.
try:
opts, args = getopt.getopt(sys.argv[1:], 'ha:p:Po:uvd:D:f:n', ['help', 'appleId=', 'password=', 'passwordStdin', 'outputDirectory=', 'unzip', 'verbose', 'days=', 'date=', 'format=', 'noOverWriteFiles', 'proxy=', 'debug'])
except getopt.GetoptError, err:
#print help information and exit
print str(err) # will print something like "option -x not recongized"
usage()
return 2
for o, a in opts:
if o in ('-h', '--help'):
usage()
return 2
elif o in ('-a', '--appleId'):
appleId = a
elif o in ('-p', '--password'):
password = a
elif o in ('-P', '--passwordStdin'):
password = getpass.getpass()
elif o in ('-o', '--outputDirectory'):
outputDirectory = a
elif o in ('-u', '--unzip'):
unzipFile = True
elif o in ('-v', '--verbose'):
verbose = True
elif o in ('-d', '--days'):
daysToDownload = a
elif o in ('-D', '--date'):
dateToDownload = a
elif o in ('-f', '--format'):
outputFormat = a
elif o in ('-n', '--noOverWriteFiles'):
overWriteFiles = False
elif o in ('-o', '--proxy'):
proxy = a
elif o in ('--debug'):
debug = True
verbose = True # Turn on verbose if debug option is on.
else:
assert False, 'unhandled option'
# There is an issue with Python 2.5 where it assumes the 'version'
# cookie value is always interger. However, itunesconnect.apple.com
# returns this value as a string, i.e., "1" instead of 1. Because
# of this we need a workaround that "fixes" the version field.
#
# More information at: http://bugs.python.org/issue3924
class MyCookieJar(cookielib.CookieJar):
def _cookie_from_cookie_tuple(self, tup, request):
name, value, standard, rest = tup
version = standard.get('version', None)
if version is not None:
version = version.replace('"', '')
standard["version"] = version
return cookielib.CookieJar._cookie_from_cookie_tuple(self, tup, request)
def showCookies(cj):
for index, cookie in enumerate(cj):
print index, ' : ', cookie
def readHtml(opener, url, data=None, options=None):
request = urllib2.Request(url, data)
urlHandle = opener.open(request)
html = urlHandle.read()
if options and options.debug:
f = open(os.path.join(options.outputDirectory, 'temp.html'), 'w')
try:
f.write(html)
finally:
f.close()
return html
def downloadFile(options):
if options.verbose == True:
print '-- begin script --'
if (options.outputDirectory != '' and not os.path.exists(options.outputDirectory)):
os.makedirs(options.outputDirectory)
urlITCBase = 'https://itunesconnect.apple.com%s'
handlers = [] # proxy support
if options.proxy: # proxy support
handlers.append(urllib2.ProxyHandler({"https": options.proxy})) # proxy support
cj = MyCookieJar();
cj.set_policy(cookielib.DefaultCookiePolicy(rfc2965=True))
cjhdr = urllib2.HTTPCookieProcessor(cj)
handlers.append(cjhdr) # proxy support
opener = urllib2.build_opener(*handlers) # proxy support
if options.verbose == True:
print 'Signing into iTunes Connect web site.'
# Go to the iTunes Connect website and retrieve the
# form action for logging into the site.
urlWebsite = urlITCBase % '/WebObjects/iTunesConnect.woa'
html = readHtml(opener, urlWebsite, options=options)
match = re.search('" action="(.*)"', html)
urlActionLogin = urlITCBase % match.group(1)
# Login to iTunes Connect web site and go to the sales
# report page, get the form action url and form fields.
# Note the sales report page will actually load a blank
# page that redirects to the static URL. Best guess here
# is that the server is setting some session variables
# or something.
webFormLoginData = urllib.urlencode({'theAccountName':options.appleId, 'theAccountPW':options.password, '1.Continue':'0'})
html = readHtml(opener, urlActionLogin, webFormLoginData, options=options)
if (html.find('Your Apple ID or password was entered incorrectly.') != -1):
raise ITCException, 'User or password incorrect.'
# Find the Sales and Trends URL.
if options.verbose == True:
print 'Accessing Sales and Trends reporting web site.'
# Sometimes the vendor default page does not load right away.
# This causes the script to fail, so as a work around, the
# script will attempt to load the page 3 times before abend.
vendorDefaultPageAttempts = 3
while vendorDefaultPageAttempts > 0:
vendorDefaultPageAttempts = vendorDefaultPageAttempts - 1
urlSalesAndTrends = 'https://reportingitc.apple.com/'
html = readHtml(opener, urlSalesAndTrends, options=options)
# We're at the vendor default page. Might need additional work if your account
# has more than one vendor.
try:
match = re.findall('"javax.faces.ViewState" value="(.*?)"', html)
viewState = match[0]
match = re.findall('script id="defaultVendorPage:(.*?)"', html)
defaultVendorPage = match[0]
ajaxName = re.sub('_2', '_0', defaultVendorPage)
if options.debug == True:
print 'viewState: ', viewState
print 'defaultVendorPage: ', defaultVendorPage
print 'ajaxName: ', ajaxName
vendorDefaultPageAttempts = 0 # exit loop
except:
if vendorDefaultPageAttempts < 1:
errMessage = 'Unable to load default vendor page.'
if options.verbose == True:
print errMessage
raise
else:
raise ITCException, errMessage
# This may seem confusing because we just accessed the vendor default page in the
# code above. However, the vendor default page as a piece of javascript that runs
# once the page is loaded in the browser. The javascript does a resubmit. My guess
# is this action is needed to set the default vendor on the server-side. Regardless
# we must call the page again but no parsing of the HTML is needed this time around.
urlDefaultVendorPage = 'https://reportingitc.apple.com/vendor_default.faces'
webFormSalesReportData = urllib.urlencode({'AJAXREQUEST':ajaxName, 'javax.faces.ViewState':viewState, 'defaultVendorPage':defaultVendorPage, 'defaultVendorPage:'+defaultVendorPage:'defaultVendorPage:'+defaultVendorPage})
html = readHtml(opener, urlDefaultVendorPage, webFormSalesReportData, options=options)
# Check for notification messages.
urlDashboard = 'https://reportingitc.apple.com/subdashboard.faces'
html = readHtml(opener, urlDashboard, options=options)
try:
# Note the (?s) is an inline re.DOTALL, makes . match new lines.
match = re.findall('(?s)<div class="notification">(.*?)</span>', html)
notificationDiv = match[0]
match = re.findall('(?s)<td>(.*?)</td>', notificationDiv)
notificationMessage = match[0]
if options.verbose == True:
print notificationMessage
except:
pass # Do nothing. We're just checking for notifications.
# Access the sales report page.
if options.verbose == True:
print 'Accessing sales report web page.'
urlSalesReport = 'https://reportingitc.apple.com/sales.faces'
html = readHtml(opener, urlSalesReport, options=options)
# Get the form field names needed to download the report.
try:
match = re.findall('"javax.faces.ViewState" value="(.*?)"', html)
viewState = match[0]
match = re.findall('theForm:j_id_jsp_[0-9]*_51', html)
dailyName = match[0]
ajaxName = dailyName.replace('_51', '_2')
dateName = dailyName.replace('_51', '_8')
selectName = dailyName.replace('_51', '_29')
if options.debug == True:
print 'viewState: ', viewState
print 'dailyName: ', dailyName
print 'ajaxName: ', ajaxName
print 'dateName: ', dateName
print 'selectName:', selectName
except:
errMessage = 'Unable to load the sales report web page at this time. A number of reasons can cause this including delayed reporting, unsigned contracts, and change to the web site breaking this script. Try again later or sign into iTunes Connect and verify access.'
if options.verbose == True:
print errMessage
raise
else:
raise ITCException, errMessage
# Get the list of available dates.
try:
# Note the (?s) is an inline re.DOTALL, makes . match new lines.
match = re.findall('(?s)<div class="pickList">(.*?)</div>', html)
dateListAvailableDays = re.findall('<option value="(.*?)"', match[0])
dateListAvailableWeeks = re.findall('<option value="(.*?)"', match[1])
if options.debug == True:
print 'dateListAvailableDays: ', dateListAvailableDays
print 'dateListAvailableWeeks: ', dateListAvailableWeeks
except:
errMessage = 'Unable to retrieve the list of available dates.'
if options.verbose == True:
print errMessage
raise
else:
raise ITCException, errMessage
# Click through from the dashboard to the sales page.
webFormSalesReportData = urllib.urlencode({'AJAXREQUEST':ajaxName, 'theForm':'theForm', 'theForm:xyz':'notnormal', 'theForm:vendorType':'Y', 'theForm:datePickerSourceSelectElementSales':dateListAvailableDays[0], 'theForm:weekPickerSourceSelectElement':dateListAvailableWeeks[0], 'javax.faces.ViewState':viewState, dailyName:dailyName, 'theForm:optInVar':'A', 'theForm:dateType':'D', 'theForm:optInVarRender':'false', 'theForm:wklyBool':'false'})
html = readHtml(opener, urlSalesReport, webFormSalesReportData, options=options)
match = re.findall('"javax.faces.ViewState" value="(.*?)"', html)
viewState = match[0]
# Set the list of report dates.
# A better approach is to grab the list of available dates
# from the web site instead of generating the dates. Will
# consider doing this in the future.
reportDates = []
if options.dateToDownload == None:
for i in range(int(options.daysToDownload)):
today = datetime.date.today() - datetime.timedelta(i + 1)
reportDates.append( today )
else:
reportDates = [datetime.datetime.strptime(options.dateToDownload, '%m/%d/%Y').date()]
if options.debug == True:
print 'reportDates: ', reportDates
####
if options.verbose == True:
print 'Downloading daily sales reports.'
unavailableCount = 0
filenames = []
for downloadReportDate in reportDates:
# Set the date within the web page.
dateString = downloadReportDate.strftime('%m/%d/%Y')
if dateString in dateListAvailableDays:
# If told not to overwrite files, check before downloading
if (options.overWriteFiles == False):
# Only if custom formatting was enabled, we can check if file exists before downloading
if (options.outputFormat):
filename = downloadReportDate.strftime(options.outputFormat)
filename = os.path.join(options.outputDirectory, filename)
if options.unzipFile == True and filename[-3:] == '.gz': #Chop off .gz extension if not needed
filename = os.path.splitext( filename )[0]
if (os.path.exists(filename)):
if options.verbose == True:
print 'Report file', filename, 'exists, skipping.'
continue
if options.verbose == True:
print 'Downloading report for: ', dateString
webFormSalesReportData = urllib.urlencode({'AJAXREQUEST':ajaxName, 'theForm':'theForm', 'theForm:xyz':'notnormal', 'theForm:vendorType':'Y', 'theForm:datePickerSourceSelectElementSales':dateString, 'theForm:datePickerSourceSelectElementSales':dateString, 'theForm:weekPickerSourceSelectElement':dateListAvailableWeeks[0], 'javax.faces.ViewState':viewState, selectName:selectName})
html = readHtml(opener, urlSalesReport, webFormSalesReportData)
match = re.findall('"javax.faces.ViewState" value="(.*?)"', html)
viewState = match[0]
# And finally...we're ready to download yesterday's sales report.
webFormSalesReportData = urllib.urlencode({'theForm':'theForm', 'theForm:xyz':'notnormal', 'theForm:vendorType':'Y', 'theForm:datePickerSourceSelectElementSales':dateString, 'theForm:weekPickerSourceSelectElement':dateListAvailableWeeks[0], 'javax.faces.ViewState':viewState, 'theForm:downloadLabel2':'theForm:downloadLabel2'})
request = urllib2.Request(urlSalesReport, webFormSalesReportData)
urlHandle = opener.open(request)
try:
if options.debug == True:
print urlHandle.info()
# Check for the content-disposition. If present then we know we have a
# file to download. If not present then an AttributeError exception is
# thrown and we assume the file is not available for download.
filename = urlHandle.info().getheader('content-disposition').split('=')[1]
# Check for an override of the file name. If found then change the file
# name to match the outputFormat.
if (options.outputFormat):
filename = downloadReportDate.strftime(options.outputFormat)
filebuffer = urlHandle.read()
urlHandle.close()
if options.unzipFile == True:
if options.verbose == True:
print 'Unzipping archive file: ', filename
#Use GzipFile to de-gzip the data
ioBuffer = StringIO.StringIO( filebuffer )
gzipIO = gzip.GzipFile( 'rb', fileobj=ioBuffer )
filebuffer = gzipIO.read()
filename = os.path.join(options.outputDirectory, filename)
if options.unzipFile == True and filename[-3:] == '.gz': #Chop off .gz extension if not needed
filename = os.path.splitext( filename )[0]
# If optionsFormat was used, make sure the directory requested exists
if (options.outputFormat):
fileDirectory = os.path.dirname(filename)
if (os.path.exists(fileDirectory) == False):
if options.verbose == True:
print 'Directory', fileDirectory, 'doesn\'t exist, creating.'
os.makedirs(fileDirectory)
if options.verbose == True:
print 'Saving download file:', filename
downloadFile = open(filename, 'w')
downloadFile.write(filebuffer)
downloadFile.close()
filenames.append( filename )
except AttributeError:
print '%s report is not available - try again later.' % dateString
unavailableCount += 1
else:
print '%s report is not available - try again later.' % dateString
unavailableCount += 1
# End for downloadReportDate in reportDates:
####
if unavailableCount > 0:
raise ITCException, '%i report(s) not available - try again later' % unavailableCount
if options.debug == True:
os.remove(os.path.join(options.outputDirectory, "temp.html"))
if options.verbose == True:
print '-- end of script --'
return filenames
def main():
if processCmdArgs() > 0: # Will exit if usgae requested or invalid argument found.
return 2
# Set report options.
options = ReportOptions()
options.appleId = appleId
options.password = password
options.outputDirectory = outputDirectory
options.unzipFile = unzipFile
options.verbose = verbose
options.daysToDownload = daysToDownload
options.dateToDownload = dateToDownload
options.outputFormat = outputFormat
options.overWriteFiles = overWriteFiles
options.proxy = proxy
options.debug = debug
# Download the file.
try:
downloadFile(options)
except ITCException, e:
print e.value
return 1
if __name__ == '__main__':
sys.exit(main())
You can’t perform that action at this time.