Permalink
Cannot retrieve contributors at this time
Name already in use
A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
telemetry-tools/amf3reader.py /
Go to fileThis commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
547 lines (477 sloc)
19.8 KB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python | |
| # Library to parse an AMF3 formatted buffer into native python | |
| # or can be used to convert AMF3 into JSON | |
| # Copyright 2013 Adobe Systems Incorporated. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0.html | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| from struct import unpack | |
| from datetime import datetime | |
| class Metric: | |
| pass | |
| def ByteToHex( byteStr ): | |
| return ' '.join( [ "%02X" % ord( x ) for x in byteStr ] ) | |
| class amf3reader(dict): | |
| """ | |
| Reads AMF3 data | |
| This can be used with streaming data or with a static buffer | |
| The data can be passed when creating the instance or passed in pieces using addData | |
| Read using "read" to get one object at a time, or use "unpack" to unpack the entire buffer | |
| (in many cases this will amount to the same thing) | |
| if called on its own this file will convert amf content to python | |
| """ | |
| # Set verbose to True to get a detailed report on AMF3 encoding with hex | |
| verbose = False | |
| kUndefinedAtomType = 0 | |
| kNullAtomType = 1 | |
| kFalseAtomType = 2 | |
| kTrueAtomType = 3 | |
| kIntegerAtomType = 4 | |
| kDoubleAtomType = 5 | |
| kStringAtomType = 6 | |
| kAvmMinusXmlAtomType = 7 | |
| kDateAtomType = 8 | |
| kArrayAtomType = 9 | |
| kObjectAtomType = 10 | |
| kAvmPlusXmlAtomType = 11 | |
| kByteArrayAtomType = 12 | |
| kTypedVectorIntType = 13 | |
| kTypedVectorUintType = 14 | |
| kTypedVectorDoubleType = 15 | |
| kTypedVectorObjectType = 16 | |
| kDictionaryObjectType = 17 | |
| def __init__(self, newData = None): | |
| # Lock on to the file | |
| self.data = "" | |
| if newData: | |
| self.data = newData | |
| self.pos = 0 | |
| self.stringList = [] | |
| self.traitsList = [] | |
| self.objectsList = [] | |
| self.format = None | |
| self.flash11Mode = False | |
| def setData(self, data): | |
| self.data = data; | |
| self.getFormat() | |
| def addData(self, data): | |
| self.data += data; | |
| self.getFormat() | |
| def addString(self, string): | |
| index = len(self.stringList) | |
| self.stringList.append(string) | |
| return index | |
| def addObject(self, obj): | |
| self.objectsList.append(obj); | |
| def getObject(self, index): | |
| if index < len(self.objectsList): | |
| obj = self.objectsList[index]; | |
| return obj; | |
| else : | |
| print("invalid object reference" + str(index)); | |
| raise Exception("Invalid reference"); | |
| def clearObjectsList(self): | |
| self.objectsList = [] | |
| def getString(self,index): | |
| if index < len(self.stringList): | |
| string = self.stringList[index]; | |
| else: | |
| string = "unknown" | |
| print("invalid string reference " + str(index)) | |
| return string | |
| def printHex(self,count): | |
| if self.verbose and self.pos+count <= len(self.data): | |
| print ByteToHex(self.data[self.pos:self.pos+count]), | |
| def peekByte(self): | |
| if (self.pos < len(self.data)): | |
| return ord(self.data[self.pos]) | |
| else: | |
| return None | |
| def readByte(self): | |
| val = None | |
| if self.pos+1 <= len(self.data): | |
| val = self.data[self.pos] | |
| self.printHex(1) | |
| self.pos += 1 | |
| return ord(val) | |
| else: | |
| raise EOFError; | |
| return val | |
| def readInt(self): | |
| val = None | |
| if self.pos+4 <= len(self.data): | |
| val = unpack('>I', self.data[self.pos:self.pos+4])[0] | |
| self.pos += 4 | |
| else: | |
| raise EOFError; | |
| return val | |
| def readShort(self): | |
| val = None | |
| if self.pos+2 <= len(self.data): | |
| val = unpack('>H', self.data[self.pos:self.pos+2])[0] | |
| self.pos += 2; | |
| else: | |
| raise EOFError; | |
| return val | |
| def readDouble(self): | |
| val = None | |
| if self.pos+8 <= len(self.data): | |
| val = unpack('>d', self.data[self.pos:self.pos+8])[0] | |
| self.printHex(8) | |
| self.pos += 8 | |
| else: | |
| raise EOFError; | |
| return val | |
| def readBytes(self,length): | |
| val = None | |
| if (self.verbose): print "bytes: ", | |
| if self.pos+length <= len(self.data): | |
| val = self.data[self.pos:self.pos+length] | |
| self.printHex(length) | |
| self.pos += length | |
| else: | |
| raise EOFError; | |
| if (self.verbose): print ""; # Move to next line (self.printHex above would have printed all bytes) | |
| return val | |
| def readUint29(self): | |
| byte = self.readByte() | |
| if byte == None: return None | |
| if byte < 128: | |
| return byte; | |
| ref = (byte & 0x7F) << 7 | |
| byte = self.readByte() | |
| if byte == None: return None | |
| if byte < 128: | |
| return (ref | byte) | |
| ref = (ref | (byte & 0x7F)) << 7 | |
| byte = self.readByte() | |
| if byte == None: return None | |
| if byte < 128: | |
| return (ref | byte) | |
| ref = (ref | (byte & 0x7F)) << 8 | |
| byte = self.readByte() | |
| if byte == None: return None | |
| return (ref | byte); | |
| def readAmfString(self, stringWithoutMarker, noCache=False): | |
| """ reads and AMF formatted string tracking references """ | |
| if (self.verbose and stringWithoutMarker): | |
| print "String(wm) ", | |
| ref = self.readUint29() | |
| if ref == None: return None | |
| if (ref & 1) == 0: | |
| if (self.verbose) : print "Ref: %d" %(ref>>1), | |
| return self.getString(ref >> 1); | |
| length = ref >> 1; | |
| if length == 0: | |
| return "" | |
| if (self.verbose) : print " (%d) Len: %d" % (len(self.stringList), (ref>>1)), | |
| s = self.readBytes(length) | |
| if not noCache: # for flash11 support | |
| self.addString(s) | |
| return s | |
| def readAmfObject(self) : | |
| encoding = self.readByte(); | |
| if encoding == None: | |
| return None | |
| encoding = encoding & 255; | |
| value = None; | |
| if encoding == self.kIntegerAtomType: | |
| value = self.readUint29(); | |
| if (self.verbose) : print "Int29=%d" % value; | |
| elif encoding == self.kDoubleAtomType: | |
| value = self.readDouble(); | |
| if (self.verbose) : print "Double=%g" % value; | |
| elif encoding == self.kStringAtomType: | |
| if (self.verbose) : print "String ", | |
| value = self.readAmfString(False, self.flash11Mode); # early format support | |
| if (self.verbose) : print " \"%s\"" % value; | |
| elif encoding == self.kNullAtomType: | |
| value = None; | |
| if (self.verbose) : print "Null" | |
| elif encoding == self.kUndefinedAtomType: | |
| value = None; | |
| if (self.verbose) : print "Undefined" | |
| elif encoding == self.kFalseAtomType: | |
| value = False; | |
| if (self.verbose) : print "False" | |
| elif encoding == self.kTrueAtomType: | |
| value = True; | |
| if (self.verbose) : print "True" | |
| elif encoding == self.kDateAtomType: | |
| ref = self.readUint29() | |
| if (self.verbose) : print "Date ", | |
| if (ref & 1) == 0: | |
| value = self.getObject(ref>>1); | |
| if (self.verbose): print "Ref: " + str(ref>>1); | |
| else: | |
| if (self.verbose) : print "(%d)" % len(self.objectsList); | |
| value = self.readDouble(); # dates are written as doubles | |
| if (self.verbose) : print " " + str(value) + " " + datetime.fromtimestamp(value/1000).isoformat(); | |
| self.addObject(value); | |
| elif (encoding == self.kAvmMinusXmlAtomType or | |
| encoding == self.kAvmPlusXmlAtomType ): | |
| ref = self.readUint29() | |
| if (self.verbose) : | |
| print "XML ", | |
| if encoding == self.kAvmMinusXmlAtomType : | |
| print "- ", | |
| else : | |
| print "+ ", | |
| if (ref & 1) == 0: | |
| value = self.getObject(ref>>1); | |
| if (self.verbose) : print "Ref: " + str(ref>>1); | |
| else: | |
| if (self.verbose) : print "(%d)" % len(self.objectsList); | |
| if (self.verbose) : self.printHex(ref >> 1) | |
| value = self.readBytes(ref >> 1); # return as string for now | |
| if (self.verbose) : print value.encode() | |
| self.addObject(value); | |
| elif encoding == self.kDictionaryObjectType: | |
| ref = self.readUint29() | |
| if (self.verbose) : print "Dictionary ", | |
| if (ref & 1) == 0: | |
| value = self.getObject(ref>>1); | |
| if (self.verbose) : print "Ref: " + str(ref>>1); | |
| else: | |
| if (self.verbose) : | |
| print "(%d)" % len(self.objectsList) , | |
| print " count: %d" %(ref>>1); | |
| weakref = self.readByte() == 1 | |
| if (self.verbose) : print "weakRef"; | |
| count = ref >> 1 | |
| value = {} | |
| self.addObject(value); | |
| while count > 0: | |
| key = self.readAmfObject() | |
| val = self.readAmfObject() | |
| #print key | |
| #print val | |
| value[str(key)] = val | |
| count -= 1 | |
| if (self.verbose) : print '\n', | |
| elif encoding == self.kArrayAtomType: | |
| value = {} | |
| ref = self.readUint29() | |
| if (self.verbose) : print "Array ", | |
| if (ref & 1) == 0 : | |
| value = self.getObject(ref>>1); | |
| if (self.verbose) : print "Ref: " + str(ref); | |
| else : | |
| if (self.verbose) : | |
| print "(%d)" % len(self.objectsList) | |
| print "count: %d" % (ref>>1); | |
| self.addObject(value); | |
| count = ref >> 1 | |
| # read the non-dense portion | |
| s = self.readAmfString(True) | |
| while s and len(s)>0: | |
| if (self.verbose) : print "%s (dyn)" % (s) | |
| v = self.readAmfObject() | |
| value[s] = v | |
| s = self.readAmfString(True) | |
| if (self.verbose) : print "" | |
| #now read the dense portion | |
| i = 0 | |
| while i < count: | |
| if (self.verbose) : print "[%d]" % i | |
| value[i] = self.readAmfObject(); | |
| i += 1 | |
| if (self.verbose) : print '\n', | |
| elif ( encoding == self.kTypedVectorIntType or | |
| encoding == self.kTypedVectorUintType or | |
| encoding == self.kTypedVectorDoubleType or | |
| encoding == self.kTypedVectorObjectType ): | |
| ref = self.readUint29() | |
| if (self.verbose) : | |
| if (encoding == self.kTypedVectorIntType): | |
| print "Vector Int", | |
| elif (encoding == self.kTypedVectorUintType) : | |
| print "Vector Uint", | |
| elif (encoding == self.kTypedVectorDoubleType) : | |
| print "Vector Double", | |
| else: | |
| print "Vector Object", | |
| if (ref & 1) == 0: | |
| value = self.getObject(ref>>1); | |
| print " Ref: " + str(ref>>1); | |
| else: | |
| count = ref >> 1 | |
| if (self.verbose): print "length = %d" % count | |
| fixed = self.readByte() == 1 | |
| if (self.verbose) : print "fixed" | |
| value = [] | |
| self.addObject(value); | |
| if (encoding == self.kTypedVectorIntType or encoding == self.kTypedVectorUintType): | |
| while(count > 0): | |
| self.printHex(4) | |
| value.append(self.readInt()) | |
| count -= 1 | |
| if (self.verbose) : print " " | |
| if (self.verbose): print value | |
| elif encoding == self.kTypedVectorDoubleType: | |
| while(count > 0): | |
| value.append(self.readDouble()) | |
| count -= 1 | |
| elif encoding == self.kTypedVectorObjectType: | |
| className = self.readAmfString(True); | |
| if (self.verbose): print "ClassName = %s" % className | |
| while (count > 0): | |
| value.append(self.readAmfObject()); | |
| count -= 1 | |
| if (self.verbose) : print '\n', | |
| elif encoding == self.kByteArrayAtomType: | |
| ref = self.readUint29(); | |
| if (self.verbose): | |
| print "ByteArray ", | |
| if (ref & 1) == 0: | |
| value = self.getObject(ref>>1); | |
| if (self.verbose) : print "Ref: " + str(ref>>1) | |
| else: | |
| if (self.verbose) : print "count: " + str(ref>>1); | |
| value = self.readBytes(ref>>1); | |
| self.addObject(value); | |
| elif encoding == self.kObjectAtomType: | |
| ref = self.readUint29() | |
| #if ref == None: return None | |
| if (self.verbose): | |
| print "Object ", | |
| if (ref & 1) == 0: | |
| value = self.getObject(ref>>1); | |
| if (self.verbose) : print "Ref: " + str(ref>>1); | |
| else : | |
| if (self.verbose) : print "(%d)" % len(self.objectsList), | |
| if ((ref & 3) == 1): | |
| if (self.verbose) : print "Traits Ref: " + str(ref>>2) + " (class: %s slots: %d dynamic: %d)" %(self.traitsList[ref>>2]['className'], len(self.traitsList[ref>>2]['slots']), self.traitsList[ref>>2]['dynamic']); | |
| traits = self.traitsList[ref >> 2]; | |
| else: | |
| traits = {} | |
| traits['dynamic'] = ((ref & 8) >> 3); | |
| if ref & 4: | |
| traits['externalizable'] = True | |
| traits['count'] = ref >> 4 | |
| if self.verbose: | |
| print "Traits (%d) slots: %d dynamic: %d" % (len(self.traitsList), (ref>>4), ((ref & 8) >> 3)) | |
| className = self.readAmfString(True) | |
| if self.verbose: | |
| print "class: " + className | |
| if className and len(className) > 0: | |
| traits['className'] = className | |
| else : | |
| traits['className'] = "" | |
| #new_class = type(className', (object,), {)); | |
| slots = []; | |
| count = traits['count'] | |
| while (count): | |
| slots.append(self.readAmfString(True)) | |
| if self.verbose: print slots[len(slots)-1] | |
| count -= 1 | |
| traits['slots'] = slots | |
| self.traitsList.append(traits) | |
| value = {} | |
| self.addObject(value); | |
| #if traits.has_key('className'): | |
| # value['_className'] = traits['className'] | |
| for slot in traits['slots']: | |
| value[slot] = self.readAmfObject() | |
| if (traits['dynamic'] == 1): | |
| s = self.readAmfString(True) | |
| while s and len(s)>0: | |
| if (self.verbose) : print "%s (dyn)" % (s) | |
| v = self.readAmfObject() | |
| value[s] = v | |
| s = self.readAmfString(True) | |
| if (self.verbose) : print '\n', | |
| else: | |
| print("invalid data type", encoding); | |
| #throw new Error("invalid data stream"); | |
| value = None; | |
| return value; | |
| # Examine data stream to find its format | |
| def getFormat(self): | |
| if (len(self.data)<1): | |
| return None | |
| firstByte = ord(self.data[0]) | |
| if firstByte == self.kObjectAtomType: | |
| self.format = "amfstream" # raw stream from player | |
| elif firstByte == self.kArrayAtomType: | |
| self.format = "amfarray" # saved array of telemetry data | |
| else: | |
| self.format = "oldstyle" # early format | |
| self.flash11Mode = True | |
| return self.format | |
| def readMetric(self): | |
| recordPos = self.pos; | |
| record = None; | |
| if self.flash11Mode: | |
| # this is to support older telemetry, remove eventually | |
| try: | |
| name = self.readAmfString(True) | |
| if (self.verbose) : print "String=%s" % name; | |
| if name.endswith(".span"): | |
| name = name.rsplit('.',1)[0] # strip the .span extension | |
| span = self.readAmfObject(); | |
| tname = self.readAmfString(True); | |
| if (self.verbose) : print "String=%s" % tname; | |
| time = self.readAmfObject(); | |
| if name is not None and tname is not None and tname.endswith(".time") \ | |
| and span is not None and time is not None: | |
| record = {'name':name,"span":span,"time":time}; | |
| elif name.endswith(".time"): | |
| name = name.rsplit('.',1)[0] # strip unused types | |
| time = self.readAmfObject() | |
| if time: | |
| record = {'name':name,'time':time} | |
| else: | |
| if name.endswith(".count"): | |
| name = name.rsplit('.',1)[0] # strip unused types | |
| record = {'name':name} | |
| record['value'] = self.readAmfObject(); | |
| except: | |
| self.pos = recordPos; #rewind to start | |
| name = None | |
| else: | |
| traitsLen = len(self.traitsList) | |
| stringCount = len(self.stringList) | |
| try: | |
| record = self.readAmfObject(); | |
| except EOFError: | |
| if (self.verbose): | |
| print "Partial record warning" | |
| self.pos = recordPos; #rewind to start | |
| self.traitsList = self.traitsList[0:traitsLen]; | |
| self.stringList = self.stringList[0:stringCount]; | |
| record = None; | |
| finally: | |
| self.clearObjectsList(); | |
| return record | |
| # reads the entire buffer as one array of objects | |
| def unpack(self): | |
| output = [] # all the data ends up in this array | |
| rec = self.readMetric() | |
| while (rec and self.pos+1 < len(self.data)): | |
| output.append(rec) | |
| rec = self.readMetric() | |
| if (rec): | |
| output.append(rec); | |
| return output | |
| if __name__ == '__main__': | |
| import sys | |
| from pprint import pprint | |
| if len(sys.argv) == 1: | |
| print 'Usage: %s filename [filename]...' % sys.argv[0] | |
| print 'Where filename is a .flm file' | |
| print 'eg. %s myfile' % sys.argv[0] | |
| for filename in sys.argv[1:]: | |
| file = open(filename, 'rb') | |
| reader = amf3reader(file.read()); | |
| file.close() | |
| output = reader.unpack() | |
| pprint(output) # print the array to stdout in JSON format |