Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Initial commit

  • Loading branch information...
commit 2ca9100eb50a2f1806b37e2b06a6e24b1fa8d362 0 parents
@mnaberez authored
9 Rakefile
@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), 'lib')
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), 'test')
+
+desc "Run tests"
+task :test do
+ Dir['test/**/*_test.rb'].each { |file| require file }
+end
+
+task :default => [:test]
16 demo.rb
@@ -0,0 +1,16 @@
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), 'lib')
+
+require 'tdms'
+
+filename = File.dirname(__FILE__) + "/test/fixtures/example.tdms"
+doc = Tdms::File.parse(filename)
+
+ch1 = doc.channels.find {|c| c.name == "StatisticsText"}
+ch2 = doc.channels.find {|c| c.name == "Res_Noise_1"}
+
+last = [ch1.values.size, ch2.values.size].min - 1
+
+puts "#{ch1.name},#{ch2.name}"
+0.upto(last) do |i|
+ puts "#{ch1.values[i]},#{ch2.values[i]}"
+end
23 doc/data_types.txt
@@ -0,0 +1,23 @@
+Data Types
+
+Identifier Name Length (bytes) Ruby
+
+0x00000000 tdsTypeVoid 1 Nil
+0x00000001 tdsTypeI8 1 Integer
+0x00000002 tdsTypeI16 2 Integer
+0x00000003 tdsTypeI32 4 Integer
+0x00000004 tdsTypeI64 8 Integer
+0x00000005 tdsTypeU8 1 Integer
+0x00000006 tdsTypeU16 2 Integer
+0x00000007 tdsTypeU32 4 Integer
+0x00000008 tdsTypeU64 8 Integer
+0x00000009 tdsTypeSingleFloat 4 Float
+0x0000000A tdsTypeDoubleFloat 8 Float
+0x0000000B tdsTypeExtendedFloat 10 ?
+0x00000019 tdsTypeSingleFloatWithUnit ? ?
+0x0000001A tdsTypeDoubleFloatWithUnit ? ?
+0x0000001B tdsTypeExtendedFloatWithUnit ? ?
+0x00000020 tdsTypeString 4 len + n chars String
+0x00000021 tdsTypeBoolean 1 True, False
+0x00000044 tdsTypeTimeStamp 16 (?) DateTime
+0xFFFFFFFF tdsTypeDAQmxRawData ? ?
47 doc/example_disasm.txt
@@ -0,0 +1,47 @@
+EXAMPLE.tdms
+ Contains 4 segments
+
+
+000000 54 44 53 6d "TDSm" id tag
+000004 0E 00 00 00 0e is ToC flag, other 3 bytes unused
+000008 68 12 00 00 6812 is little endian for 0x1268 or 4713 (TDMS standard version). Other two bytes unused.
+00000C E3 88 20 00
+000010 00 00 00 00 => 64-bit unsigned little endian 0x00000000002088E3 is the next segment offset
+000014 E3 08 00 00
+000018 00 00 00 00 => 64-bit unsigned little endian 0x00000000000008E3 is the raw data offset
+ ( End of Lead-In )
+ ( 00001C is the offset of the next byte after the lead-in )
+ ( Next segment offset = 00001C + 2088E3 = 2088FF )
+ ( Raw data offset = 00001C + 0008E3 = 0008FF )
+
+ ( Start of Metadata )
+00001C 08 00 00 00 => 32-bit unsigned little endian 0x00000008 is the number of
+ new/changed objects in this segment (8 objects)
+000020 11 00 00 00 => 32-bit unsigned little endian 0x00000011 is the length of
+ the object's path string (11 bytes)
+ (Begin Path String)
+000024 2F 27 45 58 /'EX
+000028 41 4D 50 4C AMPL
+00002C 45 27 2F 27 E'/'
+000030 54 69 6D 65 Time
+000034 27 '
+ (Begin Length of Index)
+000034 14 00 00
+000038 0A 00 00 => 32-bit unsigned little endian 0x00000014 is the
+ raw data index
+ (Begin Data Type)
+000038 0A
+00003C 00 00 00 => 32-bit unsigned little endian 0x0000000A is the
+ data type (tdsTypeDoubleFloat)
+
+ (Begin Array Dimension)
+00003C 01
+000040 00 00 00 => 32-bit unsigned little endian 0x00000001 is the
+ array dimension (always 1)
+ (Begin Number of Values)
+000040 00
+000044 04 00 00 00
+000048 04 00 00 => 64-bit unsigned little endian 0x0000000000000400 is the
+ number of values (1024 values)
+ (Begin Total Size in Bytes)
+
101 doc/tdms_format.txt
@@ -0,0 +1,101 @@
+http://zone.ni.com/devzone/cda/tut/p/id/5696
+
+Object Hierarchy
+
+ Hierarchy Path
+
+ example_events.tdms (File) /
+ |
+ +-- Measured Data (Group) /'Measured Data'
+ | |
+ | +-- Amplitude Sweep (Channel) /'Measured Data'/'Amplitude Sweep'
+ | +-- Phase Sweep (Channel) /'Measured Data'/'Phase Sweep'
+ |
+ +-- Dr. T's Events (Group) /'Dr. T''s Events'
+ |
+ +-- Time (Channel) /'Dr. T''s Events'/'Time'
+ +-- Description (Channel) /'Dr. T''s Events'/'Description'
+
+ There are exactly 3 levels of objects in TDMS:
+ - File (every TDMS file must have one)
+ - Group (has many Channels, may have none)
+ - Channel (belongs to a Group)
+
+ Every object is identified by a string path. The only thing that
+ identifies an object as being a File, Group, or Channel is the
+ number of Segments in the string path:
+
+ Object Type Example Path Number of Path Segments
+ File / 0
+ Group /'Group' 1
+ Channel /'Group'/'Channel' 2
+
+
+File Structure
+
+ File is divided into Segments.
+ Each segment contains one or more Objects
+ Every Object has a string Path
+
+
+Segment
+
+ +-----------+------------+------------+----------------------------
+ | Lead-in | Metadata | Raw Data | Lead-in (Next Segment) ...
+ +-----------+------------+------------+----------------------------
+
+ Lead-in
+ +---------+--------+---------+--------------------+--------------------+---
+ | 4: TDSm | 4: ToC | 4: Vers | 8: Next Seg Offset | 8: Raw Data Offset | Metadata ...
+ +---------+--------+---------+--------------------+--------------------+---
+ 00 04 08 0C 14 1C
+
+
+ 00-03 4 bytes: TDMS identifier (always "TDSm")
+ 04-07 4 bytes: Table of contents (only first byte used)
+ Flags to indicate what is in the segment
+ 08-0B 4 bytes: TDMS version number (always 4713)
+ 0C-13 8 bytes: Offset of the next segment (little endian). Take the
+ absolute offset of the next byte after the lead-in and
+ add it to this number to find the next segment.
+ 14-1B 8 bytes: Offset of the raw data in this segment (little endian).
+ Also calculate it from next byte after the lead-in.
+
+
+ Metadata
+ +----------+-------------+---------+-------------------+--------------+-
+ | 4: Count | 4: Path Len | n: Path | 4: Raw data index | 4: Num Props |
+ +----------+-------------+---------+-------------------+--------------+-
+ 00 04 08
+
+ 4 bytes: Number of new/changed objects in this segment
+
+ For each object in the segment:
+ 4 bytes: Length of object's string path
+ n bytes: Object's string path
+
+ Index block or markers:
+ If no raw data in the segment:
+ 4 bytes: FF FF FF FF
+ Else if raw data index block exactly matches last segment:
+ 4 bytes: 00 00 00 00
+ Else:
+ 4 bytes: Length of raw data index block + 4
+ 4 bytes: Data type of the raw data in this object
+ 4 bytes: Dimension of raw data array (only first byte used, always 1)
+ 8 bytes: Number of values
+ 8 bytes: Total size in bytes (?) -- may not be here
+
+ Properties Block:
+
+ 4 bytes: Number of properties of this object
+
+ For each property:
+ 4 bytes: Length of property name
+ n bytes: Property name
+ 4 bytes: Data type of property value (only first byte used)
+ n bytes: Property value
+ String is 4 bytes for length, then bytes of string
+ Others are fixed number of bytes for value based on type
+
+ Raw Data
46 doc/usage.txt
@@ -0,0 +1,46 @@
+READING
+=======
+
+# display properties of a channel
+
+ group = segment.groups.find {|grp| grp.path == "/'EXAMPLE'" }
+ speed = group.channels.find {|ch| ch.path == "/'EXAMPLE'/'Time'" }
+ speed.properties.each_pair do |k,v|
+ puts k,v
+ end
+
+# loop through a channel
+
+ group = segment.groups.find {|grp| grp.path == "/'EXAMPLE'" }
+ speed = group.channels.find {|ch| ch.path == "/'EXAMPLE'/'Time'" }
+ speed.values.each do |v|
+ puts v #=> float
+ end
+
+# spreadsheet of two channels
+
+ group = segment.groups.find {|grp| grp.path == "/'EXAMPLE'" }
+
+ time = group.channels.find { |ch| ch.path == "/'EXAMPLE'/'Time'" }
+ speed = group.channels.find { |ch| ch.path == "/'EXAMPLE'/'Speed'" }
+
+ max = [time.values.size, speed.values.size].max - 1
+ 0.upto(max) do |i|
+ puts "%f,%f" % (time.values[i], speed.values[i])
+ end
+
+
+WRITING
+=======
+
+tdms = Tdms::File.new("some filename")
+seg = tdms.segment.build
+
+group = seg.groups.build("foo")
+
+chan = group.channels.build("bar")
+chan.properties["Flux Capacitor"] = "On"
+channel.values << 1.02
+channel.values << 1.02
+
+seg.save
8 lib/tdms.rb
@@ -0,0 +1,8 @@
+require 'tdms/document'
+require 'tdms/streaming'
+require 'tdms/property'
+require 'tdms/datatypes'
+require 'tdms/segment'
+require 'tdms/channel'
+require 'tdms/path'
+require 'tdms/aggregate'
71 lib/tdms/aggregate.rb
@@ -0,0 +1,71 @@
+module Tdms
+
+ class AggregateChannel
+ def initialize(channels=[])
+ @channels = channels
+ end
+
+ def path
+ @channels[0].path
+ end
+
+ def name
+ @channels[0].name
+ end
+
+ def data_type
+ @channels[0].data_type
+ end
+
+ def values
+ @values ||= AggregateChannelEnumerator.new(@channels)
+ end
+ end
+
+ class AggregateChannelEnumerator
+ include Enumerable
+
+ def initialize(channels)
+ @channels = channels
+ @offsets = []
+
+ size = 0
+ @channels.inject(0) do |size, channel|
+ @offsets << size
+ size += channel.values.size
+ end
+ end
+
+ def size
+ @size ||= @channels.inject(0) { |sum, chan| sum += chan.values.size }
+ end
+
+ def each
+ @channels.each do |channel|
+ channel.values.each { |value| yield value }
+ end
+ end
+
+ def [](i)
+ if (i < 0) || (i >= size)
+ raise RangeError, "Channel %s has a range of 0 to %d, got invalid index: %d" %
+ [@channels[0].path, size - 1, i]
+ end
+
+ channel, offset = nil, nil
+ j = @offsets.size - 1
+ @offsets.reverse_each do |o|
+ if i >= o
+ channel = @channels[j]
+ offset = @offsets[j]
+ break
+ else
+ j -= 1
+ end
+ end
+
+ channel.values[i - offset]
+ end
+ end
+
+end
100 lib/tdms/channel.rb
@@ -0,0 +1,100 @@
+module Tdms
+
+ class Channel < Object
+ attr_accessor :file, :path, :data_type_id, :dimension, :num_values,
+ :raw_data_pos
+
+ def name
+ path.channel
+ end
+
+ def values
+ @values ||= begin
+ klass = if data_type::LengthInBytes.nil?
+ StringChannelEnumerator
+ else
+ ChannelEnumerator
+ end
+
+ klass.new(self)
+ end
+ end
+
+ def data_type
+ @data_type ||= DataType.find_by_id(data_type_id)
+ end
+ end
+
+ class ChannelEnumerator
+ include Enumerable
+
+ def initialize(channel)
+ @channel = channel
+ end
+
+ def size
+ @size ||= @channel.num_values
+ end
+
+ def each
+ 0.upto(size - 1) { |i| yield self[i] }
+ end
+
+ def [](i)
+ if (i < 0) || (i >= size)
+ raise RangeError, "Channel %s has a range of 0 to %d, got invalid index: %d" %
+ [@channel.path, size - 1, i]
+ end
+
+ @channel.file.seek @channel.raw_data_pos + (i * @channel.data_type::LengthInBytes)
+ @channel.data_type.read_from_stream(@channel.file).value
+ end
+ end
+
+ class StringChannelEnumerator
+ include Enumerable
+
+ def initialize(channel)
+ @channel = channel
+
+ @index_pos = @channel.raw_data_pos
+ @data_pos = @index_pos + (4 * @channel.num_values)
+ end
+
+ def size
+ @size ||= @channel.num_values
+ end
+
+ def each
+ data_pos = @data_pos
+
+ 0.upto(size - 1) do |i|
+ index_pos = @index_pos + (4 * i)
+
+ @channel.file.seek index_pos
+ next_data_pos = @data_pos + @channel.file.read_u32
+
+ length = next_data_pos - data_pos
+
+ @channel.file.seek data_pos
+ yield @channel.file.read(length)
+
+ data_pos = next_data_pos
+ end
+ end
+
+ def [](i)
+ if (i < 0) || (i >= size)
+ raise RangeError, "Channel %s has a range of 0 to %d, got invalid index: %d" %
+ [@channel.path, size - 1, i]
+ end
+
+ inject(0) do |j, value|
+ return value if j == i
+ j += 1
+ end
+ end
+
+ end
+
+end
63 lib/tdms/datatypes.rb
@@ -0,0 +1,63 @@
+module Tdms
+
+ module DataType
+
+ class Base
+ attr_accessor :value
+
+ def initialize(value=nil)
+ @value = value
+ end
+ end
+
+ class Utf8String < Base
+ Id = 0x20
+ LengthInBytes = nil
+
+ def self.read_from_stream(tdms_file)
+ new(tdms_file.read_utf8_string)
+ end
+ end
+
+ class Int32 < Base
+ Id = 0x03
+ LengthInBytes = 4
+
+ def self.read_from_stream(tdms_file)
+ new(tdms_file.read_i32)
+ end
+ end
+
+ class Double < Base
+ Id = 0x0A
+ LengthInBytes = 8
+
+ def self.read_from_stream(tdms_file)
+ new(tdms_file.read_double)
+ end
+ end
+
+ class Timestamp < Base
+ Id = 0x44
+ LengthInBytes = 16
+
+ def self.read_from_stream(tdms_file)
+ new(tdms_file.read_timestamp)
+ end
+ end
+
+ DataTypesById = {
+ Utf8String::Id => Utf8String,
+ Int32::Id => Int32,
+ Double::Id => Double,
+ Timestamp::Id => Timestamp
+ }
+
+ def find_by_id(id_byte)
+ DataTypesById[id_byte] || raise(ArgumentError, "Don't know type %d" % id_byte)
+ end
+ module_function :find_by_id
+
+ end
+
+end
105 lib/tdms/document.rb
@@ -0,0 +1,105 @@
+module Tdms
+
+ class Document
+ attr_reader :segments, :channels, :file
+
+ def initialize(file)
+ @file = file
+ parse_segments
+ build_aggregates
+ end
+
+ private
+
+ def parse_segments
+ @segments = []
+ prev_segment = nil
+
+ until file.eof?
+ segment = Tdms::Segment.new
+ segment.prev_segment = prev_segment
+ @segments << segment
+
+ lead_in = @file.read(0x1C)
+ metadata_pos = @file.pos
+
+ unpacked = lead_in.unpack("a4VVQQ")
+ tdms_tag = unpacked[0] # char[4]
+ toc_flags = unpacked[1] # u32
+ tdms_version = unpacked[2] # u32
+ next_seg_pos = unpacked[3] + metadata_pos # u64
+ raw_data_pos = unpacked[4] + metadata_pos # u64
+
+ new_changed_objs = @file.read_u32
+
+ raw_data_pos_obj = raw_data_pos
+
+ 1.upto(new_changed_objs) do |obj_index|
+ path = Tdms::Path.new(:path => @file.read_utf8_string)
+ index_block_len = @file.read_u32
+
+ num_props = 0
+
+ if index_block_len == 0xFFFFFFFF
+ #puts " no index block"
+ elsif index_block_len == 0x000000
+ #puts " index block is same as last segment"
+ else
+ # XXX why does the number of properties seem to be
+ # included in the raw data index block size?
+ # -4 is a hack
+ index_block = @file.read(index_block_len - 4)
+ decoded = index_block.unpack("VVQ")
+
+ chan = Tdms::Channel.new
+ chan.file = @file
+ chan.raw_data_pos = raw_data_pos_obj
+ chan.path = path
+ chan.data_type_id = decoded[0] # first 4 bytes u32
+ chan.dimension = decoded[1] # next 4 bytes u32
+ chan.num_values = decoded[2] # next 8 bytes u64
+ segment.objects << chan
+
+ data_type = Tdms::DataType.find_by_id(chan.data_type_id)
+ fixed_length = data_type::LengthInBytes
+
+ raw_data_pos_obj += if fixed_length
+ chan.num_values * fixed_length
+ else
+ # if the values are variable length (strings only) then
+ # the index block contains 8 additional bytes at the
+ # end with the total length of the raw data in u64
+ index_block[-8,8].unpack("Q")[0]
+ end
+ end
+
+ num_props = @file.read_u32
+
+ newline = false
+ 1.upto(num_props) do |n|
+ prop = @file.read_property
+ end
+ end
+
+ @file.seek next_seg_pos
+ end
+
+ end
+
+ def build_aggregates
+ @channels = []
+
+ channels_by_path = {}
+ segments.each do |segment|
+ segment.objects.select { |o| o.path.channel? }.each do |ch|
+ (channels_by_path[ch.path.to_s] ||= []) << ch
+ end
+ end
+
+ channels_by_path.each do |path, channels|
+ @channels << AggregateChannel.new(channels)
+ end
+ end
+ end
+
+end
56 lib/tdms/path.rb
@@ -0,0 +1,56 @@
+module Tdms
+
+ class Path
+ attr_reader :group, :channel
+
+ def initialize(options={})
+ load(options[:path]) if options[:path]
+ @group = options[:group] if options[:group]
+ @channel = options[:channel] if options[:channel]
+ end
+
+ def load(path)
+ segments = path.split("/").map do |seg|
+ seg.sub(/^'/,'').sub(/'$/,'').sub("''", "'")
+ end
+
+ _, @group, @channel = *segments
+ end
+
+ def ==(other)
+ if other.is_a?(String)
+ self.to_s == other
+ else
+ super
+ end
+ end
+
+ def dump
+ raise ArgumentError if channel && group.nil?
+
+ parts = [""]
+ parts << ("'" + group.sub("'","''") + "'") if group
+ parts << ("'" + channel.sub("'","''") + "'") if channel
+
+ dumped = parts.join("/")
+ dumped.empty? ? "/" : dumped
+ end
+
+ def to_s
+ dump
+ end
+
+ def root?
+ (! channel?) && (! group?)
+ end
+
+ def group?
+ (! @group.nil?) && (@channel.nil?)
+ end
+
+ def channel?
+ (! @channel.nil?)
+ end
+ end
+
+end
17 lib/tdms/property.rb
@@ -0,0 +1,17 @@
+module Tdms
+
+ class Property
+ attr_accessor :name
+ attr_accessor :data # tdms::data_type
+
+ def initialize(name=nil, data=nil)
+ @name = name
+ @data = data
+ end
+
+ def value
+ data.value
+ end
+
+ end
+end
12 lib/tdms/segment.rb
@@ -0,0 +1,12 @@
+module Tdms
+
+ class Segment
+ attr_accessor :prev_segment
+ attr_reader :objects
+
+ def initialize
+ @objects = []
+ end
+ end
+
+end
50 lib/tdms/streaming.rb
@@ -0,0 +1,50 @@
+require 'rubygems'
+require 'active_support/all'
+
+module Tdms
+
+ module Streaming
+ def read_property
+ name = read_utf8_string
+ type_id = read_u32
+
+ data = Tdms::DataType.find_by_id(type_id).read_from_stream(self)
+ Tdms::Property.new(name, data)
+ end
+
+ def read_u32
+ read(4).unpack("V")[0]
+ end
+
+ def read_i32
+ read_u32 # XXX
+ end
+
+ def read_double
+ read(8).unpack("E")[0]
+ end
+
+ def read_utf8_string
+ length = read_u32
+ read length
+ end
+
+ def read_timestamp
+ seconds_since_labview_epoch = read(8).unpack("Q")[0]
+ positive_fractions_of_second = read(8).unpack("Q")[0] # ignored
+
+ labview_epoch = ::DateTime.new(1904, 1, 1)
+ labview_epoch.advance(:seconds => seconds_since_labview_epoch)
+ end
+ end
+
+ class File < ::File
+ include Streaming
+
+ def self.parse(filename)
+ f = self.open(filename, "rb")
+ Document.new(f)
+ end
+ end
+
+end
BIN  test/fixtures/example.tdms
Binary file not shown
Please sign in to comment.
Something went wrong with that request. Please try again.