Dart updates for ingest speed, correct time zone, tlmgrapher crash

.gitignore

@@ -10,6 +10,8 @@ desktop.ini
 install/outputs
 demo/outputs
 autohotkey/outputs
+spec/install/outputs
+lib/cosmos/dart/log
 coverage/
 profile/
 demo/tools/mac/CmdExtractor.app/Contents/MacOS/CmdExtractor

bin/dart_util

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+# encoding: ascii-8bit
+require 'cosmos'
+require 'cosmos/dart/processes/dart_util'

cosmos.gemspec

@@ -45,6 +45,7 @@ spec = Gem::Specification.new do |s|
   s.executables << 'cstol_converter'
   s.executables << 'xtce_converter'
   s.executables << 'dart_import'
+  s.executables << 'dart_util'
 
   if RUBY_ENGINE == 'ruby'
     # Ruby C Extensions - MRI Only
@@ -95,11 +96,11 @@ spec = Gem::Specification.new do |s|
   s.add_runtime_dependency 'httpclient', '~> 2.8'
 
   # From http://www.rubydoc.info/gems/puma#Known_Bugs :
-  #   "For MRI versions 2.2.7, 2.2.8, 2.3.4 and 2.4.1, you may see stream closed in
+  #   "For MRI versions 2.2.7, 2.2.8, 2.2.9, 2.2.10, 2.3.4 and 2.4.1, you may see stream closed in
   #    another thread (IOError). It may be caused by a Ruby bug. It can be
   #    fixed with the gem https://rubygems.org/gems/stopgap_13632"
   # This is commented out because the gemspec is only evaluated at gem build time
-  # s.add_runtime_dependency 'stopgap_13632', '~> 1.1.1' if RUBY_ENGINE == 'ruby' and %w(2.2.7 2.2.8 2.3.4 2.4.1).include? RUBY_VERSION  # MRI Only
+  # s.add_runtime_dependency 'stopgap_13632', '~> 1.2.0' if RUBY_ENGINE == 'ruby' and %w(2.2.7 2.2.8 2.3.4 2.4.1).include? RUBY_VERSION  # MRI Only
 
   # Development Dependencies
   s.add_development_dependency 'diff-lcs', '~> 1.3' if RUBY_ENGINE == 'ruby' # Get latest for MRI

demo/config/system/system.txt

@@ -6,6 +6,7 @@ DECLARE_TARGET INST
 DECLARE_TARGET INST INST2
 DECLARE_TARGET EXAMPLE
 DECLARE_TARGET TEMPLATED
+DECLARE_TARGET DART
 DECLARE_TARGET SYSTEM
 
 # Listen Hosts - Ip addresses or hostnames to listen on when running the tools

demo/config/targets/DART/cmd_tlm/dart_cmds.txt

@@ -0,0 +1,2 @@
+COMMAND DART CLEAR_ERRORS BIG_ENDIAN "Clears error counters and messages"
+  APPEND_ID_PARAMETER PACKET_ID 8 INT 1 1 1 "Packet Id"

demo/config/targets/DART/cmd_tlm/dart_tlm.txt

@@ -0,0 +1,40 @@
+TELEMETRY DART STATUS BIG_ENDIAN "DART Status"
+  APPEND_ID_ITEM PACKET_ID 8 INT 1 "Packet Id"
+  APPEND_ITEM LAST_PLE_ID 64 INT "Database ID of the most recent PacketLogEntry created"
+  APPEND_ITEM PLE_STATE_NEED_DECOM 64 INT "Number of PacketLogEntries ready for decom"
+  APPEND_ITEM PLE_STATE_ERROR 64 INT "Number of PacketLogEntries with errors"
+  APPEND_ITEM DECOM_COUNT 64 INT "Count of packets decommutated"
+  APPEND_ITEM DECOM_ERROR_COUNT 64 INT "Count of decommutation errors"
+  APPEND_ITEM DECOM_MESSAGE 2048 STRING "Most recent decommutation message"
+  APPEND_ITEM REDUCTION_COUNT 64 INT "Count of packet reduced since DART started"
+  APPEND_ITEM REDUCTION_ERROR_COUNT 64 INT "Count of reduction errors"
+  APPEND_ITEM REDUCTION_MESSAGE 2048 STRING "Most recent reduction message"
+  APPEND_ITEM DART_DATA_BYTES 64 INT "Size of the DART Data folder in bytes"
+    POLY_READ_CONVERSION 0 0.0000000009313225746154785
+    UNITS GB GB
+    FORMAT_STRING "%0.2f"
+  APPEND_ITEM DART_LOGS_BYTES 64 INT "Size of the DART Logs folder in bytes"
+    POLY_READ_CONVERSION 0 0.0000000009313225746154785
+    UNITS GB GB
+    FORMAT_STRING "%0.2f"
+  APPEND_ITEM DART_DATABASE_BYTES 64 INT "Size of the DART Postgres database in bytes"
+    POLY_READ_CONVERSION 0 0.0000000009313225746154785
+    UNITS GB GB
+    FORMAT_STRING "%0.2f"
+  APPEND_ITEM DART_STATUS_SECONDS 64 FLOAT "Seconds to complete status queries"
+  APPEND_ITEM PLE_FIRST_TIME_S 64 INT "Time of the first PacketLogEntry in the database seconds"
+  APPEND_ITEM PLE_FIRST_TIME_US 64 INT "Time of the first PacketLogEntry in the database microseconds"
+  APPEND_ITEM PLE_LAST_TIME_S 64 INT "Time of the last PacketLogEntry in the database seconds"
+  APPEND_ITEM PLE_LAST_TIME_US 64 INT "Time of the last PacketLogEntry in the database microseconds"
+  APPEND_ITEM DECOM_MESSAGE_TIME_S 64 INT "Time of decommutation message seconds"
+  APPEND_ITEM DECOM_MESSAGE_TIME_US 64 INT "Time of decommutation message microseconds"
+  APPEND_ITEM REDUCTION_MESSAGE_TIME_S 64 INT "Time of reduction message seconds"
+  APPEND_ITEM REDUCTION_MESSAGE_TIME_US 64 INT "Time of reduction message microseconds"
+  ITEM PLE_FIRST_TIME 0 0 DERIVED "Time of the first PacketLogEntry in the database"
+    READ_CONVERSION unix_time_formatted_conversion.rb PLE_FIRST_TIME_S PLE_FIRST_TIME_US
+  ITEM PLE_LAST_TIME 0 0 DERIVED "Time of the last PacketLogEntry in the database"
+    READ_CONVERSION unix_time_formatted_conversion.rb PLE_LAST_TIME_S PLE_LAST_TIME_US
+  ITEM DECOM_MESSAGE_TIME 0 0 DERIVED "Time of decommutation message"
+    READ_CONVERSION unix_time_formatted_conversion.rb DECOM_MESSAGE_TIME_S DECOM_MESSAGE_TIME_US
+  ITEM REDUCTION_MESSAGE_TIME 0 0 DERIVED "Time of reduction message"
+    READ_CONVERSION unix_time_formatted_conversion.rb REDUCTION_MESSAGE_TIME_S REDUCTION_MESSAGE_TIME_US

demo/config/targets/DART/cmd_tlm_server.txt

@@ -0,0 +1,6 @@
+# This is a segment of the main cmd_tlm_server.txt that will be used with
+# AUTO_INTERFACE_TARGETS or INTERFACE_TARGET
+
+INTERFACE DART_INT dart_status_interface.rb
+  TARGET DART
+  DONT_CONNECT

demo/config/targets/DART/screens/status.txt

@@ -0,0 +1,54 @@
+SCREEN AUTO AUTO 1.0 FIXED
+
+VERTICAL
+
+  TITLE "DART Status"
+
+  HORIZONTAL
+    VERTICALBOX
+      SECTIONHEADER "Ingest Status"
+
+      LABELVALUE DART STATUS LAST_PLE_ID
+      LABELVALUE DART STATUS PLE_STATE_ERROR
+      LABELVALUE DART STATUS PLE_FIRST_TIME WITH_UNITS 22
+      LABELVALUE DART STATUS PLE_LAST_TIME WITH_UNITS 22
+    END
+
+    VERTICALBOX
+      SECTIONHEADER "Decom Status"
+
+      LABELVALUE DART STATUS PLE_STATE_NEED_DECOM
+      LABELVALUE DART STATUS DECOM_COUNT
+      LABELVALUE DART STATUS DECOM_ERROR_COUNT
+      LABELVALUE DART STATUS DECOM_MESSAGE_TIME WITH_UNITS 22
+      LABEL DECOM_MESSAGE:
+      VALUE DART STATUS DECOM_MESSAGE CONVERTED 50
+    END
+  END
+  HORIZONTAL
+    VERTICALBOX
+      SECTIONHEADER "Reduction Status"
+
+      LABELVALUE DART STATUS REDUCTION_COUNT
+      LABELVALUE DART STATUS REDUCTION_ERROR_COUNT
+      LABELVALUE DART STATUS REDUCTION_MESSAGE_TIME WITH_UNITS 22
+      LABEL REDUCTION_MESSAGE:
+      VALUE DART STATUS REDUCTION_MESSAGE CONVERTED 50
+    END
+
+    VERTICALBOX
+      SECTIONHEADER "Dart Database Status"
+
+      LABELVALUE DART STATUS DART_DATA_BYTES
+      LABELVALUE DART STATUS DART_LOGS_BYTES
+      LABELVALUE DART STATUS DART_DATABASE_BYTES
+      LABELVALUE DART STATUS DART_STATUS_SECONDS
+    END
+
+  END
+
+  VERTICALBOX
+    LABELVALUE DART STATUS RECEIVED_TIMEFORMATTED WITH_UNITS 22
+  END
+END
+

demo/config/targets/DART/target.txt

@@ -0,0 +1,7 @@
+# Ignored Parameters
+# IGNORE_PARAMETER parameter_name
+IGNORE_PARAMETER PACKET_ID
+
+# Ignored Items
+# IGNORE_ITEM item_name
+IGNORE_ITEM PACKET_ID

lib/cosmos/dart/app/models/status.rb

@@ -0,0 +1,2 @@
+class Status < ApplicationRecord
+end

lib/cosmos/dart/db/migrate/20180423205644_create_statuses.rb

@@ -0,0 +1,22 @@
+class CreateStatuses < ActiveRecord::Migration[5.1]
+  def change
+    create_table :statuses do |t|
+      t.bigint :decom_count, :default => 0
+      t.bigint :decom_error_count, :default => 0
+      t.text :decom_message, :default => ""
+      t.datetime :decom_message_time
+
+      t.bigint :reduction_count, :default => 0
+      t.bigint :reduction_error_count, :default => 0
+      t.text :reduction_message, :default => ""
+      t.datetime :reduction_message_time
+
+      t.timestamps
+    end
+    status = Status.new
+    time = Time.utc(1970, 1, 1)
+    status.decom_message_time = time
+    status.reduction_message_time = time
+    status.save!
+  end
+end

lib/cosmos/dart/db/migrate/20180425211340_add_decom_state_index.rb

@@ -0,0 +1,5 @@
+class AddDecomStateIndex < ActiveRecord::Migration[5.1]
+  def change
+    add_index :packet_log_entries, [:decom_state]
+  end
+end

lib/cosmos/dart/db/migrate/20180509204705_remove_indexes.rb

@@ -0,0 +1,29 @@
+class RemoveIndexes < ActiveRecord::Migration[5.1]
+
+  def change
+    remove_index :packet_log_entries, :is_tlm
+    remove_index :packet_log_entries, :target_id
+    remove_index :packet_log_entries, :meta_id
+    remove_index :packet_log_entries, :ready
+    remove_index :packet_log_entries, :packet_id
+    remove_index :packet_log_entries, :packet_log_id
+
+    ActiveRecord::Base.connection.tables.each do |table|
+      # Since the decommutation tables are created dynamically we search
+      # through all the tables looking for tables named something like
+      # tXXX_YYY where XXX is the PacketConfig ID and YYY is the table index
+      if table.to_s =~ /^t(\d+)_(\d+)$/ # ASCII art? No! Regex!
+        packet_config_id = $1.to_i
+        table_index = $2.to_i
+
+        ["", "_m", "_h", "_d"].each do |modifier|
+          table_name = table + modifier
+          remove_index table_name, :meta_id
+          remove_index table_name, :reduced_id
+          remove_index table_name, :reduced_state
+          add_index table_name, :reduced_state, :where => "reduced_state < 2"
+        end
+      end
+    end
+  end
+end

lib/cosmos/dart/db/migrate/20180510160002_remove_timestamps_from_ples.rb

@@ -0,0 +1,6 @@
+class RemoveTimestampsFromPles < ActiveRecord::Migration[5.1]
+  def change
+    remove_column :packet_log_entries, :created_at
+    remove_column :packet_log_entries, :updated_at
+  end
+end

lib/cosmos/dart/db/migrate/20180511194944_add_packet_log_id_to_tables.rb

@@ -0,0 +1,18 @@
+class AddPacketLogIdToTables < ActiveRecord::Migration[5.1]
+  def change
+    ActiveRecord::Base.connection.tables.each do |table|
+      # Since the decommutation tables are created dynamically we search
+      # through all the tables looking for tables named something like
+      # tXXX_YYY where XXX is the PacketConfig ID and YYY is the table index
+      if table.to_s =~ /^t(\d+)_(\d+)$/ # ASCII art? No! Regex!
+        packet_config_id = $1.to_i
+        table_index = $2.to_i
+
+        ["", "_m", "_h", "_d"].each do |modifier|
+          table_name = table + modifier
+          add_column table_name, :packet_log_id, :integer
+        end
+      end
+    end
+  end
+end

lib/cosmos/dart/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180116214338) do
+ActiveRecord::Schema.define(version: 20180511194944) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -54,18 +54,11 @@
     t.datetime "time", null: false
     t.integer "packet_log_id", null: false
     t.bigint "data_offset", null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
     t.bigint "meta_id"
     t.boolean "is_tlm", null: false
     t.integer "decom_state", default: 0
     t.boolean "ready", default: false
-    t.index ["is_tlm"], name: "index_packet_log_entries_on_is_tlm"
-    t.index ["meta_id"], name: "index_packet_log_entries_on_meta_id"
-    t.index ["packet_id"], name: "index_packet_log_entries_on_packet_id"
-    t.index ["packet_log_id"], name: "index_packet_log_entries_on_packet_log_id"
-    t.index ["ready"], name: "index_packet_log_entries_on_ready"
-    t.index ["target_id"], name: "index_packet_log_entries_on_target_id"
+    t.index ["decom_state"], name: "index_packet_log_entries_on_decom_state"
     t.index ["time"], name: "index_packet_log_entries_on_time"
   end
 
@@ -86,6 +79,19 @@
     t.index ["target_id", "name", "is_tlm"], name: "index_packets_on_target_id_and_name_and_is_tlm", unique: true
   end
 
+  create_table "statuses", force: :cascade do |t|
+    t.bigint "decom_count", default: 0
+    t.bigint "decom_error_count", default: 0
+    t.text "decom_message", default: ""
+    t.datetime "decom_message_time"
+    t.bigint "reduction_count", default: 0
+    t.bigint "reduction_error_count", default: 0
+    t.text "reduction_message", default: ""
+    t.datetime "reduction_message_time"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "system_configs", force: :cascade do |t|
     t.string "name", null: false
     t.datetime "created_at", null: false

lib/cosmos/dart/db/seeds.rb

@@ -5,3 +5,11 @@
 #
 #   movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }])
 #   Character.create(name: 'Luke', movie: movies.first)
+status = Status.first
+unless status
+  status = Status.new
+  time = Time.utc(1970, 1, 1)
+  status.decom_message_time = time
+  status.reduction_message_time = time
+  status.save!
+end

lib/cosmos/dart/lib/dart_common.rb

@@ -165,6 +165,7 @@ def setup_packet_config(packet, packet_id, packet_config)
         t.bigint :ple_id
         t.bigint :meta_id
         t.bigint :reduced_id
+        t.integer :packet_log_id
         t.integer :reduced_state, :default => 0
         table_data_types.each_with_index do |data_type, index|
           item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
@@ -183,9 +184,7 @@ def setup_packet_config(packet, packet_id, packet_config)
           end
         end
         t.index :time
-        t.index :meta_id
-        t.index :reduced_state
-        t.index :reduced_id
+        t.index :reduced_state, :where => "reduced_state < 2"
       end
       create_reduction_table("t#{packet_config.id}_#{table_index}_h", table_data_types, table_index) # hour
       create_reduction_table("t#{packet_config.id}_#{table_index}_m", table_data_types, table_index) # month
@@ -524,6 +523,7 @@ def lookup_item_id(packet_id, item_name)
   # @param array_size [Integer, nil] Size of the array or nil if no array
   # @return [Symbol] Database type such as :integer, :bigint, :string, etc.
   def cosmos_data_type_to_db_type(data_type, bit_size, array_size)
+    return nil if data_type.nil? or bit_size.nil?
     db_type = nil
     case data_type
     when :INT
@@ -674,6 +674,7 @@ def create_reduction_table(table_name, table_data_types, table_index)
       t.integer :num_samples
       t.bigint :meta_id
       t.bigint :reduced_id
+      t.integer :packet_log_id
       t.integer :reduced_state, :default => 0
       table_data_types.each_with_index do |data_type, index|
         item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
@@ -686,9 +687,7 @@ def create_reduction_table(table_name, table_data_types, table_index)
         end
       end
       t.index :start_time
-      t.index :meta_id
-      t.index :reduced_state
-      t.index :reduced_id
+      t.index :reduced_state, :where => "reduced_state < 2"
     end
   end