Added yus_migrate_to_utf_8

yus_migrate_to_utf_8

@@ -0,0 +1,213 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+require 'fileutils'
+require 'syck'
+
+here = File.expand_path(File.join(File.dirname(File.dirname(__FILE__)), 'lib'))
+$: << here
+require 'logger'
+require 'needle'
+require 'odba/id_server'
+require 'rrba/server'
+require 'yus/entity'
+require 'yus/privilege'
+require 'odba/connection_pool'
+require 'odba/drbwrapper'
+require 'odba/storage'
+require 'rclconf'
+require 'odba/18_19_loading_compatibility'
+
+module Yus
+  attr_reader :config
+
+  def self.load_config
+    default_dir = File.join(ENV['HOME'], '.yus')
+    default_config_files = [
+      File.join(default_dir, 'yus.yml'),
+      '/etc/yus/yus.yml',
+    ]
+    defaults = {
+      'cleaning_interval' =>  300,
+      'config'            => default_config_files,
+      'db_name'           => 'yus',
+      'db_user'           => 'yus',
+      'db_auth'           => 'yus',
+      'db_backend'        => :psql,
+      'digest'            => Digest::SHA256,
+      'log_file'          => STDERR,
+      'log_level'         => 'INFO',
+      'persistence'       => 'odba',
+      'root_name'         => 'admin',
+      'root_pass'         => nil,
+      'server_url'        => 'drbssl://localhost:9997',
+      'session_timeout'   => 300,
+      'ssl_key'           => File.expand_path('../data/yus.key',
+                                              File.dirname(__FILE__)),
+      'ssl_cert'          => File.expand_path('../data/yus.crt',
+                                              File.dirname(__FILE__)),
+      'token_lifetime'    => 30,
+      'yus_dir'           => default_dir,
+    }
+
+    @config = RCLConf::RCLConf.new(ARGV, defaults)
+    @config.load(@config.config)
+
+    require File.join('yus', 'persistence', @config.persistence)
+  end
+
+  class Server
+    # http://stackoverflow.com/questions/2982677/ruby-1-9-invalid-byte-sequence-in-utf-8
+    # https://robots.thoughtbot.com/fight-back-utf-8-invalid-byte-sequences
+    def sanitize_utf8(string)
+      return nil if string.nil?
+      # return string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+      return string if string.valid_encoding?
+      if string.force_encoding(Encoding::ISO_8859_1).valid_encoding?
+        string.force_encoding(Encoding::ISO_8859_1).clone.encode(Encoding::UTF_8)
+      else
+        string.chars.select { |c| c.valid_encoding? }.join
+      end
+    end
+    def _migrate_to_utf8 queue, table, opts={}
+      while obj = queue.shift do
+        if obj.is_a?(Numeric)
+          begin
+            obj = ODBA.cache.fetch obj
+          rescue ODBA::OdbaError
+            return
+          end
+        else
+          obj = obj.odba_instance
+        end
+        puts " #{__LINE__}: Migrating #{obj.class} #{obj.to_s}" if $VERBOSE
+        return unless obj
+        _migrate_obj_to_utf8 obj, queue, table, opts
+        obj.odba_store unless obj.odba_unsaved?
+      end
+    end
+    def _migrate_obj_to_utf8 obj, queue, table, opts={}
+      obj.instance_variables.each do |name|
+        child = obj.instance_variable_get name
+        if child.respond_to?(:odba_unsaved?) && !child.odba_unsaved? \
+          && obj.respond_to?(:odba_serializables) \
+          && obj.odba_serializables.include?(name)
+          child.instance_variable_set '@odba_persistent', nil
+        end
+        child = _migrate_child_to_utf8 child, queue, table, opts
+        obj.instance_variable_set name, child
+      end
+      if obj.is_a?(Array)
+        obj.collect! do |child|
+          _migrate_child_to_utf8 child, queue, table, opts
+        end
+      end
+      if obj.is_a?(Hash)
+        obj.dup.each do |key, child|
+          obj.store key, _migrate_child_to_utf8(child, queue, table, opts)
+        end
+      end
+      obj
+    end
+    def _migrate_child_to_utf8 child, queue, table, opts={}
+      @serialized ||= {}
+      case child
+      when ODBA::Persistable, ODBA::Stub
+        if child = child.odba_instance
+          if child.odba_unsaved?
+            _migrate_to_utf8 [child], table, opts
+          elsif opts[:all]
+            odba_id = child.odba_id
+            unless table[odba_id]
+              table.store odba_id, true
+              queue.push odba_id
+            end
+          end
+        end
+      when String
+        old = child.encoding
+        if ( child.encoding != Encoding::UTF_8 && child.force_encoding(Encoding::ISO_8859_1).valid_encoding? ) ||
+           ( child.encoding == Encoding::UTF_8 && !child.valid_encoding? )
+          child = child.force_encoding(Encoding::ISO_8859_1).clone.encode(Encoding::UTF_8)
+          puts "force_encoding from ISO_8859_1 #{old}. is now #{child}" unless child.to_i > 0
+        end
+        case child.encoding.to_s
+        when /ASCII-8BIT|US-ASCII/
+          # nothing todo
+        when /UTF-8/
+          puts "UTF-8: for #{child.to_s}" if $VERBOSE
+          child = sanitize_utf8(child)
+        when /ISO-8859-1/i
+          child = sanitize_utf8(child)
+          # child = child.force_encoding('UTF-8')
+          puts "force_encoding from #{old}. is now #{child}"
+        else
+          puts "Unhandeled encoding #{child.encoding}"
+        # child = child.force_encoding
+        end
+      when
+        Yus::Entity,
+        Yus::Privilege
+        child = _migrate_obj_to_utf8 child, queue, table, opts
+      when Float, Fixnum, TrueClass, FalseClass, NilClass,
+        Symbol, Time, Date, DateTime
+        # do nothing
+      else
+        @ignored ||= {}
+        unless @ignored[child.class]
+          @ignored.store child.class, true
+          warn "ignoring #{child.class}"
+        end
+      end
+      child
+    rescue SystemStackError
+      puts child.class
+      raise
+    end
+  end
+  class Entity
+    include ODBA::Persistable
+    class << self
+      alias :all :odba_extent
+    end
+  end
+  class Privilege
+    include ODBA::Persistable
+    class << self
+      alias :all :odba_extent
+    end
+  end
+  def self.migrate_to_uf_8
+    self.load_config
+    @logger = Logger.new("/tmp/#{File.basename(__FILE__)}.log")
+    @logger.level = Logger::DEBUG
+    @logger.info  "#{Time.now}: Calling  #{File.basename(__FILE__)}"
+
+    puts "storage #{@config.db_name} name #{@config.db_user} auth #{@config.db_auth} pass #{@config.db_pass} storage is #{@config.persistence}"
+    require 'odba/connection_pool'
+    require 'odba/drbwrapper'
+    DRb.install_id_conv ODBA::DRbIdConv.new
+    ODBA.storage.dbi = ODBA::ConnectionPool.new("DBI:Pg:#{@config.db_name}", @config.db_user, @config.db_auth)
+    ODBA.cache.setup
+    @persistence = Yus::Persistence::Odba.new
+    ODBA.cache.setup
+
+    DRb.install_id_conv ODBA::DRbIdConv.new
+    @server = Yus::Server.new(@persistence, @config, @logger)
+    @server.extend(DRbUndumped)
+    {
+      :entitites => Yus::Entity.odba_extent,
+      :privileges => Yus::Privilege.odba_extent,
+      }.each do |name, to_migrate|
+      msg = "#{Time.now}: Start migrating #{to_migrate.size} #{name}"
+      @logger.info msg
+      puts msg
+      @server._migrate_to_utf8(to_migrate, {})
+    end
+    puts "#{Time.now}: Finished #{File.basename(__FILE__)}"
+  rescue Exception => error
+    @logger.error('fatal') { error } if @logger
+    raise
+  end
+end
+
+Yus.migrate_to_uf_8