diff --git a/src/onedb/fsck.rb b/src/onedb/fsck.rb index d97c60f4028..3ba9dbbd8ef 100644 --- a/src/onedb/fsck.rb +++ b/src/onedb/fsck.rb @@ -109,8 +109,14 @@ def federated_tables FEDERATED_TABLES end - def nokogiri_doc(body) - Nokogiri::XML(body, nil, NOKOGIRI_ENCODING) do |c| + def nokogiri_doc(body, table = nil) + nk_enconding = NOKOGIRI_ENCODING + + unless table.nil? + nk_enconding = get_table_enconding(table) + end + + Nokogiri::XML(body, nil, nk_enconding) do |c| c.default_xml.noblanks end end diff --git a/src/onedb/fsck/cluster.rb b/src/onedb/fsck/cluster.rb index f9d415a4384..e4f146f9432 100644 --- a/src/onedb/fsck/cluster.rb +++ b/src/onedb/fsck/cluster.rb @@ -24,7 +24,8 @@ def check_fix_cluster @db.transaction do @db.fetch('SELECT * from cluster_pool') do |row| cluster_id = row[:oid] - doc = Document.new(row[:body]) + + doc = nokogiri_doc(row[:body], 'cluster_pool') # Hosts process_hosts(doc, cluster_id, cluster[cluster_id][:hosts]) @@ -53,21 +54,25 @@ def check_fix_cluster # @param cid [Integer] Cluster ID # @param hosts [Array] Hosts to process def process_hosts(doc, cid, hosts) - hosts_elem = doc.root.elements.delete('HOSTS') - hosts_new_elem = doc.root.add_element('HOSTS') + hosts_elem = doc.xpath('//HOSTS').remove + + hosts_new_elem = doc.create_element('HOSTS') + doc.root.add_child(hosts_new_elem) hosts.each do |id| - id_elem = hosts_elem.elements.delete("ID[.=#{id}]") + id_elem = hosts_elem.xpath("ID[.=#{id}]").remove - if id_elem.nil? + if id_elem.empty? log_error("Host #{id} is missing from cluster " \ "#{cid} host id list") end - hosts_new_elem.add_element('ID').text = id.to_s + aux = doc.create_element('ID') + aux.content = id.to_s + hosts_new_elem.add_child(aux) end - hosts_elem.each_element('ID') do |id| + hosts_elem.children.each do |id| id = id.text log_error("Host #{id} is in cluster #{cid} " \ @@ -81,18 +86,22 @@ def process_hosts(doc, cid, hosts) # @param cid [Integer] Cluster ID # @param datastores [Array] Datastores to process def process_ds(doc, cid, datastores) - ds_elem = doc.root.elements.delete('DATASTORES') - ds_new_elem = doc.root.add_element('DATASTORES') + ds_elem = doc.xpath('//DATASTORES').remove + + ds_new_elem = doc.create_element('DATASTORES') + doc.root.add_child(ds_new_elem) datastores.each do |id| - id_elem = ds_elem.elements.delete("ID[.=#{id}]") + id_elem = ds_elem.xpath("ID[.=#{id}]").remove - if id_elem.nil? + if id_elem.empty? log_error("Datastore #{id} is missing from cluster " \ "#{cid} datastore id list") end - ds_new_elem.add_element('ID').text = id.to_s + aux = doc.create_element('ID') + aux.content = id.to_s + ds_new_elem.add_child(aux) next unless @db.fetch('SELECT * FROM cluster_datastore_relation ' \ "WHERE cid=#{cid} AND oid=#{id}").empty? @@ -103,7 +112,7 @@ def process_ds(doc, cid, datastores) @db[:cluster_datastore_relation].insert(:cid => cid, :oid => id) end - ds_elem.each_element('ID') do |id| + ds_elem.children.each do |id| id = id.text log_error("Datastore #{id} is in cluster #{cid} datastore id " \ @@ -117,18 +126,22 @@ def process_ds(doc, cid, datastores) # @param cid [Integer] Cluster ID # @param vnets [Array] VNets to process def process_vnets(doc, cid, vnets) - vnets_elem = doc.root.elements.delete('VNETS') - vnets_new_elem = doc.root.add_element('VNETS') + vnets_elem = doc.xpath('//VNETS').remove + + vnets_new_elem = doc.create_element('VNETS') + doc.root.add_child(vnets_new_elem) vnets.each do |id| - id_elem = vnets_elem.elements.delete("ID[.=#{id}]") + id_elem = vnets_elem.xpath("ID[.=#{id}]").remove - if id_elem.nil? + if id_elem.empty? log_error("VNet #{id} is missing from cluster #{cid} " \ 'vnet id list') end - vnets_new_elem.add_element('ID').text = id.to_s + aux = doc.create_element('ID') + aux.content = id.to_s + vnets_new_elem.add_child(aux) next unless @db.fetch('SELECT * FROM cluster_network_relation ' \ "WHERE cid=#{cid} AND oid=#{id}").empty? @@ -139,7 +152,7 @@ def process_vnets(doc, cid, vnets) @db[:cluster_network_relation].insert(:cid => cid, :oid => id) end - vnets_elem.each_element('ID') do |id| + vnets_elem.children.each do |id| id = id.text log_error("VNet #{id} is in cluster #{cid} vnet id list, " \ diff --git a/src/onedb/fsck/datastore.rb b/src/onedb/fsck/datastore.rb index 140992c0f7e..5e95eeb1eec 100644 --- a/src/onedb/fsck/datastore.rb +++ b/src/onedb/fsck/datastore.rb @@ -19,12 +19,14 @@ def check_fix_datastore @db.transaction do @db.fetch('SELECT * from datastore_pool') do |row| ds_id = row[:oid] - doc = Document.new(row[:body]) - images_elem = doc.root.elements.delete('IMAGES') - images_new_elem = doc.root.add_element('IMAGES') + doc = nokogiri_doc(row[:body], 'datastore_pool') + images_elem = doc.root.xpath('IMAGES').remove + images_new_elem = doc.create_element('IMAGES') + + doc.root.add_child(images_new_elem) datastore[ds_id][:images].each do |id| - id_elem = images_elem.elements.delete("ID[.=#{id}]") + id_elem = images_elem.xpath("ID[.=#{id}]").remove if id_elem.nil? log_error( @@ -33,10 +35,11 @@ def check_fix_datastore ) end - images_new_elem.add_element('ID').text = id.to_s + i_e = doc.create_element('ID') + images_new_elem.add_child(i_e).content = id.to_s end - images_elem.each_element('ID') do |id_elem| + images_elem.children.each do |id_elem| log_error( "Image #{id_elem.text} is in Datastore #{ds_id} " \ 'image id list, but it should not' @@ -61,7 +64,7 @@ def check_datastore_cluster @fixes_datastore_cluster = {} @db.fetch('SELECT oid,body FROM datastore_pool') do |row| - doc = nokogiri_doc(row[:body]) + doc = nokogiri_doc(row[:body], 'datastore_pool') oid = row[:oid] doc.root.xpath('CLUSTERS/ID').each do |e| @@ -88,9 +91,9 @@ def check_datastore_image @fixes_datastore_image = {} @db.fetch('SELECT oid,body FROM image_pool') do |row| - doc = Document.new(row[:body]) - ds_id = doc.root.get_text('DATASTORE_ID').to_s.to_i - ds_name = doc.root.get_text('DATASTORE') + doc = nokogiri_doc(row[:body], 'datastore_pool') + ds_id = doc.root.xpath('DATASTORE_ID').text.to_i + ds_name = doc.root.xpath('DATASTORE').text if ds_id != -1 ds_entry = datastore[ds_id] @@ -100,7 +103,7 @@ def check_datastore_image 'but it does not exist. The image is probably ' \ "unusable, and needs to be deleted manually:\n" \ ' * The image contents should be deleted ' \ - "manually:\n #{doc.root.get_text('SOURCE')}\n" \ + "manually:\n #{doc.root.xpath('SOURCE').text}\n" \ ' * The DB entry can be then deleted with ' \ "the command:\n" \ ' DELETE FROM image_pool WHERE ' \ @@ -111,7 +114,7 @@ def check_datastore_image "for datastore #{ds_id}, #{ds_name}. " \ "It will be changed to #{ds_entry[:name]}") - doc.root.each_element('DATASTORE') do |e| + doc.root.xpath('DATASTORE').each do |e| e.text = ds_entry[:name] end diff --git a/src/onedb/fsck/host.rb b/src/onedb/fsck/host.rb index 3bcf1c0a580..6eec66b9e97 100644 --- a/src/onedb/fsck/host.rb +++ b/src/onedb/fsck/host.rb @@ -19,9 +19,10 @@ def check_host_cluster hosts_fix = @fixes_host_cluster = {} @db.fetch('SELECT oid,body,cid FROM host_pool') do |row| - doc = Document.new(row[:body]) - cid = doc.root.get_text('CLUSTER_ID').to_s.to_i - cname = doc.root.get_text('CLUSTER') + doc = nokogiri_doc(row[:body], 'host_pool') + + cid = doc.root.xpath("CLUSTER_ID").text.to_i + cname = doc.root.xpath("CLUSTER").text if cid != row[:cid] log_error("Host #{row[:oid]} is in cluster #{cid}, but cid " \ @@ -38,11 +39,11 @@ def check_host_cluster log_error("Host #{row[:oid]} is in cluster #{cid}, " \ 'but it does not exist') - doc.root.each_element('CLUSTER_ID') do |e| + doc.root.xpath('CLUSTER_ID').each do |e| e.text = '-1' end - doc.root.each_element('CLUSTER') do |e| + doc.root.xpath('CLUSTER').each do |e| e.text = '' end @@ -55,7 +56,7 @@ def check_host_cluster "cluster #{cid}, #{cname}. " \ "It will be changed to #{new_cluster}") - doc.root.each_element('CLUSTER') do |e| + doc.root.xpath('CLUSTER').each do |e| e.text = new_cluster end diff --git a/src/onedb/fsck/image.rb b/src/onedb/fsck/image.rb index 65c12bdddf9..fbd3329334a 100644 --- a/src/onedb/fsck/image.rb +++ b/src/onedb/fsck/image.rb @@ -11,9 +11,9 @@ def init_image_counters } end - doc = Document.new(row[:body]) + doc = nokogiri_doc(row[:body], 'image_pool') - doc.root.each_element("CLONING_ID") do |e| + doc.root.xpath("CLONING_ID").each do |e| img_id = e.text.to_i if counters[:image][img_id].nil? @@ -34,12 +34,12 @@ def check_image @db.transaction do @db[:image_pool].each do |row| - doc = Document.new(row[:body]) + doc = nokogiri_doc(row[:body], 'image_pool') oid = row[:oid] - persistent = ( doc.root.get_text('PERSISTENT').to_s == "1" ) - current_state = doc.root.get_text('STATE').to_s.to_i + persistent = ( doc.root.xpath('PERSISTENT').text == "1" ) + current_state = doc.root.xpath('STATE').text.to_i counters_img = counters[:image][oid] @@ -48,7 +48,7 @@ def check_image # DATA: CHECK: running vm counter with this image # rewrite running_vms - doc.root.each_element("RUNNING_VMS") {|e| + doc.root.xpath("RUNNING_VMS") {|e| if e.text != rvms.to_s log_error("Image #{oid} RUNNING_VMS has #{e.text} \tis\t#{rvms}") e.text = rvms @@ -56,22 +56,24 @@ def check_image } # re-do list of VM IDs - vms_elem = doc.root.elements.delete("VMS") + vms_elem = doc.root.xpath("VMS").remove - vms_new_elem = doc.root.add_element("VMS") + vms_new_elem = doc.create_element("VMS") + doc.root.add_child(vms_new_elem) # DATA: CHECK: running vm list with this image counters_img[:vms].each do |id| - id_elem = vms_elem.elements.delete("ID[.=#{id}]") + id_elem = vms_elem.xpath("ID[.=#{id}]").remove if id_elem.nil? log_error("VM #{id} is missing from Image #{oid} VM id list") end - vms_new_elem.add_element("ID").text = id.to_s + i_e = doc.create_element('ID') + vms_new_elem.add_child(i_e).content = id.to_s end - vms_elem.each_element("ID") do |id_elem| + vms_elem.children.each do |id_elem| log_error("VM #{id_elem.text} is in Image #{oid} VM id list, but it should not") end @@ -83,7 +85,7 @@ def check_image end # DATA: CHECK: Check number of clones - doc.root.each_element("CLONING_OPS") { |e| + doc.root.xpath("CLONING_OPS") { |e| if e.text != n_cloning_ops.to_s log_error("Image #{oid} CLONING_OPS has #{e.text} \tis\t#{n_cloning_ops}") e.text = n_cloning_ops @@ -91,43 +93,47 @@ def check_image } # re-do list of Images cloning this one - clones_elem = doc.root.elements.delete("CLONES") + clones_elem = doc.root.xpath("CLONES").remove - clones_new_elem = doc.root.add_element("CLONES") + clones_new_elem = doc.create_element("CLONES") + doc.root.add_child(clones_new_elem) # DATA: CHECK: image clones (is it used?) counters_img[:clones].each do |id| - id_elem = clones_elem.elements.delete("ID[.=#{id}]") + id_elem = clones_elem.xpath("ID[.=#{id}]").remove if id_elem.nil? log_error("Image #{id} is missing from Image #{oid} CLONES id list") end - clones_new_elem.add_element("ID").text = id.to_s + i_e = doc.create_element('ID') + clones_new_elem.add_child(i_e).content = id.to_s end - clones_elem.each_element("ID") do |id_elem| + clones_elem.children.each do |id_elem| log_error("Image #{id_elem.text} is in Image #{oid} CLONES id list, but it should not") end # re-do list of Apps cloning this one - clones_elem = doc.root.elements.delete("APP_CLONES") + clones_elem = doc.root.xpath("APP_CLONES").remove - clones_new_elem = doc.root.add_element("APP_CLONES") + clones_new_elem = doc.create_element("APP_CLONES") + doc.root.add_child(clones_new_elem) # DATA: CHECK: check app clones # DATA: TODO: understand app clones and image clones counters_img[:app_clones].each do |id| - id_elem = clones_elem.elements.delete("ID[.=#{id}]") + id_elem = clones_elem.xpath("ID[.=#{id}]").remove if id_elem.nil? log_error("Marketplace App #{id} is missing from Image #{oid} APP_CLONES id list") end - clones_new_elem.add_element("ID").text = id.to_s + h_e = doc.create_element('ID') + clones_new_elem.add_child(i_e).content = id.to_s end - clones_elem.each_element("ID") do |id_elem| + clones_elem.children.each do |id_elem| log_error("Marketplace App #{id_elem.text} is in Image #{oid} APP_CLONES id list, but it should not") end @@ -159,7 +165,7 @@ def check_image end end - doc.root.each_element("STATE") { |e| + doc.root.xpath("STATE") { |e| if e.text != state.to_s log_error("Image #{oid} has STATE " << OpenNebula::Image::IMAGE_STATES[e.text.to_i] << diff --git a/src/onedb/fsck/marketplaceapp.rb b/src/onedb/fsck/marketplaceapp.rb index 8666b79c762..6dc17943477 100644 --- a/src/onedb/fsck/marketplaceapp.rb +++ b/src/onedb/fsck/marketplaceapp.rb @@ -22,10 +22,10 @@ def check_marketplaceapp # DATA: go through all apps @db.fetch("SELECT oid,body FROM marketplaceapp_pool") do |row| - doc = Document.new(row[:body]) + doc = nokogiri_doc(row[:body], 'marketplaceapp_pool') - market_id = doc.root.get_text('MARKETPLACE_ID').to_s.to_i - market_name = doc.root.get_text('MARKETPLACE') + market_id = doc.root.xpath('MARKETPLACE_ID').text.to_i + market_name = doc.root.xpath('MARKETPLACE').text #################################################################### # DATA: TODO, BUG: this code will only work for a standalone oned. @@ -34,8 +34,8 @@ def check_marketplaceapp #################################################################### # DATA: get image origin id. Does it work? - origin_id = doc.root.get_text('ORIGIN_ID').to_s.to_i - if origin_id >= 0 && doc.root.get_text('STATE').to_s.to_i == 2 # LOCKED + origin_id = doc.root.xpath('ORIGIN_ID').text.to_i + if origin_id >= 0 && doc.root.xpath('STATE').text.to_i == 2 # LOCKED counters[:image][origin_id][:app_clones].add(row[:oid]) end @@ -56,7 +56,7 @@ def check_marketplaceapp if market_name != market_entry[:name] log_error("Marketplace App #{row[:oid]} has a wrong name for marketplace #{market_id}, #{market_name}. It will be changed to #{market_entry[:name]}") - doc.root.each_element('MARKETPLACE') do |e| + doc.root.xpath('MARKETPLACE').each do |e| e.text = market_entry[:name] end diff --git a/src/onedb/onedb_backend.rb b/src/onedb/onedb_backend.rb index 395fed99e6a..8f58656efd9 100644 --- a/src/onedb/onedb_backend.rb +++ b/src/onedb/onedb_backend.rb @@ -319,6 +319,54 @@ def encoding @encoding = table_enc end + def get_table_enconding(table) + enconding = nil + + @db.fetch( + 'select CCSA.character_set_name FROM information_schema.' \ + '`TABLES` T, information_schema.' \ + '`COLLATION_CHARACTER_SET_APPLICABILITY` CCSA WHERE ' \ + 'CCSA.collation_name = T.table_collation AND ' \ + "T.table_schema = '#{@db_name}' AND "\ + "T.table_name = '#{table}';" + ) do |row| + enconding = row[:character_set_name] + end + + table_to_nk(enconding) + end + + def table_to_nk(enconding) + case(enconding) + when 'utf8mb4' + 'UTF-8' + when 'utf16le' + 'UTF16LE' + when 'utf16' + 'UTF16BE' + when 'ucs2' + 'UCS2' + when 'latin1' + 'ISO-8859-1' + when 'latin2' + 'ISO-8859-2' + when 'greek' + 'ISO-8859-7' + when 'hebrew' + 'ISO-8859-8' + when 'latin5' + 'ISO-8859-9' + when 'sjis' + 'SHIFT-JIS' + when 'ujis' + 'EUC-JP' + when 'ascii' + 'ASCII' + else + 'NONE' + end + end + def restore(bck_file, force=nil, federated=false) connect_db @@ -465,6 +513,10 @@ def backup(bck_file, federated = false) puts "Use 'onedb restore' to restore the DB." end + def get_table_enconding(table = nil) + 'UTF-8' + end + def restore(bck_file, force=nil, federated=false) if !federated if File.exists?(@sqlite_file) && !force