Skip to content

Commit

Permalink
B #4165: Fsck take enconding from database tables
Browse files Browse the repository at this point in the history
  • Loading branch information
Alejandro Huertas committed Feb 7, 2020
1 parent 358dbd2 commit 6d4cc38
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 37 deletions.
10 changes: 8 additions & 2 deletions src/onedb/fsck.rb
View file
Expand Up @@ -109,8 +109,14 @@ def federated_tables
FEDERATED_TABLES
end

def nokogiri_doc(body)
Nokogiri::XML(body, nil, NOKOGIRI_ENCODING) do |c|
def nokogiri_doc(body, table = nil)
nk_enconding = NOKOGIRI_ENCODING

unless table.nil?
nk_enconding = get_table_enconding(table)
end

Nokogiri::XML(body, nil, nk_enconding) do |c|
c.default_xml.noblanks
end
end
Expand Down
27 changes: 15 additions & 12 deletions src/onedb/fsck/datastore.rb
View file
Expand Up @@ -19,12 +19,14 @@ def check_fix_datastore
@db.transaction do
@db.fetch('SELECT * from datastore_pool') do |row|
ds_id = row[:oid]
doc = Document.new(row[:body])
images_elem = doc.root.elements.delete('IMAGES')
images_new_elem = doc.root.add_element('IMAGES')
doc = nokogiri_doc(row[:body], 'datastore_pool')
images_elem = doc.root.xpath('IMAGES').remove
images_new_elem = doc.create_element('IMAGES')

doc.root.add_child(images_new_elem)

datastore[ds_id][:images].each do |id|
id_elem = images_elem.elements.delete("ID[.=#{id}]")
id_elem = images_elem.xpath("ID[.=#{id}]").remove

if id_elem.nil?
log_error(
Expand All @@ -33,10 +35,11 @@ def check_fix_datastore
)
end

images_new_elem.add_element('ID').text = id.to_s
i_e = doc.create_element('ID')
images_new_elem.add_child(i_e).content = id.to_s
end

images_elem.each_element('ID') do |id_elem|
images_elem.children.each do |id_elem|
log_error(
"Image #{id_elem.text} is in Datastore #{ds_id} " \
'image id list, but it should not'
Expand All @@ -61,7 +64,7 @@ def check_datastore_cluster
@fixes_datastore_cluster = {}

@db.fetch('SELECT oid,body FROM datastore_pool') do |row|
doc = nokogiri_doc(row[:body])
doc = nokogiri_doc(row[:body], 'datastore_pool')
oid = row[:oid]

doc.root.xpath('CLUSTERS/ID').each do |e|
Expand All @@ -88,9 +91,9 @@ def check_datastore_image
@fixes_datastore_image = {}

@db.fetch('SELECT oid,body FROM image_pool') do |row|
doc = Document.new(row[:body])
ds_id = doc.root.get_text('DATASTORE_ID').to_s.to_i
ds_name = doc.root.get_text('DATASTORE')
doc = nokogiri_doc(row[:body], 'datastore_pool')
ds_id = doc.root.xpath('DATASTORE_ID').text.to_i
ds_name = doc.root.xpath('DATASTORE').text

if ds_id != -1
ds_entry = datastore[ds_id]
Expand All @@ -100,7 +103,7 @@ def check_datastore_image
'but it does not exist. The image is probably ' \
"unusable, and needs to be deleted manually:\n" \
' * The image contents should be deleted ' \
"manually:\n #{doc.root.get_text('SOURCE')}\n" \
"manually:\n #{doc.root.xpath('SOURCE').text}\n" \
' * The DB entry can be then deleted with ' \
"the command:\n" \
' DELETE FROM image_pool WHERE ' \
Expand All @@ -111,7 +114,7 @@ def check_datastore_image
"for datastore #{ds_id}, #{ds_name}. " \
"It will be changed to #{ds_entry[:name]}")

doc.root.each_element('DATASTORE') do |e|
doc.root.xpath('DATASTORE') do |e|
e.text = ds_entry[:name]
end

Expand Down
52 changes: 29 additions & 23 deletions src/onedb/fsck/image.rb
View file
Expand Up @@ -11,9 +11,9 @@ def init_image_counters
}
end

doc = Document.new(row[:body])
doc = nokogiri_doc(row[:body], 'image_pool')

doc.root.each_element("CLONING_ID") do |e|
doc.root.xpath("CLONING_ID") do |e|
img_id = e.text.to_i

if counters[:image][img_id].nil?
Expand All @@ -34,12 +34,12 @@ def check_image

@db.transaction do
@db[:image_pool].each do |row|
doc = Document.new(row[:body])
doc = nokogiri_doc(row[:body], 'image_pool')

oid = row[:oid]

persistent = ( doc.root.get_text('PERSISTENT').to_s == "1" )
current_state = doc.root.get_text('STATE').to_s.to_i
persistent = ( doc.root.xpath('PERSISTENT').text == "1" )
current_state = doc.root.xpath('STATE').text.to_i

counters_img = counters[:image][oid]

Expand All @@ -48,30 +48,32 @@ def check_image

# DATA: CHECK: running vm counter with this image
# rewrite running_vms
doc.root.each_element("RUNNING_VMS") {|e|
doc.root.xpath("RUNNING_VMS") {|e|
if e.text != rvms.to_s
log_error("Image #{oid} RUNNING_VMS has #{e.text} \tis\t#{rvms}")
e.text = rvms
end
}

# re-do list of VM IDs
vms_elem = doc.root.elements.delete("VMS")
vms_elem = doc.root.xpath("VMS").remove

vms_new_elem = doc.root.add_element("VMS")
vms_new_elem = doc.create_element("VMS")
doc.root.add_child(vms_new_elem)

# DATA: CHECK: running vm list with this image
counters_img[:vms].each do |id|
id_elem = vms_elem.elements.delete("ID[.=#{id}]")
id_elem = vms_elem.xpath("ID[.=#{id}]").remove

if id_elem.nil?
log_error("VM #{id} is missing from Image #{oid} VM id list")
end

vms_new_elem.add_element("ID").text = id.to_s
i_e = doc.create_element('ID')
vms_new_elem.add_child(i_e).content = id.to_s
end

vms_elem.each_element("ID") do |id_elem|
vms_elem.children.each do |id_elem|
log_error("VM #{id_elem.text} is in Image #{oid} VM id list, but it should not")
end

Expand All @@ -83,51 +85,55 @@ def check_image
end

# DATA: CHECK: Check number of clones
doc.root.each_element("CLONING_OPS") { |e|
doc.root.xpath("CLONING_OPS") { |e|
if e.text != n_cloning_ops.to_s
log_error("Image #{oid} CLONING_OPS has #{e.text} \tis\t#{n_cloning_ops}")
e.text = n_cloning_ops
end
}

# re-do list of Images cloning this one
clones_elem = doc.root.elements.delete("CLONES")
clones_elem = doc.root.xpath("CLONES").remove

clones_new_elem = doc.root.add_element("CLONES")
clones_new_elem = doc.create_element("CLONES")
doc.root.add_child(clones_new_elem)

# DATA: CHECK: image clones (is it used?)
counters_img[:clones].each do |id|
id_elem = clones_elem.elements.delete("ID[.=#{id}]")
id_elem = clones_elem.xpath("ID[.=#{id}]").remove

if id_elem.nil?
log_error("Image #{id} is missing from Image #{oid} CLONES id list")
end

clones_new_elem.add_element("ID").text = id.to_s
i_e = doc.create_element('ID')
clones_new_elem.add_child(i_e).content = id.to_s
end

clones_elem.each_element("ID") do |id_elem|
clones_elem.children.each do |id_elem|
log_error("Image #{id_elem.text} is in Image #{oid} CLONES id list, but it should not")
end

# re-do list of Apps cloning this one
clones_elem = doc.root.elements.delete("APP_CLONES")
clones_elem = doc.root.xpath("APP_CLONES").remove

clones_new_elem = doc.root.add_element("APP_CLONES")
clones_new_elem = doc.create_element("APP_CLONES")
doc.root.add_child(clones_new_elem)

# DATA: CHECK: check app clones
# DATA: TODO: understand app clones and image clones
counters_img[:app_clones].each do |id|
id_elem = clones_elem.elements.delete("ID[.=#{id}]")
id_elem = clones_elem.xpath("ID[.=#{id}]").remove

if id_elem.nil?
log_error("Marketplace App #{id} is missing from Image #{oid} APP_CLONES id list")
end

clones_new_elem.add_element("ID").text = id.to_s
h_e = doc.create_element('ID')
clones_new_elem.add_child(i_e).content = id.to_s
end

clones_elem.each_element("ID") do |id_elem|
clones_elem.children.each do |id_elem|
log_error("Marketplace App #{id_elem.text} is in Image #{oid} APP_CLONES id list, but it should not")
end

Expand Down Expand Up @@ -159,7 +165,7 @@ def check_image
end
end

doc.root.each_element("STATE") { |e|
doc.root.xpath("STATE") { |e|
if e.text != state.to_s
log_error("Image #{oid} has STATE " <<
OpenNebula::Image::IMAGE_STATES[e.text.to_i] <<
Expand Down
52 changes: 52 additions & 0 deletions src/onedb/onedb_backend.rb
View file
Expand Up @@ -319,6 +319,54 @@ def encoding
@encoding = table_enc
end

def get_table_enconding(table)
enconding = nil

@db.fetch(
'select CCSA.character_set_name FROM information_schema.' \
'`TABLES` T, information_schema.' \
'`COLLATION_CHARACTER_SET_APPLICABILITY` CCSA WHERE ' \
'CCSA.collation_name = T.table_collation AND ' \
"T.table_schema = '#{@db_name}' AND "\
"T.table_name = '#{table}';"
) do |row|
enconding = row[:character_set_name]
end

table_to_nk(enconding)
end

def table_to_nk(enconding)
case(enconding)
when 'utf8mb4'
'UTF-8'
when 'utf16le'
'UTF16LE'
when 'utf16'
'UTF16BE'
when 'ucs2'
'UCS2'
when 'latin1'
'ISO-8859-1'
when 'latin2'
'ISO-8859-2'
when 'greek'
'ISO-8859-7'
when 'hebrew'
'ISO-8859-8'
when 'latin5'
'ISO-8859-9'
when 'sjis'
'SHIFT-JIS'
when 'ujis'
'EUC-JP'
when 'ascii'
'ASCII'
else
'NONE'
end
end

def restore(bck_file, force=nil, federated=false)
connect_db

Expand Down Expand Up @@ -465,6 +513,10 @@ def backup(bck_file, federated = false)
puts "Use 'onedb restore' to restore the DB."
end

def get_table_enconding(table = nil)
'UTF-8'
end

def restore(bck_file, force=nil, federated=false)
if !federated
if File.exists?(@sqlite_file) && !force
Expand Down

0 comments on commit 6d4cc38

Please sign in to comment.