Skip to content

Commit

Permalink
Improve accounting, warnings and other use of located metadata files.
Browse files Browse the repository at this point in the history
  • Loading branch information
@gkellogg
gkellogg committed Jul 9, 2015
1 parent 5d192e6 commit 655aef5
Show file tree
Hide file tree
Showing 5 changed files with 47 additions and 46 deletions.
26 changes: 16 additions & 10 deletions lib/rdf/tabular/metadata.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,18 +192,19 @@ def self.for_input(input, options = {})
# Search for metadata until found

# load link metadata, if available
all_locs = []
if !metadata && input.respond_to?(:links) &&
link = input.links.find_link(%w(rel describedby))
loc = RDF::URI(base).join(link.href).to_s
md = Metadata.open(loc, options.merge(filenames: loc, reason: "load linked metadata: #{loc}"))
all_locs << loc if md
# Metadata must describe file to be useful
metadata = md if md && md.describes_file(base)
metadata = md if md && md.describes_file?(base)
end

found_metadata = !!metadata
locs = []
# If we still don't have metadata, load the site-wide configuration file and use templates found there as locations
if !found_metadata && base
if !metadata && base
templates = site_wide_config(base)
debug("for_input", options) {"templates: #{templates.map(&:to_s).inspect}"}
locs = templates.map do |template|
Expand All @@ -216,8 +217,8 @@ def self.for_input(input, options = {})
metadata ||= begin
md = Metadata.open(loc, options.merge(filenames: loc, reason: "load found metadata: #{loc}"))
# Metadata must describe file to be useful
found_metadata ||= loc if md
md if md && md.describes_file(base)
all_locs << loc if md
md if md && md.describes_file?(base)
rescue IOError
debug("for_input", options) {"failed to load found metadata #{loc}: #{$!}"}
nil
Expand All @@ -226,9 +227,9 @@ def self.for_input(input, options = {})
end

# If Metadata was found, but no metadata describes the file, issue a warning
if found_metadata && !metadata
if !all_locs.empty? && !metadata
warnings = options.fetch(:warnings, [])
warnings << "Found metadata at #{locs.join(",")}, which does not describe #{base}, ignoring"
warnings << "Found metadata at #{all_locs.join(",")}, which does not describe #{base}, ignoring"
if options[:validate] && !options[:warnings]
$stderr.puts "Warnings: #{warnings.join("\n")}"
end
Expand Down Expand Up @@ -954,7 +955,7 @@ def has_annotations?
# Does this metadata describe the file (URL)?
# @param [RDF::URL] url
# @return [Boolean]
def describes_file(url)
def describes_file?(url)
case self
when TableGroup
tables.any? {|t| t.url == url}
Expand All @@ -977,8 +978,13 @@ def verify_compatible!(other)
end
else
# Tables must have the same url
raise Error, "Tables must have the same url: #{url.inspect} vs #{other.url.inspect}}" unless
url == other.url
unless url == other.url
if @options[:validate]
raise Error, "Tables must have the same url: #{url.inspect} vs #{other.url.inspect}}"
else
warn "Tables must have the same url: #{url.inspect} vs #{other.url.inspect}}"
end
end

# Each column description within B MUST match the corresponding column description in A for non-virtual columns
non_virtual_columns = Array(tableSchema.columns).reject(&:virtual)
Expand Down
28 changes: 17 additions & 11 deletions lib/rdf/tabular/reader.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,11 +156,14 @@ def each_statement(&block)
end
end unless minimal?

# If we were originally given tabular data as input, simply use that, rather than opening the table URL. This allows buffered data to be used as input
if options[:original_input] && input.tables.first.url.to_s.empty?
# If we were originally given tabular data as input, simply use that, rather than opening the table URL. This allows buffered data to be used as input.
# This case also handles found metadata that doesn't describe the input file
if options[:original_input] && !input.describes_file?(options[:base_uri])
table_resource = RDF::Node.new
add_statement(0, table_group, CSVW.table, table_resource) unless minimal?
Reader.new(options[:original_input], options.merge(
metadata: input.tables.first,
base: input.tables.first.url,
no_found_metadata: true,
table_resource: table_resource,
warnings: @warnings,
Expand All @@ -180,7 +183,6 @@ def each_statement(&block)
table_resource = table.id || RDF::Node.new
add_statement(0, table_group, CSVW.table, table_resource) unless minimal?
Reader.open(table.url, options.merge(
format: :tabular,
metadata: table,
base: table.url,
no_found_metadata: true,
Expand Down Expand Up @@ -434,26 +436,30 @@ def to_hash(options = {})

table_group['tables'] = tables

if options[:original_input] && input.tables.first.url.to_s.empty?
if options[:original_input] && !input.describes_file?(options[:base_uri])
Reader.new(options[:original_input], options.merge(
base: options[:base],
metadata: input.tables.first,
base: input.tables.first.url,
minimal: minimal?,
no_found_metadata: true
no_found_metadata: true,
warnings: @warnings,
errors: @errors,
)) do |r|
case table = r.to_hash(options)
when Array then tables += table
when Hash then tables << table
case t = r.to_hash(options)
when Array then tables += t unless input.tables.first.suppressOutput
when Hash then tables << t unless input.tables.first.suppressOutput
end
end
else
input.each_table do |table|
next if table.suppressOutput && !validate?
Reader.open(table.url, options.merge(
format: :tabular,
metadata: table,
base: table.url,
minimal: minimal?,
no_found_metadata: true
no_found_metadata: true,
warnings: @warnings,
errors: @errors,
)) do |r|
case t = r.to_hash(options)
when Array then tables += t unless table.suppressOutput
Expand Down
8 changes: 3 additions & 5 deletions script/tc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,17 +66,14 @@ def run_tc(t, options)
else # RDF or Validation
result = if result
result
elsif t.sparql?
# FIXME
"passed"
elsif t.evaluate?
graph << reader
output_graph = RDF::Repository.load(t.result, format: :turtle, base_uri: t.base)
graph.isomorphic_with?(output_graph) ? "passed" : "failed"
elsif t.validation?
reader.validate!
result = (t.warning? ? (t.warnings.length > 0) : t.warnings.empty?) ? "passed" : "failed"
end
result = "failed" if result == "passed" && t.warning? && t.warnings.empty?
end
else
begin
Expand All @@ -96,7 +93,7 @@ def run_tc(t, options)
end
rescue ::RDF::Tabular::Error, IOError => e
# Special case
if t.negative_test? && t.validation?
if t.negative_test?
result = "passed"
else
STDERR.puts "#{"exception:" unless options[:quiet]}: #{e}"
Expand All @@ -115,6 +112,7 @@ def run_tc(t, options)
end


options[:output].puts("\nWarnings:\n" + t.warnings.join("\n")) unless options[:quiet]
if t.json?
options[:output].puts("\nOutput:\n" + json_result) unless options[:quiet]
else
Expand Down
4 changes: 0 additions & 4 deletions spec/suite_helper.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,10 +144,6 @@ def evaluate?
type.to_s.include?("To")
end

def sparql?
type.to_s.include?("Sparql")
end

def rdf?
result.to_s.end_with?(".ttl")
end
Expand Down
27 changes: 11 additions & 16 deletions spec/suite_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
Fixtures::SuiteTest::Manifest.open(manifest, manifest[0..-8]) do |m|
describe m.comment do
m.entries.each do |t|
#next unless t.id.match(/test098/)
#next unless t.id.match(/test13/)
next if t.approval =~ /Rejected/
specify "#{t.id.split("/").last}: #{t.name} - #{t.comment}" do
t.debug = []
Expand Down Expand Up @@ -50,31 +50,26 @@
expect(::JSON.parse(result)).to be_a(Hash)
end
else # RDF or Validation
if t.sparql?
graph << reader
RDF::Util::File.open_file(t.result) do |query|
expect(graph).to pass_query(query, t)
end
elsif t.evaluate?
if t.evaluate?
graph << reader
output_graph = RDF::Repository.load(t.result, format: :ttl, base_uri: t.base)
expect(graph).to be_equivalent_graph(output_graph, t)
elsif t.validation?
expect {reader.validate!}.not_to raise_error

if t.warning?
expect(t.warnings.length).to be >= 1
else
expect(t.warnings).to produce [], t
end
expect(t.errors).to produce [], t
end
end

if t.warning?
expect(t.warnings.length).to be >= 1
else
expect(t.warnings).to produce [], t
end
expect(t.errors).to produce [], t
elsif t.json?
expect {
reader.to_json
}.to raise_error(RDF::Tabular::Error)
elsif t.sparql? || t.evaluate?
elsif t.evaluate?
expect {
graph << reader
}.to raise_error(RDF::ReaderError)
Expand All @@ -84,7 +79,7 @@
end
rescue IOError, RDF::Tabular::Error
# Special case
unless t.negative_test? && t.validation?
unless t.negative_test?
raise
end
end
Expand Down

0 comments on commit 655aef5

Please sign in to comment.