Implement a Wikidata identifier for Siegfried

Wikidata contains a host of file format information. Wikidata records
provide information about format names, identifiers (PUID, LoC FDD),
file-extensions, MIMEtypes, and format identification patterns. This
means it might be quite handy as an extension to Siegfried's
capabilties.

This is the first-cut of that work. We are able to harvest
information from the Wikidata Query Service. Create an identifier.
And consume the information in the identifier to match file-formats
using format-identification patterns taken directly from the
service.

We attempt to return new information not otherwise present in other
identifiers yet, for example, signature provenance. Provenance is
recorded in Wikidata as well as the date a format-identification
pattern was added. We try and replay this to the user to enrich
what is available to them.

This work is graciously supported by Yale University Library
(Euan Cochran, Kat Thornton) and Richard Lehane. It has been fun
trying to put this out there for folk.

Author: ross-spencer

cmd/roy/data/wikidata/wikidata-test-definitions

@@ -0,0 +1,62 @@
+// Copyright 2020 Ross Spencer, Richard Lehane. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/richardlehane/siegfried/pkg/config"
+	"github.com/ross-spencer/spargo/pkg/spargo"
+)
+
+// harvestWikidata will connect to the configured Wikidata query service
+// and save the results of the configured query to disk.
+func harvestWikidata() error {
+	log.Printf(
+		"Roy (Wikidata): Harvesting Wikidata definitions: lang '%s'",
+		config.WikidataLang(),
+	)
+	err := os.MkdirAll(config.WikidataHome(), os.ModePerm)
+	if err != nil {
+		return fmt.Errorf(
+			"Roy (Wikidata): Error harvesting Wikidata definitions: '%s'",
+			err,
+		)
+	}
+	log.Printf(
+		"Roy (Wikidata): Harvesting definitions from: '%s'",
+		config.WikidataEndpoint(),
+	)
+	sparqlMe := spargo.SPARQLClient{}
+	sparqlMe.ClientInit(config.WikidataEndpoint(), config.WikidataSPARQL())
+	sparqlMe.SetUserAgent(config.UserAgent())
+	res := sparqlMe.SPARQLGo()
+	path := config.WikidataDefinitionsPath()
+	err = ioutil.WriteFile(path, []byte(res.Human), config.WikidataFileMode())
+	if err != nil {
+		return fmt.Errorf(
+			"Roy (Wikidata): Error harvesting Wikidata: '%s'",
+			err,
+		)
+	}
+	log.Printf(
+		"Roy (Wikidata): Harvesting Wikidata definitions '%s' complete",
+		path,
+	)
+	return nil
+}

cmd/roy/data/wikidata/wikidata-test-definitions-small

@@ -10,6 +10,7 @@ import (
 	"github.com/richardlehane/siegfried/pkg/mimeinfo"
 	"github.com/richardlehane/siegfried/pkg/pronom"
 	"github.com/richardlehane/siegfried/pkg/sets"
+	wd "github.com/richardlehane/siegfried/pkg/wikidata"
 )
 
 var testhome = flag.String("home", "data", "override the default home directory")
@@ -66,6 +67,20 @@ func TestFreedesktop(t *testing.T) {
 	}
 }
 
+func TestWikidata(t *testing.T) {
+	s := siegfried.New()
+	config.SetHome(*testhome)
+	config.SetWikidataDefinitions("wikidata-test-definitions-small")
+	m, err := wd.New(config.SetWikidataNamespace())
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Add(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestPronomTikaLoc(t *testing.T) {
 	s := siegfried.New()
 	config.SetHome(*testhome)

cmd/roy/harvest_wikidata.go

@@ -64,6 +64,7 @@ var (
 	name         = flag.String("name", "", "provide a filename when scanning a stream e.g. sf -name myfile.txt -")
 	conff        = flag.String("conf", "", "set the configuration file")
 	setconff     = flag.Bool("setconf", false, "record flags used with this command in configuration file")
+	sourceinline = flag.Bool("sourceinline", false, "display provenance in-line (basis field) when it is available for an identifier, e.g. Wikidata")
 )
 
 var (
@@ -377,6 +378,11 @@ func main() {
 		serveFpr(config.Fpr(), s)
 		return
 	}
+	// present source in the basis field within the Wikidata identifier
+	// instead of its own field.
+	if *sourceinline {
+		config.SetWikidataSourceFieldOff()
+	}
 	// check -multi
 	if *multi > maxMulti || *multi < 1 || (*archive && *multi > 1) {
 		log.Println("[WARN] -multi must be > 0 and =< 1024. If -z, -multi must be 1. Resetting -multi to 1")

cmd/roy/roy.go

@@ -126,6 +126,10 @@ func TestSuite(t *testing.T) {
 		return nil
 	}
 	suite := filepath.Join(*testdata, "skeleton-suite")
+	_, err = os.Stat(suite)
+	if err != nil {
+		t.Fatal(err)
+	}
 	err = filepath.Walk(suite, wf)
 	if err != nil {
 		t.Fatal(err)

cmd/roy/roy_test.go

@@ -0,0 +1 @@
+�