Implement Library of Congress Subjects/Languages

Create LCBase/LCCase for shared implementation and specs across Library of Congress authorities

Implement LCSH

Implement LC Languages

Use empty search for MARC Languages to hit the 30/50 result threshold

Refactor module layout/naming

Add coveralls.json

Fix the typespec for search callback

Turn LOC test case into Authoritex test case

Discover authorities through config

Implement Authoritex.LOC which will search all LOC linked data

Only test 500 and "bad 200" responses on LOC, not every subauthority

.circleci/config.yml

@@ -6,15 +6,24 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
-      - run: mix do local.hex --force, local.rebar --force
       - restore_cache:
           keys:
             - hex-cache-{{ .Environment.CACHE_PREFIX }}-{{ checksum "mix.lock" }}
             - hex-cache-{{ .Environment.CACHE_PREFIX }}-
-      - run: mix deps.get
-      - run: mix coveralls.circle
+      - run:
+          name: Install Hex & Rebar
+          command: mix do local.hex --force, local.rebar --force
+      - run:
+          name: Install Dependencies
+          command: mix do deps.get, deps.compile
       - save_cache:
           key: hex-cache-{{ .Environment.CACHE_PREFIX }}-{{ checksum "mix.lock" }}
           paths:
             - ~/meadow/deps
             - ~/meadow/_build
+      - run:
+          name: Static Analysis
+          command: mix credo || true
+      - run:
+          name: Run Tests & Coverage Analysis
+          command: mix coveralls.circle --trace

config/config.exs

@@ -1,3 +1,11 @@
 use Mix.Config
 
+config :authoritex,
+  authorities: [
+    Authoritex.LOC.Languages,
+    Authoritex.LOC.Names,
+    Authoritex.LOC.SubjectHeadings,
+    Authoritex.LOC
+  ]
+
 import_config "#{Mix.env()}.exs"

coveralls.json

@@ -0,0 +1,4 @@
+{
+  "skip_files": ["lib/authoritex/loc/base.ex"],
+  "treat_no_relevant_lines_as_covered": true
+}

lib/authoritex.ex

@@ -4,9 +4,7 @@ defmodule Authoritex do
   @callback code() :: String.t()
   @callback description :: String.t()
   @callback fetch(String.t()) :: {:ok, String.t() | nil} | {:error, term()}
-  @callback search(String.t()) :: {:ok, list(:result)} | {:error, term()}
-
-  @authorities [Authoritex.LCNAF]
+  @callback search(String.t(), integer()) :: {:ok, list(:result)} | {:error, term()}
 
   @doc """
   Returns a label given an id.
@@ -17,7 +15,7 @@ defmodule Authoritex do
   """
   def fetch(id) do
     case authority_for(id) do
-      nil -> nil
+      nil -> {:error, :unknown_authority}
       {authority, _, _} -> authority.fetch(id)
     end
   end
@@ -49,7 +47,7 @@ defmodule Authoritex do
   end
 
   def authorities do
-    @authorities
+    Application.get_env(:authoritex, :authorities, [])
     |> Enum.map(fn mod -> {mod, mod.code(), mod.description()} end)
   end
 

lib/authoritex/loc.ex

@@ -0,0 +1,9 @@
+defmodule Authoritex.LOC do
+  @desc "Library of Congress Linked Data"
+  @moduledoc "Authoritex implementation for #{@desc}"
+
+  use Authoritex.LOC.Base,
+    subauthority: nil,
+    code: "loc",
+    description: @desc
+end

lib/authoritex/loc/base.ex

@@ -0,0 +1,117 @@
+defmodule Authoritex.LOC.Base do
+  @moduledoc "Abstract Authoritex implementation for Library of Congress authorities & vocabularies"
+
+  # Unresolved issues:
+  # * Language of labels
+  # * Stemming/wildcards during search (e.g., "English" vs. "englis" vs. "eng")
+
+  defmacro __using__(use_opts) do
+    {suffix, query_filter} =
+      if is_nil(use_opts[:subauthority]) do
+        {"", []}
+      else
+        {
+          "/#{use_opts[:subauthority]}",
+          [q: "scheme:http://id.loc.gov/#{use_opts[:subauthority]}"]
+        }
+      end
+
+    quote bind_quoted: [
+            lc_code: use_opts[:code],
+            lc_desc: use_opts[:description],
+            subauthority: use_opts[:subauthority],
+            http_uri: "http://id.loc.gov#{suffix}",
+            info_uri: "info:lc#{suffix}",
+            query_filter: query_filter
+          ] do
+      @behaviour Authoritex
+
+      import SweetXml, only: [sigil_x: 2]
+
+      @impl Authoritex
+      def can_resolve?(unquote(http_uri) <> "/" <> _), do: true
+      def can_resolve?(unquote(info_uri) <> "/" <> _), do: true
+      def can_resolve?(_), do: false
+
+      @impl Authoritex
+      def code, do: unquote(lc_code)
+
+      @impl Authoritex
+      def description, do: unquote(lc_desc)
+
+      @impl Authoritex
+      def fetch(unquote(info_uri) <> "/" <> rest),
+        do: fetch(unquote(http_uri) <> "/" <> rest)
+
+      def fetch(id) do
+        case HTTPoison.get(id <> ".rdf") do
+          {:ok, response} ->
+            parse_fetch_result(response)
+
+          {:error, error} ->
+            {:error, error}
+        end
+      end
+
+      @impl Authoritex
+      def search(query, max_results \\ 30) do
+        query_params = [{:q, query} | unquote(query_filter)]
+
+        case HTTPoison.get(
+               "http://id.loc.gov/search/",
+               [{"User-Agent", "Authoritex"}],
+               params: query_params ++ [count: max_results, format: "xml+atom"]
+             ) do
+          {:ok, response} ->
+            parse_search_result(response)
+
+          {:error, error} ->
+            {:error, error}
+        end
+      end
+
+      defp parse_fetch_result(%{body: response, status_code: 200}) do
+        with doc <- SweetXml.parse(response) do
+          case doc |> SweetXml.xpath(~x"/rdf:RDF") do
+            nil ->
+              {:error, {:bad_response, response}}
+
+            rdf ->
+              {:ok,
+               SweetXml.xpath(rdf, ~x"//madsrdf:authoritativeLabel", label: ~x"./text()"s)
+               |> Map.get(:label)}
+          end
+        end
+      rescue
+        _ -> {:error, {:bad_response, response}}
+      end
+
+      defp parse_fetch_result(%{status_code: code} = response) when code in 300..399 do
+        response.headers
+        |> Enum.into(%{})
+        |> Map.get("Location")
+        |> String.replace(~r"\.rdf$", "")
+        |> fetch()
+      end
+
+      defp parse_fetch_result(%{status_code: status_code}), do: {:error, status_code}
+
+      defp parse_search_result(%{body: response, status_code: 200}) do
+        with doc <- SweetXml.parse(response) do
+          case doc |> SweetXml.xpath(~x"/feed") do
+            nil ->
+              {:error, {:bad_response, response}}
+
+            feed ->
+              {:ok,
+               SweetXml.xpath(feed, ~x"//entry"l, id: ~x"./id/text()"s, label: ~x"./title/text()"s)}
+          end
+        end
+      rescue
+        _ -> {:error, {:bad_response, response}}
+      end
+
+      defp parse_search_result(%{status_code: status_code}), do: {:error, status_code}
+    end
+  end
+end

lib/authoritex/loc/languages.ex

@@ -0,0 +1,9 @@
+defmodule Authoritex.LOC.Languages do
+  @desc "Library of Congress MARC List for Languages"
+  @moduledoc "Authoritex implementation for #{@desc}"
+
+  use Authoritex.LOC.Base,
+    subauthority: "vocabulary/languages",
+    code: "lclang",
+    description: @desc
+end

lib/authoritex/loc/names.ex

@@ -0,0 +1,9 @@
+defmodule Authoritex.LOC.Names do
+  @desc "Library of Congress Name Authority File"
+  @moduledoc "Authoritex implementation for #{@desc}"
+
+  use Authoritex.LOC.Base,
+    subauthority: "authorities/names",
+    code: "lcnaf",
+    description: @desc
+end

lib/authoritex/loc/subject_headings.ex

@@ -0,0 +1,9 @@
+defmodule Authoritex.LOC.SubjectHeadings do
+  @desc "Library of Congress Subject Headings"
+  @moduledoc "Authoritex implementation for #{@desc}"
+
+  use Authoritex.LOC.Base,
+    subauthority: "authorities/subjects",
+    code: "lcsh",
+    description: @desc
+end