Generate resources based on template directories.

betty/assets/templates/base.html.j2

@@ -13,20 +13,20 @@
     {% endif %}
     <meta name="og:title" content="{%- if page_title is defined -%}{{ page_title | striptags }}{%- else -%}{{ site.configuration.title }}{%- endif -%}"/>
     <meta name="og:site_name" content="{{ site.configuration.title }}"/>
-    {% if page_resource is defined and page_recource is has_files %}
-        {% for file in page_resource.files %}
+    {% if file_resource is defined and file_resource is has_files %}
+        {% for file in file_resource.files %}
             {% if file.media_type.startswith('image/') %}
                 <meta name="og:image" content="{{ file | image(600, 600) | static_url }}"/>
                 {% if file.description %}
                     <meta name="og:image:description" content="{{ file.description | escape }}"/>
                 {% endif %}
             {% endif %}
         {% endfor %}
-        <link rel="canonical" href="{{ page_resource | url(absolute=true) }}"/>
-        {% if page_resource is defined and page_resource is resource and resource is identifiable %}
-            <link rel="alternate" href="{{ page_resource | url(media_type='application/json') }}" type="application/json"/>
+        <link rel="canonical" href="{{ file_resource | url(absolute=true) }}"/>
+        {% if file_resource is defined and file_resource is resource and resource is identifiable %}
+            <link rel="alternate" href="{{ file_resource | url(media_type='application/json') }}" type="application/json"/>
             <script type="application/ld+json">
-              {{ page_resource | tojson }}
+              {{ file_resource | tojson }}
             </script>
         {% endif %}
     {% endif %}
@@ -46,8 +46,8 @@
         {% endfor %}
     {% endfor %}
 </head>
-{% if page_resource is has_files %}
-    {% set page_image = page_resource.associated_files | selectattr('media_type', 'starts_with', 'image/') | first %}
+{% if file_resource is has_files %}
+    {% set page_image = file_resource.associated_files | selectattr('media_type', 'starts_with', 'image/') | first %}
 {% elif site.configuration.theme.background_image_id and site.configuration.theme.background_image_id in site.ancestry.files %}
     {% set page_image = site.ancestry.files[site.configuration.theme.background_image_id] %}
 {% endif %}
@@ -100,7 +100,7 @@
                 </ul>
             </div>
         </div>
-        {% if page_resource is defined and site.configuration.multilingual %}
+        {% if file_resource is defined and site.configuration.multilingual %}
             <div id="nav-locale" class="nav-primary-expandable">
                 <h2 class="nav-primary-action">{% trans %}Language{% endtrans %}</h2>
                 <div class="nav-primary-expanded">
@@ -111,7 +111,7 @@
                         {% do ns.available_locales.append((available_locale, available_locale_data.get_display_name())) %}
                     {% endfor %}
                     {% for available_locale, available_locale_name in ns.available_locales | sort(attribute='1') %}
-                        <li><a href="{{ page_resource | url(locale=available_locale) }}" hreflang="{{ available_locale }}" lang="{{ available_locale }}" rel="alternate">{{ available_locale_name }}</a></li>
+                        <li><a href="{{ file_resource | url(locale=available_locale) }}" hreflang="{{ available_locale }}" lang="{{ available_locale }}" rel="alternate">{{ available_locale_name }}</a></li>
                     {% endfor %}
                     </ul>
                 </div>
@@ -123,8 +123,8 @@
             <h1>{{ page_title }}</h1>
         {% endif %}
         {% block page_content %}{% endblock %}
-        {% if page_resource is has_links %}
-            {% set links = page_resource.links | rejectattr('label', 'none') | list %}
+        {% if file_resource is has_links %}
+            {% set links = file_resource.links | rejectattr('label', 'none') | list %}
             {% set links = links | select_localizeds | list + links | selectattr('locale', 'none') | list %}
             {% if links | length > 0 %}
                 <h2>{% trans %}External links{% endtrans %}</h2>

betty/assets/templates/page/list-event.html.j2 → betty/assets/templates/resource-collection/event/index.html.j2

@@ -1,7 +1,6 @@
 {% extends 'base.html.j2' %}
+{% set events = file_resources %}
 {% set page_title = _('Events') %}
 {% block page_content %}
-    {% with events=entities %}
-        {% include 'list-event.html.j2' %}
-    {% endwith %}
+    {% include 'list-event.html.j2' %}
 {% endblock %}

betty/assets/templates/resource-collection/event/index.json.j2

@@ -0,0 +1,4 @@
+{{ {
+    '$schema': 'schema.json#/definitions/eventCollection' | static_url(absolute=True),
+    'collection': file_resources | list,
+} | json | safe }}

betty/assets/templates/page/list-person.html.j2 → betty/assets/templates/resource-collection/person/index.html.j2

@@ -1,7 +1,8 @@
 {% extends 'base.html.j2' %}
+{% set people = file_resources %}
 {% set page_title = _('People') %}
 {% block page_content %}
-    {% with persons=entities | rejectattr('private', 'eq', True), group_by_affiliation=True %}
+    {% with persons=people | rejectattr('private', 'eq', True), group_by_affiliation=True %}
         {% include 'list-person.html.j2' %}
     {% endwith %}
 {% endblock %}

betty/assets/templates/resource-collection/person/index.json.j2

@@ -0,0 +1,4 @@
+{{ {
+    '$schema': 'schema.json#/definitions/personCollection' | static_url(absolute=True),
+    'collection': file_resources | list,
+} | json | safe }}

betty/assets/templates/page/list-place.html.j2 → betty/assets/templates/resource-collection/place/index.html.j2

@@ -1,7 +1,8 @@
 {% extends 'base.html.j2' %}
+{% set places = file_resources %}
 {% set page_title = _('Places') %}
 {% block page_content %}
-    {% with places=entities, canonical=True %}
+    {% with canonical=True %}
         {% include 'list-place.html.j2' %}
     {% endwith %}
 {% endblock %}

betty/assets/templates/resource-collection/place/index.json.j2

@@ -0,0 +1,4 @@
+{{ {
+    '$schema': 'schema.json#/definitions/placeCollection' | static_url(absolute=True),
+    'collection': file_resources | list,
+} | json | safe }}

betty/assets/templates/page/list-source.html.j2 → betty/assets/templates/resource-collection/source/index.html.j2

@@ -1,8 +1,9 @@
 {% extends 'base.html.j2' %}
+{% set sources = file_resources %}
 {% set page_title = _('Sources') %}
 {% block page_content %}
     <ul class="entities">
-        {% for source in entities | sort(attribute='name') %}
+        {% for source in sources | sort(attribute='name') %}
             <li class="{{ loop.cycle('odd', 'even') }}">
                 {% include 'label/source.html.j2' %}
             </li>

betty/assets/templates/resource-collection/source/index.json.j2

@@ -0,0 +1,4 @@
+{{ {
+    '$schema': 'schema.json#/definitions/sourceCollection' | static_url(absolute=True),
+    'collection': file_resources | list,
+} | json | safe }}

betty/assets/templates/page/citation.html.j2 → betty/assets/templates/resource/citation/index.html.j2

@@ -1,4 +1,5 @@
 {% extends 'base.html.j2' %}
+{% set citation = file_resource %}
 {% set page_title = citation.location %}
 {% block page_content %}
     <div class="meta">

betty/assets/templates/resource/citation/index.json.j2

@@ -0,0 +1 @@
+{{ file_resource | json | safe }}

betty/assets/templates/page/event.html.j2 → betty/assets/templates/resource/event/index.html.j2

@@ -1,4 +1,5 @@
 {% extends 'base.html.j2' %}
+{% set event = file_resource %}
 {% set page_title %}
     {% with embedded=True %}
         {% include 'label/event.html.j2' %}

betty/assets/templates/resource/event/index.json.j2

@@ -0,0 +1 @@
+{{ file_resource | json | safe }}

betty/assets/templates/page/file.html.j2 → betty/assets/templates/resource/file/index.html.j2

@@ -1,4 +1,5 @@
 {% extends 'base.html.j2' %}
+{% set file = file_resource %}
 {% set page_title = file.description %}
 {% block page_content %}
     {% if file.media_type and file.media_type.startswith('image/') %}

betty/assets/templates/resource/file/index.json.j2

@@ -0,0 +1 @@
+{{ file_resource | json | safe }}

betty/assets/templates/page/person.html.j2 → betty/assets/templates/resource/person/index.html.j2

@@ -1,4 +1,5 @@
 {% extends 'base.html.j2' %}
+{% set person = file_resource %}
 {% set page_title %}
     {% with embedded=True %}
         {% include 'label/person.html.j2' %}

betty/assets/templates/resource/person/index.json.j2

@@ -0,0 +1 @@
+{{ file_resource | json | safe }}

betty/assets/templates/page/place.html.j2 → betty/assets/templates/resource/place/index.html.j2

@@ -1,4 +1,5 @@
 {% extends 'base.html.j2' %}
+{% set place = file_resource %}
 {% set page_title = place.names | negotiate_localizeds %}
 {% block page_content %}
     {% include 'meta/place.html.j2' %}

betty/assets/templates/resource/place/index.json.j2

@@ -0,0 +1 @@
+{{ file_resource | json | safe }}

betty/assets/templates/page/source.html.j2 → betty/assets/templates/resource/source/index.html.j2

@@ -1,4 +1,5 @@
 {% extends 'base.html.j2' %}
+{% set source = file_resource %}
 {% set page_title = source.name %}
 {% block page_content %}
     {% include 'meta/source.html.j2' %}

betty/assets/templates/resource/source/index.json.j2

@@ -0,0 +1 @@
+{{ file_resource | json | safe }}

betty/cli.py

@@ -1,4 +1,6 @@
+import cProfile
 import logging
+import pstats
 import shutil
 import sys
 from contextlib import suppress, contextmanager
@@ -132,5 +134,7 @@ async def _clear_caches():
 @click.command(help='Generate a static site.')
 @site_command
 async def _generate(site: Site):
-    await parse.parse(site)
-    await generate.generate(site)
+    with cProfile.Profile() as pr:
+        await parse.parse(site)
+        await generate.generate(site)
+    pstats.Stats(pr).strip_dirs().sort_stats(pstats.SortKey.CUMULATIVE).print_stats()

betty/fs.py

@@ -4,58 +4,78 @@
 from collections import deque
 from contextlib import suppress
 from os import walk, path
-from os.path import join, dirname, exists, relpath, getmtime
-from shutil import copy2
 from tempfile import mkdtemp
-from typing import AsyncIterable
+from typing import Iterable, Dict
 
 
-async def iterfiles(path: str) -> AsyncIterable[str]:
-    for dir_path, _, filenames in walk(path):
+def iterfiles(file_path: str) -> Iterable[str]:
+    for dir_path, _, filenames in walk(file_path):
         for filename in filenames:
-            yield join(dir_path, filename)
+            yield path.join(dir_path, filename)
 
 
-def makedirs(path: str) -> None:
-    os.makedirs(path, 0o755, True)
+def makedirs(file_path: str) -> None:
+    os.makedirs(file_path, 0o755, True)
 
 
-def hashfile(path: str) -> str:
-    return hashlib.md5(':'.join([str(getmtime(path)), path]).encode('utf-8')).hexdigest()
+def hashfile(file_path: str) -> str:
+    return hashlib.md5(':'.join([str(path.getmtime(file_path)), file_path]).encode('utf-8')).hexdigest()
+
+
+async def copy_directory(source_path: str, destination_path: str):
+    for file_source_path in iterfiles(source_path):
+        file_destination_path = path.join(destination_path, path.relpath(file_source_path, source_path))
+        try:
+            shutil.copy2(file_source_path, file_destination_path)
+        except FileNotFoundError:
+            makedirs(path.dirname(file_destination_path))
+            shutil.copy2(file_source_path, file_destination_path)
 
 
 class FileSystem:
-    def __init__(self, *paths):
+    def __init__(self, *paths: str):
+        """
+        :param paths: The paths to each individual layer of the file system, from high to low priority.
+        """
         self._paths = deque(paths)
 
     @property
     def paths(self) -> deque:
         return self._paths
 
-    async def open(self, *file_paths: str):
-        for file_path in file_paths:
-            for fs_path in self._paths:
-                with suppress(FileNotFoundError):
-                    return open(join(fs_path, file_path))
-        raise FileNotFoundError
+    async def copy_file(self, source_path: str, destination_path: str) -> None:
+        makedirs(path.dirname(destination_path))
 
-    async def copy2(self, source_path: str, destination_path: str) -> str:
         for fs_path in self._paths:
             with suppress(FileNotFoundError):
-                return copy2(join(fs_path, source_path), destination_path)
-        tried_paths = [join(fs_path, source_path) for fs_path in self._paths]
-        raise FileNotFoundError('Could not find any of %s.' %
-                                ', '.join(tried_paths))
-
-    async def copytree(self, source_path: str, destination_path: str) -> str:
-        for fs_path in self._paths:
-            async for file_source_path in iterfiles(join(fs_path, source_path)):
-                file_destination_path = join(destination_path, relpath(
-                    file_source_path, join(fs_path, source_path)))
-                if not exists(file_destination_path):
-                    makedirs(dirname(file_destination_path))
-                    copy2(file_source_path, file_destination_path)
-        return destination_path
+                return shutil.copyfile(path.join(fs_path, source_path), destination_path)
+
+        tried_paths = [path. join(fs_path, source_path) for fs_path in self._paths]
+        raise FileNotFoundError('Could not find any of %s.' % ', '.join(tried_paths))
+
+    async def copy_directory(self, source_path: str, destination_path: str) -> None:
+        makedirs(destination_path)
+
+        tries = []
+        for fs_path in reversed(self._paths):
+            try:
+                await copy_directory(path.join(fs_path, source_path), destination_path)
+                tries.append(True)
+            except FileNotFoundError:
+                tries.append(False)
+
+        if True not in tries:
+            tried_paths = [path. join(fs_path, source_path) for fs_path in self._paths]
+            raise FileNotFoundError('Could not find any of %s.' % ', '.join(tried_paths))
+
+    def iterfiles(self, directory_path: str) -> Dict[str, str]:
+        file_paths = {}
+        for fs_path in reversed(self._paths):
+            for dir_path, _, filenames in walk(directory_path):
+                for filename in filenames:
+                    file_path = path.join(dir_path, filename)
+                    file_paths[path.relpath(file_path, fs_path)] = file_path
+        return file_paths
 
 
 class DirectoryBackup:

betty/functools.py

@@ -22,18 +22,13 @@ def walk(item, attribute_name):
         yield from walk(child, attribute_name)
 
 
-async def asynciter(items: Union[Iterable, AsyncIterable]) -> AsyncIterable:
-    if hasattr(items, '__aiter__'):
-        async for item in items:
-            yield item
-        return
-    for item in items:
-        yield item
-
-
 def sync(f):
     if iscoroutine(f):
-        return asyncio.get_event_loop().run_until_complete(f)
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = asyncio.get_event_loop()
+        return loop.run_until_complete(f)
     elif iscoroutinefunction(f):
         @wraps(f)
         def _synced(*args, **kwargs):