-
${item.crawlCount}
${pluralOf("items", item.crawlCount)}
@@ -69,26 +82,40 @@ export class LinkedCollectionsListItem extends TailwindElement {
}
if (this.removable) {
+ const button = html`
+ this.dispatchEvent(
+ new CustomEvent(
+ "btrix-remove",
+ {
+ detail: {
+ item: item,
+ },
+ bubbles: true,
+ composed: true,
+ },
+ ),
+ )}
+ >`;
+
content.push(
html`
-
-
- this.dispatchEvent(
- new CustomEvent(
- "btrix-remove",
- {
- detail: {
- item: item,
- },
- bubbles: true,
- composed: true,
- },
- ),
+ ${dedupeEnabled
+ ? html`
-
+ >
+ ${button}
+ `
+ : html`
+ ${button}
+ `}
+
+
`,
);
}
diff --git a/frontend/src/features/collections/linked-collections/linked-collections-list.ts b/frontend/src/features/collections/linked-collections/linked-collections-list.ts
index 7b9508f2ed..9e9837ea65 100644
--- a/frontend/src/features/collections/linked-collections/linked-collections-list.ts
+++ b/frontend/src/features/collections/linked-collections/linked-collections-list.ts
@@ -21,6 +21,9 @@ export class LinkedCollectionsList extends TailwindElement {
@property({ type: String })
baseUrl?: string;
+ @property({ type: String })
+ dedupeId?: string;
+
@property({ type: Boolean })
removable?: boolean;
@@ -37,6 +40,7 @@ export class LinkedCollectionsList extends TailwindElement {
class=${clsx(tw`contents`, i > 0 && tw`part-[base]:border-t`)}
.item=${until(request, item)}
baseUrl=${ifDefined(this.baseUrl)}
+ ?dedupeSource=${Boolean(this.dedupeId && item.id === this.dedupeId)}
?removable=${this.removable}
?loading=${until(
request.then(() => false),
diff --git a/frontend/src/features/collections/linked-collections/linked-collections.ts b/frontend/src/features/collections/linked-collections/linked-collections.ts
index ad73a6b3e4..6b9fc7568a 100644
--- a/frontend/src/features/collections/linked-collections/linked-collections.ts
+++ b/frontend/src/features/collections/linked-collections/linked-collections.ts
@@ -28,6 +28,12 @@ export class LinkedCollections extends BtrixElement {
@property({ type: Array, hasChanged: isNotEqual })
collections: (string | CollectionLikeItem)[] = [];
+ /**
+ * ID of collection that is used for deduplication
+ */
+ @property({ type: String })
+ dedupeId?: string;
+
@property({ type: Boolean })
removable?: boolean;
@@ -112,6 +118,7 @@ export class LinkedCollections extends BtrixElement {
aria-live="polite"
.collections=${collections}
baseUrl="${this.navigate.orgBasePath}/collections/view"
+ .dedupeId=${this.dedupeId}
?removable=${this.removable}
>`;
}
diff --git a/frontend/src/features/crawl-workflows/workflow-editor.ts b/frontend/src/features/crawl-workflows/workflow-editor.ts
index cc3e629c63..2970f18dc6 100644
--- a/frontend/src/features/crawl-workflows/workflow-editor.ts
+++ b/frontend/src/features/crawl-workflows/workflow-editor.ts
@@ -40,9 +40,13 @@ import compact from "lodash/fp/compact";
import flow from "lodash/fp/flow";
import isEqual from "lodash/fp/isEqual";
import throttle from "lodash/fp/throttle";
+import union from "lodash/fp/union";
import uniq from "lodash/fp/uniq";
+import without from "lodash/fp/without";
import queryString from "query-string";
+import type { CollectionNameInputChangeEvent } from "../collections/collection-name-input";
+
import {
SELECTOR_DELIMITER,
type LinkSelectorTable,
@@ -205,6 +209,10 @@ const getDefaultProgressState = (hasConfigId = false): ProgressState => {
error: false,
completed: hasConfigId,
},
+ deduplication: {
+ error: false,
+ completed: hasConfigId,
+ },
collections: {
error: false,
completed: hasConfigId,
@@ -387,6 +395,11 @@ export class WorkflowEditor extends BtrixElement {
"": "",
};
+ private readonly dedupeTypeLabels: Record
= {
+ collection: msg("Deduplicate using a collection"),
+ none: msg("No deduplication"),
+ };
+
@query(`form[name="${formName}"]`)
private readonly formElem?: HTMLFormElement;
@@ -652,20 +665,15 @@ export class WorkflowEditor extends BtrixElement {
const el = e.currentTarget as SlDetails;
// Check if there's any invalid elements before hiding
- let invalidEl: SlInput | null = null;
-
- if (required) {
- invalidEl = el.querySelector("[required][data-invalid]");
- }
-
- invalidEl =
- invalidEl || el.querySelector("[data-user-invalid]");
+ const invalidEl =
+ el.querySelector("[data-invalid]") ||
+ el.querySelector(":invalid");
if (invalidEl) {
e.preventDefault();
invalidEl.focus();
- invalidEl.checkValidity();
+ invalidEl.reportValidity();
}
})}
@sl-after-show=${this.handleCurrentTarget(
@@ -1666,7 +1674,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
- ${labelFor.selectLink}
+ ${labelFor.selectLinks}
${isCustom
? html`${selectors.length}`
: ""}
@@ -1784,7 +1792,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
`;
return html`
- ${this.renderSectionHeading(labelFor.behaviors)}
+ ${this.renderSectionHeading(sectionStrings.behaviors)}
${inputCol(
html` {
+ this.updateProgressState({ activeTab: "collections" });
+ await this.updateComplete;
+ void this.scrollToActivePanel();
+ }}
+ >
+ ${msg("Auto-Add to Collections")}
+ `;
+
+ return html` ${inputCol(html`
+ {
+ const dedupeType = (e.target as SlRadio)
+ .value as FormState["dedupeType"];
+
+ const formState: Partial = {
+ dedupeType,
+ dedupeCollectionId: null,
+ dedupeCollectionName: null,
+ };
+
+ if (dedupeType === "none" && this.formState.dedupeCollectionId) {
+ formState.autoAddCollections = without(
+ [this.formState.dedupeCollectionId],
+ this.formState.autoAddCollections,
+ );
+ }
+
+ this.updateFormState(formState, true);
+ }}
+ >
+ ${this.dedupeTypeLabels["none"]}
+ ${this.dedupeTypeLabels["collection"]}
+
+ ${when(
+ this.formState.dedupeType === "none" &&
+ this.initialWorkflow?.dedupCollId,
+ () => html`
+
+
+
+ ${msg(
+ "Disabling deduplication will also disable auto-adding to the collection.",
+ )}
+
+ ${msg(
+ html`To continue to auto-add to the collection without
+ deduplication enabled, update the
+ ${link_to_collections_settings} setting.`,
+ )}
+
+ `,
+ )}
+
+ `)}
+ ${this.renderHelpTextCol(
+ msg(
+ `Enable duplication checks before and during a crawl to avoid duplicate content in archived items.`,
+ ),
+ )}
+ ${when(
+ this.formState.dedupeType === "collection",
+ this.renderDedupeCollection,
+ )}`;
+ }
+
+ private readonly renderDedupeCollection = () => {
+ return html`
+ ${this.renderSectionHeading(msg("Collection to Use"))}
+ ${inputCol(html`
+ {
+ const { id, name } = e.detail.value;
+
+ if (id) {
+ this.updateFormState(
+ {
+ dedupeCollectionId: id,
+ dedupeCollectionName: name || "",
+ autoAddCollections: union(this.formState.autoAddCollections, [
+ id,
+ ]),
+ },
+ true,
+ );
+ } else if (name) {
+ this.updateFormState({
+ dedupeCollectionId: null,
+ dedupeCollectionName: name,
+ });
+ }
+ }}
+ @btrix-clear=${() => {
+ if (this.formState.dedupeCollectionId) {
+ this.updateFormState(
+ {
+ dedupeCollectionId: null,
+ dedupeCollectionName: null,
+ autoAddCollections: without(
+ [this.formState.dedupeCollectionId],
+ this.formState.autoAddCollections,
+ ),
+ },
+ true,
+ );
+ } else {
+ this.updateFormState({
+ dedupeCollectionName: null,
+ });
+ }
+ }}
+ >
+
+ ${when(
+ this.formState.dedupeCollectionName &&
+ !this.formState.dedupeCollectionId,
+ () => {
+ const workflow_name = html`${this.formState.dedupeCollectionName}`;
+ return html`
+
+ ${msg(
+ html`A new collection named “${workflow_name}” will be created
+ when this workflow is saved.`,
+ )}
+
+ `;
+ },
+ )}
+ `)}
+ ${this.renderHelpTextCol(
+ msg(
+ "This collection will be used as the deduplication source for all crawls of this workflow.",
+ ),
+ )}
+ `;
+ };
+
private renderCollections() {
+ const newDedupeCollectionName =
+ this.formState.dedupeType === "collection" &&
+ !this.formState.dedupeCollectionId &&
+ this.formState.dedupeCollectionName;
+ const showDedupeWarning =
+ !isEqual(
+ this.initialWorkflow?.autoAddCollections,
+ this.formState.autoAddCollections,
+ ) &&
+ (this.formState.dedupeCollectionId || newDedupeCollectionName);
+
return html`
${inputCol(html`
this.updateFormState(
{
@@ -2298,14 +2475,37 @@ https://archiveweb.page/images/${"logo.svg"}`}
},
true,
)}
- >
+ >
+ ${when(
+ showDedupeWarning,
+ () => html`
+
+ ${msg(
+ "Adding deduplicated crawls to a collection other than the deduplication source may result in incomplete replay of the non-deduplicated collection.",
+ )}
+
+ `,
+ )}
+
${when(
- !this.formState.autoAddCollections.length,
- () => html`
-
-
${msg("No collections selected.")}
-
+
${msg("No collections selected.")}
+