From cdd0416a5f86d6f1ccd376ef136f399367420c49 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Tue, 17 Jun 2025 16:44:30 +0100 Subject: [PATCH 01/66] Sort by Taken Date UI Changes --- .../gr-notifications-banner.tsx | 7 + .../gr-panel-button/gr-panel-button.js | 12 +- .../gr-search-wrapper/gr-search-wrapper.html | 2 - .../gr-sort-control/gr-sort-control-config.ts | 15 +- .../gr-sort-control/gr-sort-control.css | 7 +- .../gr-sort-control/gr-sort-control.tsx | 97 +++++++---- kahuna/public/js/search/index.js | 2 + kahuna/public/js/search/query.js | 2 + kahuna/public/js/search/results.html | 32 ++-- kahuna/public/js/search/results.js | 155 +++++++++++++++++- kahuna/public/stylesheets/main.css | 2 +- 11 files changed, 271 insertions(+), 62 deletions(-) diff --git a/kahuna/public/js/components/gr-notifications-banner/gr-notifications-banner.tsx b/kahuna/public/js/components/gr-notifications-banner/gr-notifications-banner.tsx index d8b81d406c..a19bd943f5 100644 --- a/kahuna/public/js/components/gr-notifications-banner/gr-notifications-banner.tsx +++ b/kahuna/public/js/components/gr-notifications-banner/gr-notifications-banner.tsx @@ -156,6 +156,11 @@ const NotificationsBanner: React.FC = () => { setNotifications(prev_notifs => mergeArraysByKey(prev_notifs, [notification], 'announceId')); }; + const removeNotification = (event:any) => { + const notificationId = event.detail.announceId; + setNotifications(prev_notifs => prev_notifs.filter(notif => notif.announceId !== notificationId)); + }; + useEffect(() => { const announce = window._clientConfig.announcements; const tdy = todayStr(); @@ -175,6 +180,7 @@ const NotificationsBanner: React.FC = () => { document.addEventListener("scroll", autoHideListener); document.addEventListener("keydown", autoHideListener); window.addEventListener("newNotification", newNotification); + window.addEventListener("removeNotification", removeNotification); // clean up cookie if (notif_cookie) { @@ -190,6 +196,7 @@ const NotificationsBanner: React.FC = () => { document.removeEventListener("scroll", autoHideListener); document.removeEventListener("keydown", autoHideListener); window.removeEventListener("newNotification", newNotification); + window.removeEventListener("removeNotification", removeNotification); clearInterval(checkNotificationsRef); }; diff --git a/kahuna/public/js/components/gr-panel-button/gr-panel-button.js b/kahuna/public/js/components/gr-panel-button/gr-panel-button.js index ed7592056d..e6e1b04505 100644 --- a/kahuna/public/js/components/gr-panel-button/gr-panel-button.js +++ b/kahuna/public/js/components/gr-panel-button/gr-panel-button.js @@ -16,12 +16,22 @@ panelButton.controller('GrPanelButton', ['$scope', 'inject$', function($scope, i 'Panel name': ctrl.name, 'Action': action }); - ctrl.showPanel = () => panel.setHidden(false); + ctrl.showPanel = () => { + panel.setHidden(false); + window.dispatchEvent(new CustomEvent("panelShow", { + detail: {panel: ctrl.name}, + bubbles: true + })); + } ctrl.lockPanel = () => panel.setLocked(true); ctrl.unlockPanel = () => panel.setLocked(false); ctrl.hidePanel = () => { panel.setLocked(false); panel.setHidden(true); + window.dispatchEvent(new CustomEvent("panelHide", { + detail: {panel: ctrl.name}, + bubbles: true + })); }; ctrl.toolTipPosition = () => { diff --git a/kahuna/public/js/components/gr-search-wrapper/gr-search-wrapper.html b/kahuna/public/js/components/gr-search-wrapper/gr-search-wrapper.html index 4e8e3238f3..d6eb238b6d 100644 --- a/kahuna/public/js/components/gr-search-wrapper/gr-search-wrapper.html +++ b/kahuna/public/js/components/gr-search-wrapper/gr-search-wrapper.html @@ -26,6 +26,4 @@ - - diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts index d6853d42dc..b825b50695 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts @@ -29,27 +29,32 @@ export const SortOptions: SortDropdownOption[] = [ { value: "uploadNewOld", label: "Upload date (new to old)", - isCollection: false + isCollection: false, + isTaken: false }, { value: "oldest", label: "Upload date (old to new)", - isCollection: false + isCollection: false, + isTaken: false }, { value: "-taken", label: "Taken date (new to old)", - isCollection: false + isCollection: false, + isTaken: true }, { value: "taken", label: "Taken date (old to new)", - isCollection: false + isCollection: false, + isTaken: true }, { value: "dateAddedToCollection", label: "Added to collection (new to old)", - isCollection: true + isCollection: true, + isTaken: false } ]; diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css index 772c620b52..47ce1ffd45 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css @@ -16,6 +16,10 @@ padding-bottom: 10px; } +.sort-panel-margin { + margin-right: 285px; +} + .sort-dropdown-toggle-advanced { display: block; padding-top: 7px; @@ -87,11 +91,10 @@ .sort-dropdown-menu { position: absolute; width: 250px; - right: 0px; + right: 10px; border: 1px solid #ccc; border-collapse: collapse; background-color: #333; - margin-right: 8px; } .sort-dropdown-item { diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index 0438c6fac4..bb95716532 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -11,6 +11,8 @@ const DEFAULT_OPTION = DefaultSortOption.value; const COLLECTION_OPTION = CollectionSortOption.value; const CONTROL_TITLE = "Sort by:"; const SORT_ORDER = "Sort order"; +const PANEL_IDENTIFIER = "info"; +const HAS_DATE_TAKEN_QUERY = "has:dateTaken"; const downArrowIcon = () => @@ -38,6 +40,7 @@ export interface SortDropdownOption { value: string; label: string; isCollection: boolean; + isTaken: boolean; } export interface SortDropdownProps { @@ -46,6 +49,8 @@ export interface SortDropdownProps { onSelect: (option: SortDropdownOption) => void; query?: string | ""; orderBy?: string | ""; + clearTakenVisible: () => void; + panelVisible?: boolean | false; } export interface SortWrapperProps { @@ -78,16 +83,7 @@ const SortControl: React.FC = ({ props }) => { const [selectedOption, setSelection] = useState(defSort); const [previousOption, setPrevious] = useState(defSort); const [currentIndex, setCurrentIndex] = useState(-1); - - const autoHideListener = (event: any) => { - if (event.type === "keydown" && event.key === "Escape") { - setIsOpen(false); - } else if (event.type !== "keydown") { - if (!hasClassInSelfOrParent(event.target, "sort-control")) { - setIsOpen(false); - } - } - }; + const [panelVisible, setPanelVisible] = useState(props.panelVisible); const handleArrowKeys = (event:KeyboardEvent) => { if (event.key === 'ArrowDown' || @@ -113,15 +109,6 @@ const SortControl: React.FC = ({ props }) => { } }; - const handleQueryChange = (e: any) => { - const newQuery = e.detail.query ? (" " + e.detail.query) : ""; - setHasCollection(checkForCollection(newQuery)); - }; - - const handleLogoClick = (e: any) => { - setSelection(defSort); - }; - useEffect(() => { if (hasCollection) { const collOpt = options.filter(opt => opt.value == COLLECTION_OPTION)[0]; @@ -140,8 +127,47 @@ const SortControl: React.FC = ({ props }) => { }, [selectedOption]); useEffect(() => { + const autoHideListener = (event: any) => { + if (event.type === "keydown" && event.key === "Escape") { + setIsOpen(false); + } else if (event.type !== "keydown") { + if (!hasClassInSelfOrParent(event.target, "sort-control")) { + setIsOpen(false); + } + } + }; + + const handleLogoClick = (e: any) => { + setSelection(defSort); + }; + + const handleQueryChange = (e: any) => { + const newQuery = e.detail.query ? (" " + e.detail.query) : ""; + setHasCollection(checkForCollection(newQuery)); + }; + + const handlePanelShow = (event: any) => { + const panel = event.detail.panel; + if (panel === PANEL_IDENTIFIER) { + setPanelVisible(true); + } + }; + + const handlePanelHide = (event: any) => { + const panel = event.detail.panel; + if (panel === PANEL_IDENTIFIER) { + setPanelVisible(false); + } + }; + if (props.options.filter(o => o.value === props.orderBy).length > 0) { - setSelection(props.options.filter(o => o.value === props.orderBy)[0]); + const selOpt = props.options.filter(o => o.value === props.orderBy)[0]; + const tempQuery = props.query ? props.query : ""; + if (!tempQuery.includes(HAS_DATE_TAKEN_QUERY) && selOpt.isTaken) { + setSelection(defSort); + } else { + setSelection(selOpt); + } } else { setSelection(defSort); } @@ -159,6 +185,8 @@ const SortControl: React.FC = ({ props }) => { window.addEventListener("mouseup", autoHideListener); window.addEventListener("scroll", autoHideListener); window.addEventListener("keydown", autoHideListener); + window.addEventListener("panelHide", handlePanelHide); + window.addEventListener("panelShow", handlePanelShow); // Clean up the event listener when the component unmounts return () => { @@ -168,6 +196,8 @@ const SortControl: React.FC = ({ props }) => { window.removeEventListener("mouseup", autoHideListener); window.removeEventListener("scroll", autoHideListener); window.removeEventListener("keydown", autoHideListener); + window.removeEventListener("panelHide", handlePanelHide); + window.removeEventListener("panelShow", handlePanelShow); }; }, []); @@ -176,24 +206,23 @@ const SortControl: React.FC = ({ props }) => { if (option.value !== selectedOption.value) { setSelection(option); props.onSelect(option); - //-notification banner- - if (option.value.includes("taken")) { - const notificationEvent = new CustomEvent("newNotification", { - detail: { - announceId: "sortByTakenDate", - description: "Images without a Taken Date will appear at the end of the list", - category: "information", - lifespan: "transient" - }, - bubbles: true - }); - window.dispatchEvent(notificationEvent); + if (!option.isTaken) { + props.clearTakenVisible(); } + + const orderByChangeEvent = new CustomEvent("orderByChange", { + detail: { + sortTaken: option.isTaken, + sort: option.value + }, + bubbles: true + }); + window.dispatchEvent(orderByChangeEvent); } }; return ( -
+
{CONTROL_TITLE}
setIsOpen(!isOpen)}> @@ -209,7 +238,7 @@ const SortControl: React.FC = ({ props }) => {
{isOpen && ( - +
{options.map((option) => (hasCollection || option.value != COLLECTION_OPTION) && ( -1 && options[currentIndex].value) === option.value ? "sort-dropdown-item sort-dropdown-highlight" : "sort-dropdown-item"} diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 842a6ad462..2cc438f4bc 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -16,6 +16,7 @@ import '../components/gr-info-panel/gr-info-panel'; import '../components/gr-collections-panel/gr-collections-panel'; import '../components/gr-keyboard-shortcut/gr-keyboard-shortcut'; import '../components/gr-sort-control/gr-sort-control'; +import '../components/gr-tab-swap/gr-tab-swap'; import '../components/gr-permissions-filter/gr-permissions-filter'; import '../components/gr-my-uploads/gr-my-uploads'; import '../components/gr-search-wrapper/gr-search-wrapper'; @@ -43,6 +44,7 @@ export var search = angular.module('kahuna.search', [ 'gr.panels', 'gr.keyboardShortcut', 'gr.sortControl', + 'gr.tabSwapControl', 'gr.permissionsFilter', 'gr.myUploads', 'gr-searchWrapper', diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index abd7c0fdf6..7319a90a3e 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -12,6 +12,7 @@ import template from './query.html'; import {syntax} from './syntax/syntax'; import {grStructuredQuery} from './structured-query/structured-query'; import '../components/gr-sort-control/gr-sort-control'; +import '../components/gr-tab-swap/gr-tab-swap'; import '../components/gr-permissions-filter/gr-permissions-filter'; import '../components/gr-my-uploads/gr-my-uploads'; import { sendTelemetryForQuery } from '../services/telemetry'; @@ -33,6 +34,7 @@ export var query = angular.module('kahuna.search.query', [ grStructuredQuery.name, 'util.storage', 'gr.sortControl', + 'gr.tabSwapControl', 'gr.permissionsFilter', 'gr.myUploads' ]); diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index 32a6abdc05..dbb1c12a0c 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -21,7 +21,7 @@
+ image-results-count"> {{ctrl.totalResults | toLocaleString}} matches
+ +
+ + +
+
    $window._clientConfig.showSendToPhotoSales; }; + //-taken and sort controls- + ctrl.clearTakenVisible = () => storage.setJs("takenTabVisible", "hidden", true); + ctrl.setTakenVisible = () => storage.setJs("takenTabVisible", "visible", true); + ctrl.getTakenVisible = () => storage.getJs("takenTabVisible", true) ? storage.getJs("takenTabVisible", true) : "hidden"; + ctrl.getCollectionsPanelVisible = () => storage.getJs("collectionsPanelState", false) ? !(storage.getJs("collectionsPanelState", false).hidden) : false; + ctrl.getInfoPanelVisible = () => { + const infoPanelState = storage.getJs("metadataPanelState", false); + if (!infoPanelState) return false; + return !infoPanelState.hidden; + } + + //-sort control- + function updateSortChips (sortSel) { + ctrl.sortProps.selectedOption = sortSel; + let toParams = { + ...$stateParams, + orderBy: manageSortSelection(sortSel.value) + } + $state.transitionTo( + $state.current, + toParams, + { reload: true, inherit: false, notify: true } + ); + } + + ctrl.sortProps = { + options: SortOptions, + selectedOption: DefaultSortOption, + onSelect: updateSortChips, + query: $stateParams.query, + orderBy: $stateParams.orderBy, + clearTakenVisible: ctrl.clearTakenVisible, + panelVisible: ctrl.getInfoPanelVisible() + }; + //-end sort control- + + //-tab swap control- + var hasTakenDateClause = "has:dateTaken"; + var noTakenDateClause = "-has:dateTaken"; + var takenSort = "taken"; + + function manageTakenDateTab(tabSelected, sortOrder) { + let toParams = { + ...$stateParams + }; + //-clean- + let oldQuery = $stateParams.query ? $stateParams.query : ''; + oldQuery = oldQuery.replace(noTakenDateClause, '').replace(hasTakenDateClause, '').trim(); + let newSortOrder = manageSortSelection(sortOrder); + let takenTabVisible = ctrl.getTakenVisible(); + + if (tabSelected === 'with') { + toParams = { + ...$stateParams, + query: `${oldQuery} ${hasTakenDateClause}`, + orderBy: newSortOrder + } + } else { + if ((sortOrder && sortOrder.includes(takenSort)) || (takenTabVisible === 'visible')) { + toParams = { + ...$stateParams, + query: `${oldQuery} ${noTakenDateClause}`, + orderBy: newSortOrder + } + } else { + toParams = { + ...$stateParams, + query: oldQuery, + orderBy: newSortOrder + } + } + } + $state.transitionTo( + $state.current, + toParams, + { reload: true, inherit: false, notify: true } + ); + } + + async function checkForNoTakenDate() { + let tempQuery = $stateParams.query ? $stateParams.query : ''; + let isTaken = ($stateParams.orderBy && $stateParams.orderBy.includes(takenSort)) || tempQuery.includes(hasTakenDateClause); + if (!isTaken) return 0; + let oldQuery = storage.getJs("previousQuery", true); + tempQuery = tempQuery.replace(noTakenDateClause, '').replace(hasTakenDateClause, '').trim(); + storage.setJs("previousQuery", tempQuery, true); + let query = `${tempQuery} ${noTakenDateClause}`.trim(); + var resp = await search({query: query, length: 0}); + return resp.total; + }; + + ctrl.tabSwapProps = { + orderBy: $stateParams.orderBy ? $stateParams.orderBy : '', + query: $stateParams.query ? $stateParams.query : '', + without: noTakenDateClause, + taken: takenSort, + onSelect: manageTakenDateTab, + takenVisible: ctrl.getTakenVisible(), + clearTakenVisible: ctrl.clearTakenVisible, + setTakenVisible: ctrl.setTakenVisible, + noTakenDateCount: 0, + panelVisible: ctrl.getCollectionsPanelVisible() + }; + checkForNoTakenDate().then(noTakenTotal => { + ctrl.tabSwapProps = { ...ctrl.tabSwapProps, + noTakenDateCount: noTakenTotal + }; + }); + //-end tab swap- + // Panel control ctrl.metadataPanel = panels.metadataPanel; ctrl.collectionsPanel = panels.collectionsPanel; @@ -164,6 +286,27 @@ results.controller('SearchResultsCtrl', [ results.clear(); results.resize(totalLength); + if (ctrl.tabSwapProps.orderBy.includes(takenSort) && 0 < ctrl.tabSwapProps.noTakenDateCount) { + const notificationEvent = new CustomEvent("newNotification", { + detail: { + announceId: "sortByTakenDate", + description: "There are " + ctrl.tabSwapProps.noTakenDateCount + " images with no taken date", + category: "information", + lifespan: "transient" + }, + bubbles: true + }); + window.dispatchEvent(notificationEvent); + } else { + const notificationEvent = new CustomEvent("removeNotification", { + detail: { + announceId: "sortByTakenDate", + }, + bubbles: true + }); + window.dispatchEvent(notificationEvent); + } + imagesPositions = new Map(); checkForNewImages(); @@ -326,7 +469,7 @@ results.controller('SearchResultsCtrl', [ return $stateParams.query || '*'; } - function search({until, since, offset, length, orderBy, countAll} = {}) { + function search({query, until, since, offset, length, orderBy, countAll} = {}) { // FIXME: Think of a way to not have to add a param in a million places to add it /* @@ -344,6 +487,9 @@ results.controller('SearchResultsCtrl', [ * `checkForNewImages` deals with that. If it's the first search, we * will use `stateParams.until` if available. */ + if (angular.isUndefined(query)) { + query = $stateParams.query; + } if (angular.isUndefined(until)) { until = lastSearchFirstResultTime || $stateParams.until; } @@ -357,7 +503,8 @@ results.controller('SearchResultsCtrl', [ countAll = true; } - return mediaApi.search($stateParams.query, angular.extend({ + + return mediaApi.search(query, angular.extend({ ids: $stateParams.ids, archived: $stateParams.archived, free: $stateParams.nonFree === 'true' ? undefined : true, diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index 021975c945..96c533fc92 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -1232,7 +1232,7 @@ textarea.ng-invalid { .results { display: flex; flex-wrap: wrap; - top: 35px; + top: 96px; position: relative; } From ec8b5d0195aea6813ea593f7d7012a6b7f8f9325 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Tue, 17 Jun 2025 16:57:31 +0100 Subject: [PATCH 02/66] JS syntax corrections --- .../gr-panel-button/gr-panel-button.js | 2 +- kahuna/public/js/search/results.js | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/kahuna/public/js/components/gr-panel-button/gr-panel-button.js b/kahuna/public/js/components/gr-panel-button/gr-panel-button.js index e6e1b04505..11e1540e1e 100644 --- a/kahuna/public/js/components/gr-panel-button/gr-panel-button.js +++ b/kahuna/public/js/components/gr-panel-button/gr-panel-button.js @@ -22,7 +22,7 @@ panelButton.controller('GrPanelButton', ['$scope', 'inject$', function($scope, i detail: {panel: ctrl.name}, bubbles: true })); - } + }; ctrl.lockPanel = () => panel.setLocked(true); ctrl.unlockPanel = () => panel.setLocked(false); ctrl.hidePanel = () => { diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 7ce4c38f98..9d2806cb9f 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -125,9 +125,9 @@ results.controller('SearchResultsCtrl', [ ctrl.getCollectionsPanelVisible = () => storage.getJs("collectionsPanelState", false) ? !(storage.getJs("collectionsPanelState", false).hidden) : false; ctrl.getInfoPanelVisible = () => { const infoPanelState = storage.getJs("metadataPanelState", false); - if (!infoPanelState) return false; + if (!infoPanelState) { return false; } return !infoPanelState.hidden; - } + }; //-sort control- function updateSortChips (sortSel) { @@ -135,7 +135,7 @@ results.controller('SearchResultsCtrl', [ let toParams = { ...$stateParams, orderBy: manageSortSelection(sortSel.value) - } + }; $state.transitionTo( $state.current, toParams, @@ -174,20 +174,20 @@ results.controller('SearchResultsCtrl', [ ...$stateParams, query: `${oldQuery} ${hasTakenDateClause}`, orderBy: newSortOrder - } + }; } else { if ((sortOrder && sortOrder.includes(takenSort)) || (takenTabVisible === 'visible')) { toParams = { ...$stateParams, query: `${oldQuery} ${noTakenDateClause}`, orderBy: newSortOrder - } + }; } else { toParams = { ...$stateParams, query: oldQuery, orderBy: newSortOrder - } + }; } } $state.transitionTo( @@ -200,8 +200,7 @@ results.controller('SearchResultsCtrl', [ async function checkForNoTakenDate() { let tempQuery = $stateParams.query ? $stateParams.query : ''; let isTaken = ($stateParams.orderBy && $stateParams.orderBy.includes(takenSort)) || tempQuery.includes(hasTakenDateClause); - if (!isTaken) return 0; - let oldQuery = storage.getJs("previousQuery", true); + if (!isTaken) { return 0; } tempQuery = tempQuery.replace(noTakenDateClause, '').replace(hasTakenDateClause, '').trim(); storage.setJs("previousQuery", tempQuery, true); let query = `${tempQuery} ${noTakenDateClause}`.trim(); @@ -300,7 +299,7 @@ results.controller('SearchResultsCtrl', [ } else { const notificationEvent = new CustomEvent("removeNotification", { detail: { - announceId: "sortByTakenDate", + announceId: "sortByTakenDate" }, bubbles: true }); From 335e25e5e588e19457734d20ffd070bbeb2a43ad Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Tue, 17 Jun 2025 17:12:42 +0100 Subject: [PATCH 03/66] add in new control files --- .../js/components/gr-tab-swap/gr-tab-swap.css | 43 +++++ .../js/components/gr-tab-swap/gr-tab-swap.tsx | 149 ++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 kahuna/public/js/components/gr-tab-swap/gr-tab-swap.css create mode 100644 kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx diff --git a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.css b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.css new file mode 100644 index 0000000000..beffcae482 --- /dev/null +++ b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.css @@ -0,0 +1,43 @@ +.gr-tab-wrapper { + padding: 4px; +} + +.gr-tab-panel-margin { + margin-left: 250px; +} + +.gr-tab-container { + font-family: Arial, sans-serif; + display: flex; + width: fit-content; +} + +.gr-tab { + cursor: pointer; + font-size: 14px; + transition: background-color 0.3s ease; + padding: 8px 16px; + background-color: #444; + color: #ccc; + border: none; + outline: none; + position: relative; +} + +.gr-tab:hover { + color: white; +} + +.gr-tab.active { + background-color: #666; +} + +.gr-tab.active::before { + content: ""; + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 3px; + background-color: #00bfff; +} diff --git a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx new file mode 100644 index 0000000000..f4586b4bde --- /dev/null +++ b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx @@ -0,0 +1,149 @@ +import * as React from "react"; +import * as angular from "angular"; +import { useState, useEffect, KeyboardEvent } from "react"; +import { react2angular } from "react2angular"; + +import './gr-tab-swap.css'; + +export interface TabSwapProps { + orderBy: string; + query: string; + without: string; + taken: string; + onSelect: (selected: string, sort: string) => void; + takenVisible: string; + clearTakenVisible: () => void; + setTakenVisible: () => void; + noTakenDateCount: number; + panelVisible: boolean; +} + +export interface TabSwapWrapperProps { + props: TabSwapProps; +} + +type TabType = 'with' | 'without'; + +const TabControl: React.FC = ({props}) => { + + const withLabel = "With taken date"; + const withoutLabel = "Without taken date"; + const CONTROL_TITLE = "Has Taken Date Selector"; + const PANEL_IDENTIFIER = "collections"; + const HAS_DATE_TAKEN_QUERY = "has:dateTaken"; + + // map props to state + const hasWithoutVal = props.query.includes(props.without); + const hasTakenSort = props.orderBy.includes(props.taken); + const tempTakenVisible = (props.takenVisible == 'visible'); + const noTakenDateCount = props.query.includes(HAS_DATE_TAKEN_QUERY) ? props.noTakenDateCount : 0; + + const [activeTab, setActiveTab] = useState(hasWithoutVal ? 'without' : 'with'); + const [sortTaken, setSortTaken] = useState(hasTakenSort); + const [takenVisible, setTakenVisible] = useState(tempTakenVisible); + const [panelVisible, setPanelVisible] = useState(props.panelVisible); + + const handleTabClick = (tabSelected: string) => { + if(tabSelected !== activeTab) { + if (tabSelected === 'with') { + setActiveTab('with'); + props.clearTakenVisible(); + let orderBy = props.orderBy.includes(props.taken) ? props.orderBy : ("-" + props.taken); + props.onSelect(tabSelected, orderBy); + } else { + setActiveTab('without'); + props.setTakenVisible(); + props.onSelect(tabSelected, undefined); + } + } + } + + const handleKeyboard = (event:KeyboardEvent) => { + if (event.code === 'Space') { + event.preventDefault(); + event.stopPropagation(); + const tabSelected = activeTab; + if (tabSelected === 'without') { + setActiveTab('with'); + props.clearTakenVisible(); + let orderBy = props.orderBy.includes(props.taken) ? props.orderBy : ("-" + props.taken); + props.onSelect('with', orderBy); + } else { + setActiveTab('without'); + props.setTakenVisible(); + props.onSelect('without', undefined); + } + } + }; + + useEffect(() => { + const handleLogoClick = (e: any) => { + setActiveTab('with'); + setSortTaken(false); + setTakenVisible(false); + props.clearTakenVisible(); + }; + + const handleOrderByChange = (e: any) => { + const newTaken = e.detail.sortTaken ? e.detail.sortTaken : false; + if (newTaken) { + props.onSelect('with', e.detail.sort); + } else { + props.onSelect('without', e.detail.sort); + } + }; + + const handlePanelShow = (event: any) => { + const panel = event.detail.panel; + if (panel === PANEL_IDENTIFIER) { + setPanelVisible(true); + } + }; + + const handlePanelHide = (event: any) => { + const panel = event.detail.panel; + if (panel === PANEL_IDENTIFIER) { + setPanelVisible(false); + } + }; + + window.addEventListener("logoClick", handleLogoClick); + window.addEventListener("orderByChange", handleOrderByChange); + window.addEventListener("panelHide", handlePanelHide); + window.addEventListener("panelShow", handlePanelShow); + + // Clean up the event listener when the component unmounts + return () => { + window.removeEventListener("logoClick", handleLogoClick); + window.removeEventListener("orderByChange", handleOrderByChange); + window.removeEventListener("panelHide", handlePanelHide); + window.removeEventListener("panelShow", handlePanelShow); + }; + }, []); + + return ( +
    + {((sortTaken || takenVisible) && (noTakenDateCount > 0)) && ( +
    +
    handleTabClick('with')} + aria-label={`${withLabel} ${activeTab === 'with' ? 'selected' : ''}`} + > + {withLabel} +
    +
    handleTabClick('without')} + aria-label={`${withoutLabel} ${activeTab === 'without' ? 'selected' : ''}`} + > + {`${withoutLabel} (${props.noTakenDateCount} matches)`} +
    +
    + )} +
    + ); +}; + +export const tabSwapControl = angular.module('gr.tabSwapControl', []) + .component('tabSwapControl', react2angular(TabControl, ["props"])); From be5d9a52a4656d0341a8aa5bc6aa4b60b799019e Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Tue, 17 Jun 2025 17:15:40 +0100 Subject: [PATCH 04/66] js syntax errors --- kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx index f4586b4bde..361e170d32 100644 --- a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx +++ b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx @@ -44,11 +44,11 @@ const TabControl: React.FC = ({props}) => { const [panelVisible, setPanelVisible] = useState(props.panelVisible); const handleTabClick = (tabSelected: string) => { - if(tabSelected !== activeTab) { + if (tabSelected !== activeTab) { if (tabSelected === 'with') { setActiveTab('with'); props.clearTakenVisible(); - let orderBy = props.orderBy.includes(props.taken) ? props.orderBy : ("-" + props.taken); + const orderBy = props.orderBy.includes(props.taken) ? props.orderBy : ("-" + props.taken); props.onSelect(tabSelected, orderBy); } else { setActiveTab('without'); @@ -56,7 +56,7 @@ const TabControl: React.FC = ({props}) => { props.onSelect(tabSelected, undefined); } } - } + }; const handleKeyboard = (event:KeyboardEvent) => { if (event.code === 'Space') { @@ -66,7 +66,7 @@ const TabControl: React.FC = ({props}) => { if (tabSelected === 'without') { setActiveTab('with'); props.clearTakenVisible(); - let orderBy = props.orderBy.includes(props.taken) ? props.orderBy : ("-" + props.taken); + const orderBy = props.orderBy.includes(props.taken) ? props.orderBy : ("-" + props.taken); props.onSelect('with', orderBy); } else { setActiveTab('without'); From 5fa73daf60686ed0702f545eea1575c94ed1be66 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Mon, 23 Jun 2025 16:14:38 +0100 Subject: [PATCH 05/66] correcting sort by taken date ui defects --- .../gr-notifications-banner.tsx | 18 +++++----- .../gr-sort-control/gr-sort-control.css | 1 + .../gr-sort-control/gr-sort-control.tsx | 19 ++++------ kahuna/public/js/preview/image.html | 3 +- kahuna/public/js/search/index.js | 13 +++++-- kahuna/public/js/search/query.html | 7 ---- kahuna/public/js/search/results.js | 35 +++++++++++-------- 7 files changed, 48 insertions(+), 48 deletions(-) diff --git a/kahuna/public/js/components/gr-notifications-banner/gr-notifications-banner.tsx b/kahuna/public/js/components/gr-notifications-banner/gr-notifications-banner.tsx index a19bd943f5..0b5d93e211 100644 --- a/kahuna/public/js/components/gr-notifications-banner/gr-notifications-banner.tsx +++ b/kahuna/public/js/components/gr-notifications-banner/gr-notifications-banner.tsx @@ -151,17 +151,17 @@ const NotificationsBanner: React.FC = () => { }); }; - const newNotification = (event:any) => { - const notification = event.detail; - setNotifications(prev_notifs => mergeArraysByKey(prev_notifs, [notification], 'announceId')); - }; + useEffect(() => { + const newNotification = (event:any) => { + const notification = event.detail; + setNotifications(prev_notifs => mergeArraysByKey(prev_notifs, [notification], 'announceId')); + }; - const removeNotification = (event:any) => { - const notificationId = event.detail.announceId; - setNotifications(prev_notifs => prev_notifs.filter(notif => notif.announceId !== notificationId)); - }; + const removeNotification = (event:any) => { + const notificationId = event.detail.announceId; + setNotifications(prev_notifs => prev_notifs.filter(notif => notif.announceId !== notificationId)); + }; - useEffect(() => { const announce = window._clientConfig.announcements; const tdy = todayStr(); let notif_cookie = getCookie(NOTIFICATION_COOKIE); diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css index 47ce1ffd45..4fc78d664f 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css @@ -99,6 +99,7 @@ .sort-dropdown-item { border: 1px solid #ccc; + font-size: 1.4rem; } .sort-dropdown-item:hover { diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index bb95716532..b540d8af5a 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -49,6 +49,7 @@ export interface SortDropdownProps { onSelect: (option: SortDropdownOption) => void; query?: string | ""; orderBy?: string | ""; + previousTaken?: string | ""; clearTakenVisible: () => void; panelVisible?: boolean | false; } @@ -109,17 +110,6 @@ const SortControl: React.FC = ({ props }) => { } }; - useEffect(() => { - if (hasCollection) { - const collOpt = options.filter(opt => opt.value == COLLECTION_OPTION)[0]; - setSelection(collOpt); - } else { - if (selectedOption.isCollection) { - setSelection(previousOption); - } - } - }, [hasCollection]); - useEffect(() => { if (selectedOption && selectedOption !== previousOption && !selectedOption.isCollection ) { setPrevious(selectedOption); @@ -166,7 +156,12 @@ const SortControl: React.FC = ({ props }) => { if (!tempQuery.includes(HAS_DATE_TAKEN_QUERY) && selOpt.isTaken) { setSelection(defSort); } else { - setSelection(selOpt); + if (props.previousTaken === selOpt.value || !selOpt.isTaken) { + setSelection(selOpt); + } else { + const tOpt = props.options.filter(o => o.value === props.previousTaken)[0]; + setSelection(tOpt); + } } } else { setSelection(defSort); diff --git a/kahuna/public/js/preview/image.html b/kahuna/public/js/preview/image.html index 7af1a1a91b..11779c5576 100644 --- a/kahuna/public/js/preview/image.html +++ b/kahuna/public/js/preview/image.html @@ -118,8 +118,7 @@
    - Uploaded: {{::ctrl.image.data.uploadTime | date:'dd/MM/yy'}} - {{::ctrl.image.data.uploadTime | date:'HH:mm'}} + Uploaded: {{::ctrl.image.data.uploadTime | date:'d MMM yyyy, HH:mm'}} diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 2cc438f4bc..18dc0c6db0 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -329,15 +329,22 @@ search.run(['$rootScope', '$state', function($rootScope, $state) { $rootScope.$on('$stateChangeStart', (_, toState, toParams, fromState, fromParams) => { if (toState.name === 'search.results') { //If moving to a collection, sorts images by time added to a collection by default - //allows sorting by newest first if set by user. + //allows sorting by newest first if set by user. Need to account for 'With Taken Date' tab impacts on query if (toParams.query && toParams.query.indexOf('~') === 0) { - const sameQuery = toParams.query === fromParams.query; - toParams.orderBy = sameQuery ? toParams.orderBy : 'dateAddedToCollection'; + const toQuery = toParams.query ? toParams.query.replace('-has:dateTaken', '').replace('has:dateTaken', '').trim() : ""; + const fromQuery = fromParams.query ? fromParams.query.replace('-has:dateTaken', '').replace('has:dateTaken', '').trim() : ""; + toParams.orderBy = (toQuery === fromQuery) ? toParams.orderBy : 'dateAddedToCollection'; } //If moving from a collection to a non-collection, reset order to default. else if (toParams.orderBy === 'dateAddedToCollection') { delete toParams.orderBy; } + + // handle clear hasTaken chip from search + if ( (toParams.orderBy && toParams.orderBy.includes('taken')) && + (!toParams.query || !toParams.query.includes('has:dateTaken')) ) { + delete toParams.orderBy; + } } }); }]); diff --git a/kahuna/public/js/search/query.html b/kahuna/public/js/search/query.html index 33108efb37..e054fac050 100644 --- a/kahuna/public/js/search/query.html +++ b/kahuna/public/js/search/query.html @@ -83,13 +83,6 @@
- -
-
- -
-
- diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 9d2806cb9f..6df6593070 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -118,23 +118,31 @@ results.controller('SearchResultsCtrl', [ ctrl.showSendToPhotoSales = () => $window._clientConfig.showSendToPhotoSales; }; + // Panel control + ctrl.metadataPanel = panels.metadataPanel; + ctrl.collectionsPanel = panels.collectionsPanel; + //-taken and sort controls- + var hasTakenDateClause = "has:dateTaken"; + var noTakenDateClause = "-has:dateTaken"; + var takenSort = "taken"; ctrl.clearTakenVisible = () => storage.setJs("takenTabVisible", "hidden", true); ctrl.setTakenVisible = () => storage.setJs("takenTabVisible", "visible", true); ctrl.getTakenVisible = () => storage.getJs("takenTabVisible", true) ? storage.getJs("takenTabVisible", true) : "hidden"; ctrl.getCollectionsPanelVisible = () => storage.getJs("collectionsPanelState", false) ? !(storage.getJs("collectionsPanelState", false).hidden) : false; - ctrl.getInfoPanelVisible = () => { - const infoPanelState = storage.getJs("metadataPanelState", false); - if (!infoPanelState) { return false; } - return !infoPanelState.hidden; - }; + ctrl.getInfoPanelVisible = () => storage.getJs("metadataPanelState", false) ? !(storage.getJs("metadataPanelState", false).hidden) : false; + ctrl.getLastTakenSort = () => storage.getJs("lastTakenSort", false) ? storage.getJs("lastTakenSort", false) : ""; //-sort control- function updateSortChips (sortSel) { ctrl.sortProps.selectedOption = sortSel; + var orderBy = manageSortSelection(sortSel.value); + if (orderBy && orderBy.includes(takenSort)) { + storage.setJs("lastTakenSort", orderBy, false); + } let toParams = { ...$stateParams, - orderBy: manageSortSelection(sortSel.value) + orderBy: orderBy }; $state.transitionTo( $state.current, @@ -149,17 +157,18 @@ results.controller('SearchResultsCtrl', [ onSelect: updateSortChips, query: $stateParams.query, orderBy: $stateParams.orderBy, + previousTaken: ctrl.getLastTakenSort(), clearTakenVisible: ctrl.clearTakenVisible, panelVisible: ctrl.getInfoPanelVisible() }; //-end sort control- //-tab swap control- - var hasTakenDateClause = "has:dateTaken"; - var noTakenDateClause = "-has:dateTaken"; - var takenSort = "taken"; - - function manageTakenDateTab(tabSelected, sortOrder) { + function manageTakenDateTab(tabSelected, inSortOrder) { + let sortOrder = inSortOrder; + if (tabSelected === 'with') { + sortOrder = storage.getJs("lastTakenSort", false); + } let toParams = { ...$stateParams }; @@ -227,10 +236,6 @@ results.controller('SearchResultsCtrl', [ }); //-end tab swap- - // Panel control - ctrl.metadataPanel = panels.metadataPanel; - ctrl.collectionsPanel = panels.collectionsPanel; - ctrl.images = []; if (ctrl.image && ctrl.image.data.softDeletedMetadata !== undefined) { ctrl.isDeleted = true; } ctrl.newImagesCount = 0; From b9828e1432532f7f1301c8d7eb5d279be04a3c86 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Wed, 13 Aug 2025 17:26:04 +0100 Subject: [PATCH 06/66] revised build of sort by taken date ui --- .../js/components/gr-panels/gr-panels.js | 8 +- .../gr-sort-control/base-sort-control.tsx | 203 ++++++++++++++ .../gr-extended-sort-control.tsx | 110 ++++++++ .../gr-sort-control/gr-sort-control-config.ts | 4 +- .../gr-sort-control/gr-sort-control.css | 13 + .../gr-sort-control/gr-sort-control.tsx | 251 +----------------- .../js/components/gr-tab-swap/gr-tab-swap.tsx | 102 +++---- kahuna/public/js/search/index.js | 12 +- kahuna/public/js/search/query.js | 15 +- kahuna/public/js/search/results.html | 5 +- kahuna/public/js/search/results.js | 198 +++++++------- kahuna/public/js/search/syntax/syntax.html | 8 + kahuna/public/stylesheets/main.css | 4 +- 13 files changed, 497 insertions(+), 436 deletions(-) create mode 100644 kahuna/public/js/components/gr-sort-control/base-sort-control.tsx create mode 100644 kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx diff --git a/kahuna/public/js/components/gr-panels/gr-panels.js b/kahuna/public/js/components/gr-panels/gr-panels.js index 0d84e88ff2..5ffb9fb6ff 100644 --- a/kahuna/public/js/components/gr-panels/gr-panels.js +++ b/kahuna/public/js/components/gr-panels/gr-panels.js @@ -69,7 +69,13 @@ panels.directive('grPanel', ['$timeout', '$window', 'inject$', 'subscribe$', // Then hide the panel subscribe$(scope, scrollWhileVisAndUnlocked$, () => { - scope.$apply(() => panel.setHidden(true)); + scope.$apply(() => { + panel.setHidden(true); + window.dispatchEvent(new CustomEvent("panelHide", { + detail: {panel: "scroll"}, + bubbles: true + })); + }); }); } }; diff --git a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx new file mode 100644 index 0000000000..5c118f6386 --- /dev/null +++ b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx @@ -0,0 +1,203 @@ +import * as React from "react"; +import { useEffect, useState, KeyboardEvent } from "react"; +import { DefaultSortOption, CollectionSortOption } from "./gr-sort-control-config"; + +import "./gr-sort-control.css"; + +const SELECT_OPTION = "Select an option"; +const DEFAULT_OPTION = DefaultSortOption.value; +const COLLECTION_OPTION = CollectionSortOption.value; +const CONTROL_TITLE = "Sort by:"; +const SORT_ORDER = "Sort order"; +const PANEL_IDENTIFIER = "info"; +const SCROLL_IDENTIFIER = "scroll"; +const HAS_DATE_TAKEN_QUERY = "has:dateTaken"; + +const downArrowIcon = () => + + + ; + +const emptyIcon = () => + + + ; + +const tickIcon = () => + + + ; + +const sortIcon = () => + + + + + ; + +export interface SortDropdownOption { + value: string; + label: string; + isCollection: boolean; + isTaken: boolean; +} + +export interface SortDropdownProps { + options: SortDropdownOption[]; + startSelectedOption?: SortDropdownOption | null; + onSelect: (option: SortDropdownOption) => void; + startHasCollection?: boolean | false; + panelVisible?: boolean | false; +} + +const hasClassInSelfOrParent = (node: Element | null, className: string): boolean => { + if (node !== null && node.classList && node.classList.contains(className)) { + return true; + } + + while (node && node.parentNode && node.parentNode !== document) { + node = node.parentNode as Element; + if (node.classList && node.classList.contains(className)) { + return true; + } + } + + return false; +}; + +export const BaseSortControl: React.FC = ({ + options, + startSelectedOption, + onSelect, + startHasCollection, + panelVisible + }) => { + + const [hasCollection, setHasCollection] = useState(startHasCollection); + const startSort:SortDropdownOption = startSelectedOption ? startSelectedOption : options.filter(opt => opt.value == DEFAULT_OPTION)[0]; + const [isOpen, setIsOpen] = useState(false); + const [selectedOption, setSelection] = useState(startSort); + const [previousOption, setPrevious] = useState(startSort) + const [currentIndex, setCurrentIndex] = useState(-1); + const [isPanelVisible, setPanelVisible] = useState(panelVisible); + + const handleArrowKeys = (event:KeyboardEvent) => { + if (event.key === 'ArrowDown' || + event.key === 'ArrowUp' || + event.key === 'Enter' || + event.code === 'Space') { + event.preventDefault(); + event.stopPropagation(); + let rowCount = options.length; + if (!hasCollection) { --rowCount; } + if (event.key === 'ArrowDown') { + setCurrentIndex((prevIndex) => (prevIndex + 1) % rowCount); + } else if (event.key === 'ArrowUp') { + setCurrentIndex((prevIndex) => (prevIndex - 1 + rowCount) % rowCount); + } else if (event.key === 'Enter' || event.code === 'Space') { + if (!isOpen) { + setCurrentIndex(options.findIndex(opt => opt.value === selectedOption.value)); + setIsOpen(true); + } else { + handleOptionClick(options[currentIndex]); + } + } + } + }; + + useEffect(() => { + if (selectedOption && selectedOption !== previousOption && !selectedOption.isCollection ) { + setPrevious(selectedOption); + } + }, [selectedOption]); + + // initialisation + useEffect(() => { + const autoHideListener = (event: any) => { + if (event.type === "keydown" && event.key === "Escape") { + setIsOpen(false); + } else if (event.type !== "keydown") { + if (!hasClassInSelfOrParent(event.target, "sort-control")) { + setIsOpen(false); + } + } + }; + + const handlePanelShow = (event: any) => { + const panel = event.detail.panel; + if (panel === PANEL_IDENTIFIER) { + setPanelVisible(true); + } + }; + + const handlePanelHide = (event: any) => { + const panel = event.detail.panel; + if (panel === PANEL_IDENTIFIER || panel === SCROLL_IDENTIFIER) { + setPanelVisible(false); + } + }; + + window.addEventListener("mouseup", autoHideListener); + window.addEventListener("scroll", autoHideListener); + window.addEventListener("keydown", autoHideListener); + window.addEventListener("panelHide", handlePanelHide); + window.addEventListener("panelShow", handlePanelShow); + + // Clean up the event listener when the component unmounts + return () => { + setCurrentIndex(-1); + window.removeEventListener("mouseup", autoHideListener); + window.removeEventListener("scroll", autoHideListener); + window.removeEventListener("keydown", autoHideListener); + window.removeEventListener("panelHide", handlePanelHide); + window.removeEventListener("panelShow", handlePanelShow); + }; + }, []); + + const handleOptionClick = (option: SortDropdownOption) => { + setIsOpen(false); + if (option.value !== selectedOption.value) { + setSelection(option); + onSelect(option); + } + }; + + return ( +
+
{CONTROL_TITLE}
+
+
setIsOpen(!isOpen)}> +
+
{(selectedOption ? selectedOption.label : SELECT_OPTION)}
+
{downArrowIcon()}
+
+
+
setIsOpen(!isOpen)}> +
+
{sortIcon()}
+ {SORT_ORDER} +
+
+ {isOpen && ( +
+ + {options.map((option) => (hasCollection || option.value != COLLECTION_OPTION) && ( + -1 && options[currentIndex].value) === option.value ? "sort-dropdown-item sort-dropdown-highlight" : "sort-dropdown-item"} + key={option.value + "row"} + onClick={() => handleOptionClick(option)} + aria-label={option.label}> + + + + ))} + +
+ {(selectedOption.value == option.value) ? tickIcon() : emptyIcon()} + + {option.label} +
+ )} +
+
+ ); +}; diff --git a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx new file mode 100644 index 0000000000..b844193477 --- /dev/null +++ b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx @@ -0,0 +1,110 @@ +import * as React from "react"; +import * as angular from "angular"; +import { react2angular } from "react2angular"; +import { useEffect, useState, KeyboardEvent } from "react"; +import { BaseSortControl, SortDropdownOption, SortDropdownProps } from "./base-sort-control"; +import { SortOptions, DefaultSortOption, CollectionSortOption } from "./gr-sort-control-config"; +import { TabControl, TabSwapProps } from "../gr-tab-swap/gr-tab-swap"; + +import "./gr-sort-control.css"; + +export interface ExtendedSortProps { + onSortSelect: (option: SortDropdownOption, tabSelected: string, userTakenSelect: boolean, noTakenDateCount: number) => void; + query?: string | ""; + orderBy?: string | ""; + infoPanelVisible?: boolean | false; + collectionsPanelVisible?: boolean | false; + userTakenSelect?: boolean | false; + noTakenDateCount?: number | 0; +} + +export interface ExtendedSortWrapperProps { + props: ExtendedSortProps; +} + +const checkForCollection = (query:string): boolean => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); + +const ExtendedSortControl: React.FC = ({ props }) => { + + const noTakenDateClause = "-has:dateTaken"; + const takenDateClause = "has:dateTaken"; + const sortOptions = SortOptions; + const orderBy = props.orderBy; + const query = props.query; + + let startSortOption = DefaultSortOption; + if (!query.includes(noTakenDateClause) && (sortOptions.filter(o => o.value === orderBy)).length > 0) { + startSortOption = sortOptions.filter(o => o.value === orderBy)[0]; + } + + const startHasCollection = checkForCollection(query); + const [selSortOption, setSortOption] = useState(startSortOption); + const [userTakenSelect, setUserTakenSelect] = useState(props.userTakenSelect); + const [noTakenDateCount, setNoTakenDateCount] = useState(props.noTakenDateCount); + const [hasCollection, setHasCollection] = useState(startHasCollection); + + const onSortSelect = (selOption: SortDropdownOption) => { + setSortOption(selOption); + setUserTakenSelect(selOption.isTaken); + props.onSortSelect(selOption, 'with', selOption.isTaken, noTakenDateCount); + }; + + const onTabSelect = (withTaken: boolean) => { + let withTakenStr = 'with'; + if (!withTaken) { + withTakenStr = 'without'; + setSortOption(DefaultSortOption); + } + props.onSortSelect(selSortOption, withTakenStr, userTakenSelect, noTakenDateCount); + } + + // initialisation + useEffect(() => { + const handleLogoClick = (e: any) => { + setSortOption(DefaultSortOption); + setUserTakenSelect(false); + props.onSortSelect(DefaultSortOption, 'with', false, 0); + }; + + const handleQueryChange = (e: any) => { + const newQuery = e.detail.query ? (" " + e.detail.query) : ""; + setHasCollection(checkForCollection(newQuery)); + if (userTakenSelect && !newQuery.includes(takenDateClause)) { + props.onSortSelect(DefaultSortOption, 'with', false, 0); + } + }; + + window.addEventListener("logoClick", handleLogoClick); + window.addEventListener("queryChangeEvent", handleQueryChange); + + // Clean up the event listener when the component unmounts + return () => { + window.removeEventListener("logoClick", handleLogoClick); + window.removeEventListener("queryChangeEvent", handleQueryChange); + }; + + }, []); + + return ( +
+ + +
+ ); +} + +export const extendedSortControl = angular.module('gr.extendedSortControl', []) + .component('extendedSortControl', react2angular(ExtendedSortControl, ["props"])); + diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts index b825b50695..fda54bf9d4 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts @@ -4,7 +4,7 @@ export function manageSortSelection(newSelection:string): string { let newVal; switch (newSelection) { case "uploadNewOld": - newVal = undefined; + newVal = "newest"; break; case "oldest": newVal = "oldest"; @@ -19,7 +19,7 @@ export function manageSortSelection(newSelection:string): string { newVal = "dateAddedToCollection"; break; default: - newVal = undefined; + newVal = "newest"; break; } return newVal; diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css index 4fc78d664f..78b23df5a2 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css @@ -130,6 +130,19 @@ stroke: #fff; } +.extended-sort-control { + display:flex; + justify-content:space-between; + align-items:center; + padding:45px 1rem 0 1rem; + position:fixed; + left:0; + z-index:2; + background-color:#333; + width:100%; + box-sizing: border-box; +} + @media screen and (max-width: 880px) { .sort-selection-label { display: none; diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index b540d8af5a..cf725db4d3 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -1,260 +1,23 @@ import * as React from "react"; import * as angular from "angular"; import { react2angular } from "react2angular"; -import { useEffect, useState, KeyboardEvent } from "react"; -import { DefaultSortOption, CollectionSortOption } from "./gr-sort-control-config"; - -import "./gr-sort-control.css"; - -const SELECT_OPTION = "Select an option"; -const DEFAULT_OPTION = DefaultSortOption.value; -const COLLECTION_OPTION = CollectionSortOption.value; -const CONTROL_TITLE = "Sort by:"; -const SORT_ORDER = "Sort order"; -const PANEL_IDENTIFIER = "info"; -const HAS_DATE_TAKEN_QUERY = "has:dateTaken"; - -const downArrowIcon = () => - - - ; - -const emptyIcon = () => - - - ; - -const tickIcon = () => - - - ; - -const sortIcon = () => - - - - - ; - -export interface SortDropdownOption { - value: string; - label: string; - isCollection: boolean; - isTaken: boolean; -} - -export interface SortDropdownProps { - options: SortDropdownOption[]; - selectedOption?: SortDropdownOption | null; - onSelect: (option: SortDropdownOption) => void; - query?: string | ""; - orderBy?: string | ""; - previousTaken?: string | ""; - clearTakenVisible: () => void; - panelVisible?: boolean | false; -} +import { BaseSortControl, SortDropdownOption, SortDropdownProps } from "./base-sort-control"; +export type { SortDropdownOption }; +export type { SortDropdownProps }; export interface SortWrapperProps { props: SortDropdownProps; } -const checkForCollection = (query:string): boolean => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); - -const hasClassInSelfOrParent = (node: Element | null, className: string): boolean => { - if (node !== null && node.classList && node.classList.contains(className)) { - return true; - } - - while (node && node.parentNode && node.parentNode !== document) { - node = node.parentNode as Element; - if (node.classList && node.classList.contains(className)) { - return true; - } - } - - return false; -}; - const SortControl: React.FC = ({ props }) => { - const defOptVal:string = DEFAULT_OPTION; - const [hasCollection, setHasCollection] = useState(false); - const options = props.options; - const defSort:SortDropdownOption = options.filter(opt => opt.value == defOptVal)[0]; - const [isOpen, setIsOpen] = useState(false); - const [selectedOption, setSelection] = useState(defSort); - const [previousOption, setPrevious] = useState(defSort); - const [currentIndex, setCurrentIndex] = useState(-1); - const [panelVisible, setPanelVisible] = useState(props.panelVisible); - - const handleArrowKeys = (event:KeyboardEvent) => { - if (event.key === 'ArrowDown' || - event.key === 'ArrowUp' || - event.key === 'Enter' || - event.code === 'Space') { - event.preventDefault(); - event.stopPropagation(); - let rowCount = options.length; - if (!hasCollection) { --rowCount; } - if (event.key === 'ArrowDown') { - setCurrentIndex((prevIndex) => (prevIndex + 1) % rowCount); - } else if (event.key === 'ArrowUp') { - setCurrentIndex((prevIndex) => (prevIndex - 1 + rowCount) % rowCount); - } else if (event.key === 'Enter' || event.code === 'Space') { - if (!isOpen) { - setCurrentIndex(options.findIndex(opt => opt.value === selectedOption.value)); - setIsOpen(true); - } else { - handleOptionClick(options[currentIndex]); - } - } - } - }; - - useEffect(() => { - if (selectedOption && selectedOption !== previousOption && !selectedOption.isCollection ) { - setPrevious(selectedOption); - } - }, [selectedOption]); - - useEffect(() => { - const autoHideListener = (event: any) => { - if (event.type === "keydown" && event.key === "Escape") { - setIsOpen(false); - } else if (event.type !== "keydown") { - if (!hasClassInSelfOrParent(event.target, "sort-control")) { - setIsOpen(false); - } - } - }; - - const handleLogoClick = (e: any) => { - setSelection(defSort); - }; - - const handleQueryChange = (e: any) => { - const newQuery = e.detail.query ? (" " + e.detail.query) : ""; - setHasCollection(checkForCollection(newQuery)); - }; - - const handlePanelShow = (event: any) => { - const panel = event.detail.panel; - if (panel === PANEL_IDENTIFIER) { - setPanelVisible(true); - } - }; - - const handlePanelHide = (event: any) => { - const panel = event.detail.panel; - if (panel === PANEL_IDENTIFIER) { - setPanelVisible(false); - } - }; - - if (props.options.filter(o => o.value === props.orderBy).length > 0) { - const selOpt = props.options.filter(o => o.value === props.orderBy)[0]; - const tempQuery = props.query ? props.query : ""; - if (!tempQuery.includes(HAS_DATE_TAKEN_QUERY) && selOpt.isTaken) { - setSelection(defSort); - } else { - if (props.previousTaken === selOpt.value || !selOpt.isTaken) { - setSelection(selOpt); - } else { - const tOpt = props.options.filter(o => o.value === props.previousTaken)[0]; - setSelection(tOpt); - } - } - } else { - setSelection(defSort); - } - - if (props.query) { - setHasCollection(checkForCollection(props.query)); - } else if (props.options.filter(o => o.value === props.orderBy).length > 0) { - setHasCollection(props.options.filter(o => o.value === props.orderBy)[0].isCollection); - } else { - setHasCollection(false); - } - - window.addEventListener("logoClick", handleLogoClick); - window.addEventListener("queryChangeEvent", handleQueryChange); - window.addEventListener("mouseup", autoHideListener); - window.addEventListener("scroll", autoHideListener); - window.addEventListener("keydown", autoHideListener); - window.addEventListener("panelHide", handlePanelHide); - window.addEventListener("panelShow", handlePanelShow); - - // Clean up the event listener when the component unmounts - return () => { - setCurrentIndex(-1); - window.removeEventListener("logoClick", handleLogoClick); - window.removeEventListener("queryChangeEvent", handleQueryChange); - window.removeEventListener("mouseup", autoHideListener); - window.removeEventListener("scroll", autoHideListener); - window.removeEventListener("keydown", autoHideListener); - window.removeEventListener("panelHide", handlePanelHide); - window.removeEventListener("panelShow", handlePanelShow); - }; - }, []); - - const handleOptionClick = (option: SortDropdownOption) => { - setIsOpen(false); - if (option.value !== selectedOption.value) { - setSelection(option); - props.onSelect(option); - if (!option.isTaken) { - props.clearTakenVisible(); - } - - const orderByChangeEvent = new CustomEvent("orderByChange", { - detail: { - sortTaken: option.isTaken, - sort: option.value - }, - bubbles: true - }); - window.dispatchEvent(orderByChangeEvent); - } + const sortProps = { ...props, + simpleDisplay: true }; return ( -
-
{CONTROL_TITLE}
-
-
setIsOpen(!isOpen)}> -
-
{(selectedOption ? selectedOption.label : SELECT_OPTION)}
-
{downArrowIcon()}
-
-
-
setIsOpen(!isOpen)}> -
-
{sortIcon()}
- {SORT_ORDER} -
-
- {isOpen && ( - - - {options.map((option) => (hasCollection || option.value != COLLECTION_OPTION) && ( - -1 && options[currentIndex].value) === option.value ? "sort-dropdown-item sort-dropdown-highlight" : "sort-dropdown-item"} - key={option.value + "row"} - onClick={() => handleOptionClick(option)} - aria-label={option.label}> - - - - ))} - -
- {(selectedOption.value == option.value) ? tickIcon() : emptyIcon()} - - {option.label} -
- )} -
-
+ ); -}; +} export const sortControl = angular.module('gr.sortControl', []) .component('sortControl', react2angular(SortControl, ["props"])); diff --git a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx index 361e170d32..ec4f26e5b1 100644 --- a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx +++ b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx @@ -1,60 +1,47 @@ import * as React from "react"; -import * as angular from "angular"; import { useState, useEffect, KeyboardEvent } from "react"; -import { react2angular } from "react2angular"; import './gr-tab-swap.css'; export interface TabSwapProps { - orderBy: string; + onSelect: (withTaken: boolean) => void; query: string; - without: string; - taken: string; - onSelect: (selected: string, sort: string) => void; - takenVisible: string; - clearTakenVisible: () => void; - setTakenVisible: () => void; + showTakenTab: boolean; noTakenDateCount: number; panelVisible: boolean; } -export interface TabSwapWrapperProps { - props: TabSwapProps; -} - -type TabType = 'with' | 'without'; - -const TabControl: React.FC = ({props}) => { +export const TabControl: React.FC = ({ onSelect, query, showTakenTab, noTakenDateCount, panelVisible }) => { const withLabel = "With taken date"; const withoutLabel = "Without taken date"; const CONTROL_TITLE = "Has Taken Date Selector"; const PANEL_IDENTIFIER = "collections"; + const SCROLL_IDENTIFIER = "scroll"; const HAS_DATE_TAKEN_QUERY = "has:dateTaken"; + const taken = "taken"; + const without = `-${HAS_DATE_TAKEN_QUERY}`; + + let tabStart = 'with'; + if (query.includes(without)) { + tabStart = 'without'; + } - // map props to state - const hasWithoutVal = props.query.includes(props.without); - const hasTakenSort = props.orderBy.includes(props.taken); - const tempTakenVisible = (props.takenVisible == 'visible'); - const noTakenDateCount = props.query.includes(HAS_DATE_TAKEN_QUERY) ? props.noTakenDateCount : 0; + const [activeTab, setActiveTab] = useState(tabStart); + const [isSortTaken, setIsSortTaken] = useState(showTakenTab); + const [isPanelVisible, setIsPanelVisible] = useState(panelVisible); - const [activeTab, setActiveTab] = useState(hasWithoutVal ? 'without' : 'with'); - const [sortTaken, setSortTaken] = useState(hasTakenSort); - const [takenVisible, setTakenVisible] = useState(tempTakenVisible); - const [panelVisible, setPanelVisible] = useState(props.panelVisible); + let takenDateMsg = ""; + if (noTakenDateCount === 1) { + takenDateMsg = " (1 match)"; + } else if (noTakenDateCount > 1) { + takenDateMsg = ` (${noTakenDateCount.toLocaleString()} matches)`; + } const handleTabClick = (tabSelected: string) => { if (tabSelected !== activeTab) { - if (tabSelected === 'with') { - setActiveTab('with'); - props.clearTakenVisible(); - const orderBy = props.orderBy.includes(props.taken) ? props.orderBy : ("-" + props.taken); - props.onSelect(tabSelected, orderBy); - } else { - setActiveTab('without'); - props.setTakenVisible(); - props.onSelect(tabSelected, undefined); - } + setActiveTab(tabSelected); + onSelect('with' === tabSelected); } }; @@ -62,68 +49,44 @@ const TabControl: React.FC = ({props}) => { if (event.code === 'Space') { event.preventDefault(); event.stopPropagation(); - const tabSelected = activeTab; - if (tabSelected === 'without') { + if (activeTab === 'without') { setActiveTab('with'); - props.clearTakenVisible(); - const orderBy = props.orderBy.includes(props.taken) ? props.orderBy : ("-" + props.taken); - props.onSelect('with', orderBy); + onSelect(true); } else { setActiveTab('without'); - props.setTakenVisible(); - props.onSelect('without', undefined); + onSelect(false); } } }; useEffect(() => { - const handleLogoClick = (e: any) => { - setActiveTab('with'); - setSortTaken(false); - setTakenVisible(false); - props.clearTakenVisible(); - }; - - const handleOrderByChange = (e: any) => { - const newTaken = e.detail.sortTaken ? e.detail.sortTaken : false; - if (newTaken) { - props.onSelect('with', e.detail.sort); - } else { - props.onSelect('without', e.detail.sort); - } - }; - const handlePanelShow = (event: any) => { const panel = event.detail.panel; if (panel === PANEL_IDENTIFIER) { - setPanelVisible(true); + setIsPanelVisible(true); } }; const handlePanelHide = (event: any) => { const panel = event.detail.panel; - if (panel === PANEL_IDENTIFIER) { - setPanelVisible(false); + if (panel === PANEL_IDENTIFIER || panel === SCROLL_IDENTIFIER) { + setIsPanelVisible(false); } }; - window.addEventListener("logoClick", handleLogoClick); - window.addEventListener("orderByChange", handleOrderByChange); window.addEventListener("panelHide", handlePanelHide); window.addEventListener("panelShow", handlePanelShow); // Clean up the event listener when the component unmounts return () => { - window.removeEventListener("logoClick", handleLogoClick); - window.removeEventListener("orderByChange", handleOrderByChange); window.removeEventListener("panelHide", handlePanelHide); window.removeEventListener("panelShow", handlePanelShow); }; }, []); return ( -
- {((sortTaken || takenVisible) && (noTakenDateCount > 0)) && ( +
+ {(isSortTaken && noTakenDateCount > 0) && (
= ({props}) => { onClick={() => handleTabClick('without')} aria-label={`${withoutLabel} ${activeTab === 'without' ? 'selected' : ''}`} > - {`${withoutLabel} (${props.noTakenDateCount} matches)`} + {`${withoutLabel}${takenDateMsg}`}
)}
); }; - -export const tabSwapControl = angular.module('gr.tabSwapControl', []) - .component('tabSwapControl', react2angular(TabControl, ["props"])); diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 18dc0c6db0..b6da170661 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -16,7 +16,7 @@ import '../components/gr-info-panel/gr-info-panel'; import '../components/gr-collections-panel/gr-collections-panel'; import '../components/gr-keyboard-shortcut/gr-keyboard-shortcut'; import '../components/gr-sort-control/gr-sort-control'; -import '../components/gr-tab-swap/gr-tab-swap'; +import '../components/gr-sort-control/gr-extended-sort-control'; import '../components/gr-permissions-filter/gr-permissions-filter'; import '../components/gr-my-uploads/gr-my-uploads'; import '../components/gr-search-wrapper/gr-search-wrapper'; @@ -44,7 +44,7 @@ export var search = angular.module('kahuna.search', [ 'gr.panels', 'gr.keyboardShortcut', 'gr.sortControl', - 'gr.tabSwapControl', + 'gr.extendedSortControl', 'gr.permissionsFilter', 'gr.myUploads', 'gr-searchWrapper', @@ -341,10 +341,10 @@ search.run(['$rootScope', '$state', function($rootScope, $state) { } // handle clear hasTaken chip from search - if ( (toParams.orderBy && toParams.orderBy.includes('taken')) && - (!toParams.query || !toParams.query.includes('has:dateTaken')) ) { - delete toParams.orderBy; - } + //if ( (toParams.orderBy && toParams.orderBy.includes('taken')) && + // (!toParams.query || !toParams.query.includes('has:dateTaken')) ) { + // delete toParams.orderBy; + //} } }); }]); diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 7319a90a3e..9bd0c1cf9d 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -12,7 +12,7 @@ import template from './query.html'; import {syntax} from './syntax/syntax'; import {grStructuredQuery} from './structured-query/structured-query'; import '../components/gr-sort-control/gr-sort-control'; -import '../components/gr-tab-swap/gr-tab-swap'; +import '../components/gr-sort-control/gr-extended-sort-control'; import '../components/gr-permissions-filter/gr-permissions-filter'; import '../components/gr-my-uploads/gr-my-uploads'; import { sendTelemetryForQuery } from '../services/telemetry'; @@ -34,7 +34,7 @@ export var query = angular.module('kahuna.search.query', [ grStructuredQuery.name, 'util.storage', 'gr.sortControl', - 'gr.tabSwapControl', + 'gr.extendedSortControl', 'gr.permissionsFilter', 'gr.myUploads' ]); @@ -188,8 +188,17 @@ query.controller('SearchQueryCtrl', [ // eslint-disable-next-line complexity function watchSearchChange(newFilter, sender) { - const showPaid = newFilter.nonFree ? newFilter.nonFree : false; + let showPaid = newFilter.nonFree ? newFilter.nonFree : false; + if (sender && sender == "filterChange" && !newFilter.nonFree) { + showPaid = ctrl.user.permissions.showPaid; + } + storage.setJs("isNonFree", showPaid, true); + + // check for taken date sort contradiction + if ($stateParams.orderBy && $stateParams.orderBy.includes("taken") && (!newFilter.query || !newFilter.query.includes("has:dateTaken"))) { + ctrl.ordering["orderBy"] = "newest"; + } let sortBy = ctrl.ordering["orderBy"] ? ctrl.ordering["orderBy"] : "newest"; storage.setJs("orderBy", sortBy); diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index dbb1c12a0c..bf87f027b6 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -131,9 +131,8 @@
-
- - +
+
    { ctrl.showSendToPhotoSales = () => $window._clientConfig.showSendToPhotoSales; + ctrl.usePermissionsFilter = () => $window._clientConfig.usePermissionsFilter; }; // Panel control @@ -126,84 +127,41 @@ results.controller('SearchResultsCtrl', [ var hasTakenDateClause = "has:dateTaken"; var noTakenDateClause = "-has:dateTaken"; var takenSort = "taken"; - ctrl.clearTakenVisible = () => storage.setJs("takenTabVisible", "hidden", true); - ctrl.setTakenVisible = () => storage.setJs("takenTabVisible", "visible", true); - ctrl.getTakenVisible = () => storage.getJs("takenTabVisible", true) ? storage.getJs("takenTabVisible", true) : "hidden"; + ctrl.setTakenVisible = (isVisible) => storage.setJs("takenTabVisible", isVisible ? "visible" : "hidden", true); + ctrl.getTakenVisible = () => { + const vis = storage.getJs("takenTabVisible", true) ? storage.getJs("takenTabVisible", true) : "hidden"; + return (vis == "visible"); + } ctrl.getCollectionsPanelVisible = () => storage.getJs("collectionsPanelState", false) ? !(storage.getJs("collectionsPanelState", false).hidden) : false; ctrl.getInfoPanelVisible = () => storage.getJs("metadataPanelState", false) ? !(storage.getJs("metadataPanelState", false).hidden) : false; ctrl.getLastTakenSort = () => storage.getJs("lastTakenSort", false) ? storage.getJs("lastTakenSort", false) : ""; + ctrl.setLastTakenSort = (orderBy) => storage.setJs("lastTakenSort", orderBy, false); - //-sort control- - function updateSortChips (sortSel) { - ctrl.sortProps.selectedOption = sortSel; + //-sort control select- + function updateSortChange (sortSel, tabSelected, userSelectedTaken, noTakenCount) { var orderBy = manageSortSelection(sortSel.value); - if (orderBy && orderBy.includes(takenSort)) { - storage.setJs("lastTakenSort", orderBy, false); + var curQuery = $stateParams.query ? $stateParams.query : ''; + ctrl.setTakenVisible(userSelectedTaken); + curQuery = curQuery.replace(noTakenDateClause, "").replace(hasTakenDateClause, "").trim(); + if (sortSel.isTaken) { + ctrl.setLastTakenSort(orderBy); } - let toParams = { - ...$stateParams, - orderBy: orderBy - }; - $state.transitionTo( - $state.current, - toParams, - { reload: true, inherit: false, notify: true } - ); - } - - ctrl.sortProps = { - options: SortOptions, - selectedOption: DefaultSortOption, - onSelect: updateSortChips, - query: $stateParams.query, - orderBy: $stateParams.orderBy, - previousTaken: ctrl.getLastTakenSort(), - clearTakenVisible: ctrl.clearTakenVisible, - panelVisible: ctrl.getInfoPanelVisible() - }; - //-end sort control- - - //-tab swap control- - function manageTakenDateTab(tabSelected, inSortOrder) { - let sortOrder = inSortOrder; - if (tabSelected === 'with') { - sortOrder = storage.getJs("lastTakenSort", false); - } - let toParams = { - ...$stateParams - }; - //-clean- - let oldQuery = $stateParams.query ? $stateParams.query : ''; - oldQuery = oldQuery.replace(noTakenDateClause, '').replace(hasTakenDateClause, '').trim(); - let newSortOrder = manageSortSelection(sortOrder); - let takenTabVisible = ctrl.getTakenVisible(); - - if (tabSelected === 'with') { - toParams = { - ...$stateParams, - query: `${oldQuery} ${hasTakenDateClause}`, - orderBy: newSortOrder - }; - } else { - if ((sortOrder && sortOrder.includes(takenSort)) || (takenTabVisible === 'visible')) { - toParams = { - ...$stateParams, - query: `${oldQuery} ${noTakenDateClause}`, - orderBy: newSortOrder - }; - } else { - toParams = { - ...$stateParams, - query: oldQuery, - orderBy: newSortOrder - }; + if (userSelectedTaken) { + if (tabSelected === 'with') { + curQuery = `${curQuery} ${hasTakenDateClause}`.trim(); + orderBy = ctrl.getLastTakenSort(); + } else { // without + curQuery = `${curQuery} ${noTakenDateClause}`.trim(); + orderBy = DefaultSortOption.value; } } - $state.transitionTo( - $state.current, - toParams, - { reload: true, inherit: false, notify: true } - ); + storage.setJs("orderBy", orderBy); + const toParams = { + ...$stateParams, + orderBy: orderBy, + query: curQuery + }; + $state.go('search.results', toParams); } async function checkForNoTakenDate() { @@ -217,24 +175,32 @@ results.controller('SearchResultsCtrl', [ return resp.total; }; - ctrl.tabSwapProps = { - orderBy: $stateParams.orderBy ? $stateParams.orderBy : '', - query: $stateParams.query ? $stateParams.query : '', - without: noTakenDateClause, - taken: takenSort, - onSelect: manageTakenDateTab, - takenVisible: ctrl.getTakenVisible(), - clearTakenVisible: ctrl.clearTakenVisible, - setTakenVisible: ctrl.setTakenVisible, - noTakenDateCount: 0, - panelVisible: ctrl.getCollectionsPanelVisible() + // selected sort option + var selSortOption = DefaultSortOption; + if ((SortOptions.filter(o => o.value === $stateParams.orderBy)).length > 0) { + selSortOption = SortOptions.filter(o => o.value === $stateParams.orderBy)[0]; + } + // check for without tab + if ($stateParams.query && $stateParams.query.includes(noTakenDateClause)) { + selSortOption = DefaultSortOption; + } + + ctrl.extendedSortProps = { + onSortSelect: updateSortChange, + query: $stateParams.query ? $stateParams.query : "", + orderBy: $stateParams.orderBy ? $stateParams.orderBy : "", + infoPanelVisible: ctrl.getInfoPanelVisible(), + collectionsPanelVisible: ctrl.getCollectionsPanelVisible(), + userTakenSelect: ctrl.getTakenVisible(), + noTakenDateCount: 0 }; + checkForNoTakenDate().then(noTakenTotal => { - ctrl.tabSwapProps = { ...ctrl.tabSwapProps, + ctrl.extendedSortProps = { ...ctrl.extendedSortProps, noTakenDateCount: noTakenTotal }; }); - //-end tab swap- + //-end sort and taken tab controls- ctrl.images = []; if (ctrl.image && ctrl.image.data.softDeletedMetadata !== undefined) { ctrl.isDeleted = true; } @@ -290,25 +256,49 @@ results.controller('SearchResultsCtrl', [ results.clear(); results.resize(totalLength); - if (ctrl.tabSwapProps.orderBy.includes(takenSort) && 0 < ctrl.tabSwapProps.noTakenDateCount) { - const notificationEvent = new CustomEvent("newNotification", { - detail: { - announceId: "sortByTakenDate", - description: "There are " + ctrl.tabSwapProps.noTakenDateCount + " images with no taken date", - category: "information", - lifespan: "transient" - }, - bubbles: true - }); - window.dispatchEvent(notificationEvent); - } else { - const notificationEvent = new CustomEvent("removeNotification", { - detail: { - announceId: "sortByTakenDate" - }, - bubbles: true - }); - window.dispatchEvent(notificationEvent); + if (ctrl.extendedSortProps.orderBy.includes(takenSort)) { + if (images.total === 0) { // no images with taken date + updateSortChange(DefaultSortOption, 'with', false, ctrl.extendedSortProps.noTakenDateCount); + const noMatchesStr = "There are no matching images with a taken date" + const notificationEvent = new CustomEvent("newNotification", { + detail: { + announceId: "noTakenDateImages", + description: noMatchesStr, + category: "information", + lifespan: "transient" + }, + bubbles: true + }); + window.dispatchEvent(notificationEvent); + } else if (0 < ctrl.extendedSortProps.noTakenDateCount) { + const oldNoTakenCount = storage.getJs("lastNoTakenCount", false) ? storage.getJs("lastNoTakenCount", false) : 0; + if (oldNoTakenCount !== ctrl.extendedSortProps.noTakenDateCount) { + let imageStr = "There are " + ctrl.extendedSortProps.noTakenDateCount.toLocaleString() + " images with no taken date"; + if (ctrl.extendedSortProps.noTakenDateCount === 1) { + imageStr = "There is one image with no taken date"; + } + const notificationEvent = new CustomEvent("newNotification", { + detail: { + announceId: "sortByTakenDate", + description: imageStr, + category: "information", + lifespan: "transient" + }, + bubbles: true + }); + window.dispatchEvent(notificationEvent); + storage.setJs("lastNoTakenCount", ctrl.extendedSortProps.noTakenDateCount, false); + } + } else { + const notificationEvent = new CustomEvent("removeNotification", { + detail: { + announceId: "sortByTakenDate" + }, + bubbles: true + }); + window.dispatchEvent(notificationEvent); + storage.setJs("lastNoTakenCount", 0, false); + } } imagesPositions = new Map(); diff --git a/kahuna/public/js/search/syntax/syntax.html b/kahuna/public/js/search/syntax/syntax.html index cb660cb6fe..779833e722 100644 --- a/kahuna/public/js/search/syntax/syntax.html +++ b/kahuna/public/js/search/syntax/syntax.html @@ -167,5 +167,13 @@

    Other filters

    Search by person (email) or ftp source (folder). +
    +
    + +
    +
    + Returns images that have a 'Taken date' value set. +
    +
diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index 96c533fc92..7a59bb0b40 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -1275,7 +1275,7 @@ textarea.ng-invalid { display: none; /* above thumbnail and overlay */ - z-index: 2; + z-index: 1; } .result__select--no-pointer-events { @@ -1536,7 +1536,7 @@ textarea.ng-invalid { .image-no-results { font-size: 3rem; text-align: center; - margin-top: 4rem; + margin-top: 10rem; } .image-loading-results { From 908cbbca15914da61faa2cea479cf60b2bbab4a0 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Wed, 13 Aug 2025 18:05:50 +0100 Subject: [PATCH 07/66] js syntax corrections --- .../gr-sort-control/base-sort-control.tsx | 3 +- .../gr-extended-sort-control.tsx | 14 +-- .../gr-sort-control/gr-sort-control.tsx | 2 +- .../js/components/gr-tab-swap/gr-tab-swap.tsx | 1 - kahuna/public/js/search/results.js | 114 +++++++++--------- 5 files changed, 63 insertions(+), 71 deletions(-) diff --git a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx index 5c118f6386..c54002d2e3 100644 --- a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx @@ -11,7 +11,6 @@ const CONTROL_TITLE = "Sort by:"; const SORT_ORDER = "Sort order"; const PANEL_IDENTIFIER = "info"; const SCROLL_IDENTIFIER = "scroll"; -const HAS_DATE_TAKEN_QUERY = "has:dateTaken"; const downArrowIcon = () => @@ -77,7 +76,7 @@ export const BaseSortControl: React.FC = ({ const startSort:SortDropdownOption = startSelectedOption ? startSelectedOption : options.filter(opt => opt.value == DEFAULT_OPTION)[0]; const [isOpen, setIsOpen] = useState(false); const [selectedOption, setSelection] = useState(startSort); - const [previousOption, setPrevious] = useState(startSort) + const [previousOption, setPrevious] = useState(startSort); const [currentIndex, setCurrentIndex] = useState(-1); const [isPanelVisible, setPanelVisible] = useState(panelVisible); diff --git a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx index b844193477..eb07ac98e9 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx @@ -9,7 +9,7 @@ import { TabControl, TabSwapProps } from "../gr-tab-swap/gr-tab-swap"; import "./gr-sort-control.css"; export interface ExtendedSortProps { - onSortSelect: (option: SortDropdownOption, tabSelected: string, userTakenSelect: boolean, noTakenDateCount: number) => void; + onSortSelect: (option: SortDropdownOption, tabSelected: string, userTakenSelect: boolean) => void; query?: string | ""; orderBy?: string | ""; infoPanelVisible?: boolean | false; @@ -46,7 +46,7 @@ const ExtendedSortControl: React.FC = ({ props }) => { const onSortSelect = (selOption: SortDropdownOption) => { setSortOption(selOption); setUserTakenSelect(selOption.isTaken); - props.onSortSelect(selOption, 'with', selOption.isTaken, noTakenDateCount); + props.onSortSelect(selOption, 'with', selOption.isTaken); }; const onTabSelect = (withTaken: boolean) => { @@ -55,22 +55,22 @@ const ExtendedSortControl: React.FC = ({ props }) => { withTakenStr = 'without'; setSortOption(DefaultSortOption); } - props.onSortSelect(selSortOption, withTakenStr, userTakenSelect, noTakenDateCount); - } + props.onSortSelect(selSortOption, withTakenStr, userTakenSelect); + }; // initialisation useEffect(() => { const handleLogoClick = (e: any) => { setSortOption(DefaultSortOption); setUserTakenSelect(false); - props.onSortSelect(DefaultSortOption, 'with', false, 0); + props.onSortSelect(DefaultSortOption, 'with', false); }; const handleQueryChange = (e: any) => { const newQuery = e.detail.query ? (" " + e.detail.query) : ""; setHasCollection(checkForCollection(newQuery)); if (userTakenSelect && !newQuery.includes(takenDateClause)) { - props.onSortSelect(DefaultSortOption, 'with', false, 0); + props.onSortSelect(DefaultSortOption, 'with', false); } }; @@ -103,7 +103,7 @@ const ExtendedSortControl: React.FC = ({ props }) => { /> ); -} +}; export const extendedSortControl = angular.module('gr.extendedSortControl', []) .component('extendedSortControl', react2angular(ExtendedSortControl, ["props"])); diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index cf725db4d3..87ccb4836a 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -17,7 +17,7 @@ const SortControl: React.FC = ({ props }) => { return ( ); -} +}; export const sortControl = angular.module('gr.sortControl', []) .component('sortControl', react2angular(SortControl, ["props"])); diff --git a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx index ec4f26e5b1..0f347e1876 100644 --- a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx +++ b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx @@ -19,7 +19,6 @@ export const TabControl: React.FC = ({ onSelect, query, showTakenT const PANEL_IDENTIFIER = "collections"; const SCROLL_IDENTIFIER = "scroll"; const HAS_DATE_TAKEN_QUERY = "has:dateTaken"; - const taken = "taken"; const without = `-${HAS_DATE_TAKEN_QUERY}`; let tabStart = 'with'; diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 844bfda6ff..32d86a647b 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -124,21 +124,21 @@ results.controller('SearchResultsCtrl', [ ctrl.collectionsPanel = panels.collectionsPanel; //-taken and sort controls- - var hasTakenDateClause = "has:dateTaken"; - var noTakenDateClause = "-has:dateTaken"; - var takenSort = "taken"; + const hasTakenDateClause = "has:dateTaken"; + const noTakenDateClause = "-has:dateTaken"; + const takenSort = "taken"; ctrl.setTakenVisible = (isVisible) => storage.setJs("takenTabVisible", isVisible ? "visible" : "hidden", true); ctrl.getTakenVisible = () => { const vis = storage.getJs("takenTabVisible", true) ? storage.getJs("takenTabVisible", true) : "hidden"; return (vis == "visible"); - } + }; ctrl.getCollectionsPanelVisible = () => storage.getJs("collectionsPanelState", false) ? !(storage.getJs("collectionsPanelState", false).hidden) : false; ctrl.getInfoPanelVisible = () => storage.getJs("metadataPanelState", false) ? !(storage.getJs("metadataPanelState", false).hidden) : false; ctrl.getLastTakenSort = () => storage.getJs("lastTakenSort", false) ? storage.getJs("lastTakenSort", false) : ""; ctrl.setLastTakenSort = (orderBy) => storage.setJs("lastTakenSort", orderBy, false); //-sort control select- - function updateSortChange (sortSel, tabSelected, userSelectedTaken, noTakenCount) { + function updateSortChange (sortSel, tabSelected, userSelectedTaken) { var orderBy = manageSortSelection(sortSel.value); var curQuery = $stateParams.query ? $stateParams.query : ''; ctrl.setTakenVisible(userSelectedTaken); @@ -175,16 +175,6 @@ results.controller('SearchResultsCtrl', [ return resp.total; }; - // selected sort option - var selSortOption = DefaultSortOption; - if ((SortOptions.filter(o => o.value === $stateParams.orderBy)).length > 0) { - selSortOption = SortOptions.filter(o => o.value === $stateParams.orderBy)[0]; - } - // check for without tab - if ($stateParams.query && $stateParams.query.includes(noTakenDateClause)) { - selSortOption = DefaultSortOption; - } - ctrl.extendedSortProps = { onSortSelect: updateSortChange, query: $stateParams.query ? $stateParams.query : "", @@ -236,7 +226,7 @@ results.controller('SearchResultsCtrl', [ // TODO: avoid this initial search (two API calls to init!) ctrl.searched = search({length: 1, orderBy: 'newest'}).then(function(images) { ctrl.totalResults = images.total; - // FIXME: https://github.com/argo-rest/theseus has forced us to co-opt the actions field for this + // FIXME: https://github.com/argo-rest/theseus has forced us to co-opt the actions field for this ctrl.orgOwnedCount = images.$response?.$$state?.value?.actions; ctrl.hasQuery = !!$stateParams.query; @@ -256,50 +246,7 @@ results.controller('SearchResultsCtrl', [ results.clear(); results.resize(totalLength); - if (ctrl.extendedSortProps.orderBy.includes(takenSort)) { - if (images.total === 0) { // no images with taken date - updateSortChange(DefaultSortOption, 'with', false, ctrl.extendedSortProps.noTakenDateCount); - const noMatchesStr = "There are no matching images with a taken date" - const notificationEvent = new CustomEvent("newNotification", { - detail: { - announceId: "noTakenDateImages", - description: noMatchesStr, - category: "information", - lifespan: "transient" - }, - bubbles: true - }); - window.dispatchEvent(notificationEvent); - } else if (0 < ctrl.extendedSortProps.noTakenDateCount) { - const oldNoTakenCount = storage.getJs("lastNoTakenCount", false) ? storage.getJs("lastNoTakenCount", false) : 0; - if (oldNoTakenCount !== ctrl.extendedSortProps.noTakenDateCount) { - let imageStr = "There are " + ctrl.extendedSortProps.noTakenDateCount.toLocaleString() + " images with no taken date"; - if (ctrl.extendedSortProps.noTakenDateCount === 1) { - imageStr = "There is one image with no taken date"; - } - const notificationEvent = new CustomEvent("newNotification", { - detail: { - announceId: "sortByTakenDate", - description: imageStr, - category: "information", - lifespan: "transient" - }, - bubbles: true - }); - window.dispatchEvent(notificationEvent); - storage.setJs("lastNoTakenCount", ctrl.extendedSortProps.noTakenDateCount, false); - } - } else { - const notificationEvent = new CustomEvent("removeNotification", { - detail: { - announceId: "sortByTakenDate" - }, - bubbles: true - }); - window.dispatchEvent(notificationEvent); - storage.setJs("lastNoTakenCount", 0, false); - } - } + notificationMessages(ctrl.extendedSortProps, images.total) imagesPositions = new Map(); @@ -375,6 +322,53 @@ results.controller('SearchResultsCtrl', [ const pollingPeriod = 15 * 1000; // ms + function notificationMessages(extendedProps, imagesTotal) { + if (extendedProps.orderBy.includes('taken')) { + if (imagesTotal === 0) { // no images with taken date + updateSortChange(DefaultSortOption, 'with', false, extendedProps.noTakenDateCount); + const noMatchesStr = "There are no matching images with a taken date"; + const notificationEvent = new CustomEvent("newNotification", { + detail: { + announceId: "noTakenDateImages", + description: noMatchesStr, + category: "information", + lifespan: "transient" + }, + bubbles: true + }); + window.dispatchEvent(notificationEvent); + } else if (0 < extendedProps.noTakenDateCount) { + const oldNoTakenCount = storage.getJs("lastNoTakenCount", false) ? storage.getJs("lastNoTakenCount", false) : 0; + if (oldNoTakenCount !== extendedProps.noTakenDateCount) { + let imageStr = "There are " + extendedProps.noTakenDateCount.toLocaleString() + " images with no taken date"; + if (extendedProps.noTakenDateCount === 1) { + imageStr = "There is one image with no taken date"; + } + const notificationEvent = new CustomEvent("newNotification", { + detail: { + announceId: "sortByTakenDate", + description: imageStr, + category: "information", + lifespan: "transient" + }, + bubbles: true + }); + window.dispatchEvent(notificationEvent); + storage.setJs("lastNoTakenCount", extendedProps.noTakenDateCount, false); + } + } else { + const notificationEvent = new CustomEvent("removeNotification", { + detail: { + announceId: "sortByTakenDate" + }, + bubbles: true + }); + window.dispatchEvent(notificationEvent); + storage.setJs("lastNoTakenCount", 0, false); + } + } + } + // FIXME: this will only add up to 50 images (search capped) function checkForNewImages() { $timeout(() => { From 5da1bd4b99cd91be831bd802863cc3553c85151e Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Wed, 13 Aug 2025 18:11:18 +0100 Subject: [PATCH 08/66] further js corrections --- kahuna/public/js/search/results.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 32d86a647b..9f50a94f49 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -23,8 +23,7 @@ import '../components/gr-sort-control/gr-sort-control'; import '../components/gr-sort-control/gr-extended-sort-control'; import { manageSortSelection, - DefaultSortOption, - SortOptions + DefaultSortOption } from "../components/gr-sort-control/gr-sort-control-config"; import { INVALIDIMAGES, @@ -246,7 +245,7 @@ results.controller('SearchResultsCtrl', [ results.clear(); results.resize(totalLength); - notificationMessages(ctrl.extendedSortProps, images.total) + notificationMessages(ctrl.extendedSortProps, images.total); imagesPositions = new Map(); @@ -339,11 +338,11 @@ results.controller('SearchResultsCtrl', [ window.dispatchEvent(notificationEvent); } else if (0 < extendedProps.noTakenDateCount) { const oldNoTakenCount = storage.getJs("lastNoTakenCount", false) ? storage.getJs("lastNoTakenCount", false) : 0; + let imageStr = "There are " + extendedProps.noTakenDateCount.toLocaleString() + " images with no taken date"; + if (extendedProps.noTakenDateCount === 1) { + imageStr = "There is one image with no taken date"; + } if (oldNoTakenCount !== extendedProps.noTakenDateCount) { - let imageStr = "There are " + extendedProps.noTakenDateCount.toLocaleString() + " images with no taken date"; - if (extendedProps.noTakenDateCount === 1) { - imageStr = "There is one image with no taken date"; - } const notificationEvent = new CustomEvent("newNotification", { detail: { announceId: "sortByTakenDate", From 09c17caa3684c20507b0cdb79d9688bead5ef8b4 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Wed, 13 Aug 2025 19:58:24 +0100 Subject: [PATCH 09/66] further js corrections - removing unused vars --- .../components/gr-sort-control/base-sort-control.tsx | 2 +- .../gr-sort-control/gr-extended-sort-control.tsx | 10 +++++----- .../public/js/components/gr-tab-swap/gr-tab-swap.tsx | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx index c54002d2e3..bd8836ebd7 100644 --- a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx @@ -72,7 +72,7 @@ export const BaseSortControl: React.FC = ({ panelVisible }) => { - const [hasCollection, setHasCollection] = useState(startHasCollection); + const hasCollection = startHasCollection; const startSort:SortDropdownOption = startSelectedOption ? startSelectedOption : options.filter(opt => opt.value == DEFAULT_OPTION)[0]; const [isOpen, setIsOpen] = useState(false); const [selectedOption, setSelection] = useState(startSort); diff --git a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx index eb07ac98e9..c6c7d2c2cf 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx @@ -1,10 +1,10 @@ import * as React from "react"; import * as angular from "angular"; import { react2angular } from "react2angular"; -import { useEffect, useState, KeyboardEvent } from "react"; -import { BaseSortControl, SortDropdownOption, SortDropdownProps } from "./base-sort-control"; -import { SortOptions, DefaultSortOption, CollectionSortOption } from "./gr-sort-control-config"; -import { TabControl, TabSwapProps } from "../gr-tab-swap/gr-tab-swap"; +import { useEffect, useState } from "react"; +import { BaseSortControl, SortDropdownOption } from "./base-sort-control"; +import { SortOptions, DefaultSortOption } from "./gr-sort-control-config"; +import { TabControl } from "../gr-tab-swap/gr-tab-swap"; import "./gr-sort-control.css"; @@ -40,7 +40,7 @@ const ExtendedSortControl: React.FC = ({ props }) => { const startHasCollection = checkForCollection(query); const [selSortOption, setSortOption] = useState(startSortOption); const [userTakenSelect, setUserTakenSelect] = useState(props.userTakenSelect); - const [noTakenDateCount, setNoTakenDateCount] = useState(props.noTakenDateCount); + const noTakenDateCount = props.noTakenDateCount; const [hasCollection, setHasCollection] = useState(startHasCollection); const onSortSelect = (selOption: SortDropdownOption) => { diff --git a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx index 0f347e1876..efbe9c1abf 100644 --- a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx +++ b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx @@ -27,7 +27,7 @@ export const TabControl: React.FC = ({ onSelect, query, showTakenT } const [activeTab, setActiveTab] = useState(tabStart); - const [isSortTaken, setIsSortTaken] = useState(showTakenTab); + const isSortTaken = showTakenTab; const [isPanelVisible, setIsPanelVisible] = useState(panelVisible); let takenDateMsg = ""; From 929de2d4b6c8e47dc89f4229b52beefaecda2e3c Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Thu, 21 Aug 2025 15:56:19 +0100 Subject: [PATCH 10/66] single sort control operation (no tab control) --- .../gr-sort-control/base-sort-control.tsx | 12 ++- .../gr-sort-control/gr-sort-control-config.ts | 4 +- .../gr-sort-control/gr-sort-control.tsx | 90 +++++++++++++++++-- kahuna/public/js/search/index.js | 9 +- kahuna/public/js/search/query.html | 7 ++ kahuna/public/js/search/query.js | 44 +++++---- kahuna/public/js/search/results.js | 3 + 7 files changed, 137 insertions(+), 32 deletions(-) diff --git a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx index bd8836ebd7..ed55bb55aa 100644 --- a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx @@ -47,6 +47,7 @@ export interface SortDropdownProps { onSelect: (option: SortDropdownOption) => void; startHasCollection?: boolean | false; panelVisible?: boolean | false; + isSimple?: boolean | false; } const hasClassInSelfOrParent = (node: Element | null, className: string): boolean => { @@ -69,7 +70,8 @@ export const BaseSortControl: React.FC = ({ startSelectedOption, onSelect, startHasCollection, - panelVisible + panelVisible, + isSimple }) => { const hasCollection = startHasCollection; @@ -80,6 +82,10 @@ export const BaseSortControl: React.FC = ({ const [currentIndex, setCurrentIndex] = useState(-1); const [isPanelVisible, setPanelVisible] = useState(panelVisible); + if (startSort.value !== selectedOption.value) { + setSelection(startSort); + } + const handleArrowKeys = (event:KeyboardEvent) => { if (event.key === 'ArrowDown' || event.key === 'ArrowUp' || @@ -162,7 +168,7 @@ export const BaseSortControl: React.FC = ({ }; return ( -
+
{CONTROL_TITLE}
setIsOpen(!isOpen)}> @@ -178,7 +184,7 @@ export const BaseSortControl: React.FC = ({
{isOpen && ( - +
{options.map((option) => (hasCollection || option.value != COLLECTION_OPTION) && ( -1 && options[currentIndex].value) === option.value ? "sort-dropdown-item sort-dropdown-highlight" : "sort-dropdown-item"} diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts index fda54bf9d4..4d29ed8a7f 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts @@ -3,7 +3,7 @@ import {SortDropdownOption} from "./gr-sort-control"; export function manageSortSelection(newSelection:string): string { let newVal; switch (newSelection) { - case "uploadNewOld": + case "newest": newVal = "newest"; break; case "oldest": @@ -27,7 +27,7 @@ export function manageSortSelection(newSelection:string): string { export const SortOptions: SortDropdownOption[] = [ { - value: "uploadNewOld", + value: "newest", label: "Upload date (new to old)", isCollection: false, isTaken: false diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index 87ccb4836a..653b42a2ae 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -1,22 +1,94 @@ import * as React from "react"; import * as angular from "angular"; import { react2angular } from "react2angular"; -import { BaseSortControl, SortDropdownOption, SortDropdownProps } from "./base-sort-control"; +import { useEffect, useState } from "react"; +import { BaseSortControl, SortDropdownOption } from "./base-sort-control"; +import { SortOptions, DefaultSortOption } from "./gr-sort-control-config"; export type { SortDropdownOption }; -export type { SortDropdownProps }; +export interface SortProps { + onSortSelect: (option: SortDropdownOption) => void; + query?: string | ""; + orderBy?: string | ""; +} + export interface SortWrapperProps { - props: SortDropdownProps; + props: SortProps; } +const checkForCollection = (query:string): boolean => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); + const SortControl: React.FC = ({ props }) => { - const sortProps = { ...props, - simpleDisplay: true - }; - return ( - - ); + const sortOptions = SortOptions; + const orderBy = props.orderBy; + const query = props.query; + const startHasCollection = checkForCollection(query); + + let startSortOption = DefaultSortOption; + if (startHasCollection) { + if ((sortOptions.filter(o => o.isCollection)).length > 0) { + startSortOption = sortOptions.filter(o => o.isCollection)[0]; + } + } else { + if ((sortOptions.filter(o => o.value === orderBy)).length > 0) { + startSortOption = sortOptions.filter(o => o.value === orderBy)[0]; + } + } + + const [selSortOption, setSortOption] = useState(startSortOption); + const [hasCollection, setHasCollection] = useState(startHasCollection); + + const onSortSelect = (selOption: SortDropdownOption) => { + setSortOption(selOption); + props.onSortSelect(selOption); + }; + + // initialisation + useEffect(() => { + const handleLogoClick = (e: any) => { + setSortOption(DefaultSortOption); + props.onSortSelect(DefaultSortOption); + }; + + const handleQueryChange = (e: any) => { + const newQuery = e.detail.query ? (" " + e.detail.query) : ""; + const curHasCollec = e.detail.hasCollection ? e.detail.hasCollection : false; + const newHasCollec = checkForCollection(newQuery); + setHasCollection(newHasCollec); + if (!curHasCollec && newHasCollec) { + let collecSortOption = DefaultSortOption; + if ((sortOptions.filter(o => o.isCollection)).length > 0) { + collecSortOption = sortOptions.filter(o => o.isCollection)[0]; + } + setSortOption(collecSortOption); + } + if(!newHasCollec && curHasCollec && selSortOption.isCollection) { + setSortOption(DefaultSortOption); + } + }; + + window.addEventListener("logoClick", handleLogoClick); + window.addEventListener("queryChangeEvent", handleQueryChange); + + // Clean up the event listener when the component unmounts + return () => { + window.removeEventListener("logoClick", handleLogoClick); + window.removeEventListener("queryChangeEvent", handleQueryChange); + }; + + }, []); + + return ( + + ); }; export const sortControl = angular.module('gr.sortControl', []) diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index b6da170661..33926abdce 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -330,10 +330,15 @@ search.run(['$rootScope', '$state', function($rootScope, $state) { if (toState.name === 'search.results') { //If moving to a collection, sorts images by time added to a collection by default //allows sorting by newest first if set by user. Need to account for 'With Taken Date' tab impacts on query - if (toParams.query && toParams.query.indexOf('~') === 0) { + const checkForCollection = (query) => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); + if (toParams.query && checkForCollection(toParams.query)) { const toQuery = toParams.query ? toParams.query.replace('-has:dateTaken', '').replace('has:dateTaken', '').trim() : ""; + const toHasTaken = toParams.query ? toParams.query.includes('has:dateTaken') : false; const fromQuery = fromParams.query ? fromParams.query.replace('-has:dateTaken', '').replace('has:dateTaken', '').trim() : ""; - toParams.orderBy = (toQuery === fromQuery) ? toParams.orderBy : 'dateAddedToCollection'; + const fromHasTaken = fromParams.query ? fromParams.query.includes('has:dateTaken') : false; + if (fromHasTaken || toHasTaken) { + toParams.orderBy = (toQuery === fromQuery) ? toParams.orderBy : 'dateAddedToCollection'; + } } //If moving from a collection to a non-collection, reset order to default. else if (toParams.orderBy === 'dateAddedToCollection') { diff --git a/kahuna/public/js/search/query.html b/kahuna/public/js/search/query.html index e054fac050..33108efb37 100644 --- a/kahuna/public/js/search/query.html +++ b/kahuna/public/js/search/query.html @@ -83,6 +83,13 @@ + +
+
+ +
+
+ diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 9bd0c1cf9d..a0cbdcb71c 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -87,9 +87,9 @@ query.controller('SearchQueryCtrl', [ window.dispatchEvent(customEvent); } - function raiseQueryChangeEvent(query) { + function raiseQueryChangeEvent(query, prevHasCollec) { const customEvent = new CustomEvent('queryChangeEvent', { - detail: {query: query}, + detail: {query: query, hasCollection: prevHasCollec}, bubbles: true }); window.dispatchEvent(customEvent); @@ -186,23 +186,28 @@ query.controller('SearchQueryCtrl', [ ctrl.filter.orgOwned = false; } + function checkForCollection(query) { + return /~"[a-zA-Z0-9 #-_.://]+"/.test(query) + }; + // eslint-disable-next-line complexity function watchSearchChange(newFilter, sender) { let showPaid = newFilter.nonFree ? newFilter.nonFree : false; if (sender && sender == "filterChange" && !newFilter.nonFree) { showPaid = ctrl.user.permissions.showPaid; } - storage.setJs("isNonFree", showPaid, true); // check for taken date sort contradiction - if ($stateParams.orderBy && $stateParams.orderBy.includes("taken") && (!newFilter.query || !newFilter.query.includes("has:dateTaken"))) { + const oldOrderBy = storage.getJs("orderBy"); + if (ctrl.usePermissionsFilter && $stateParams.orderBy && $stateParams.orderBy.includes("taken") && (!newFilter.query || !newFilter.query.includes("has:dateTaken"))) { ctrl.ordering["orderBy"] = "newest"; } let sortBy = ctrl.ordering["orderBy"] ? ctrl.ordering["orderBy"] : "newest"; storage.setJs("orderBy", sortBy); - ctrl.collectionSearch = newFilter.query ? newFilter.query.indexOf('~') === 0 : false; + const curCollectionSearch = ctrl.collectionSearch; + ctrl.collectionSearch = newFilter.query ? checkForCollection(newFilter.query) : false; //--update filter elements-- manageUploadedBy(newFilter, sender); @@ -218,10 +223,18 @@ query.controller('SearchQueryCtrl', [ nonFreeCheck = undefined; } ctrl.filter.nonFree = nonFreeCheck; - raiseQueryChangeEvent(ctrl.filter.query); + raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch); sendTelemetryForQuery(ctrl.filter.query, nonFreeCheck, uploadedByMe); - $state.go('search.results', ctrl.filter); + console.log(" ==> Old OrderBy=" + oldOrderBy + ", Current OrderBy=" + ctrl.ordering["orderBy"] + ", State OrderBy=" + $stateParams.orderBy); + console.log(" ==> curCollectionSearch=" + curCollectionSearch + ", Control Collection Search=" + ctrl.collectionSearch); + if (ctrl.collectionSearch && !curCollectionSearch) { + storage.setJs("orderBy", 'dateAddedToCollection'); + ctrl.ordering["orderBy"] = 'dateAddedToCollection'; + $state.go('search.results', {...ctrl.filter, ...{orderBy: 'dateAddedToCollection'}}); + } else { + $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering["orderBy"]}}); + } } //-my uploads- @@ -238,15 +251,13 @@ query.controller('SearchQueryCtrl', [ //-sort control- function updateSortChips (sortSel) { - ctrl.sortProps.selectedOption = sortSel; ctrl.ordering['orderBy'] = manageSortSelection(sortSel.value); - watchSearchChange(ctrl.filter, "sorting"); + storage.setJs("orderBy", ctrl.ordering["orderBy"]); + $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering['orderBy']}}); } ctrl.sortProps = { - options: SortOptions, - selectedOption: DefaultSortOption, - onSelect: updateSortChips, + onSortSelect: updateSortChips, query: ctrl.filter.query, orderBy: ctrl.ordering ? ctrl.ordering.orderBy : "" }; @@ -329,14 +340,15 @@ query.controller('SearchQueryCtrl', [ ctrl.filter[key] = valOrUndefined($stateParams[key]); } - ctrl.collectionSearch = ctrl.filter.query ? ctrl.filter.query.indexOf('~') === 0 : false; + ctrl.collectionSearch = ctrl.filter.query ? checkForCollection(ctrl.filter.query) : false; $scope.$watch(() => $stateParams[key], onValChange(newVal => { // FIXME: broken for 'your uploads' // FIXME: + they triggers filter $watch and $state.go (breaks history) - if (key === 'orderBy') { - ctrl.ordering[key] = valOrUndefined(newVal); - } else { + //if (key === 'orderBy') { + // ctrl.ordering[key] = valOrUndefined(newVal); + //} else { + if (key !== 'orderBy') { ctrl.filter[key] = valOrUndefined(newVal); } diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 9f50a94f49..6c7360f78f 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -322,6 +322,9 @@ results.controller('SearchResultsCtrl', [ const pollingPeriod = 15 * 1000; // ms function notificationMessages(extendedProps, imagesTotal) { + if (!ctrl.usePermissionsFilter()) { + return; + } if (extendedProps.orderBy.includes('taken')) { if (imagesTotal === 0) { // no images with taken date updateSortChange(DefaultSortOption, 'with', false, extendedProps.noTakenDateCount); From 3db400194fdb435106b3c718a6c8c41807fc7c7e Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Thu, 21 Aug 2025 17:50:18 +0100 Subject: [PATCH 11/66] resolve css and collections issues in single control config --- .../gr-sort-control/gr-sort-control-config.ts | 3 +++ .../gr-sort-control/gr-sort-control.tsx | 4 +++- kahuna/public/js/search/index.js | 20 +++++++------------ kahuna/public/js/search/query.js | 9 ++++----- kahuna/public/js/search/results.html | 5 +++-- kahuna/public/stylesheets/main.css | 7 +++++++ 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts index 4d29ed8a7f..631ff72905 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts @@ -60,3 +60,6 @@ export const SortOptions: SortDropdownOption[] = [ export const DefaultSortOption: SortDropdownOption = SortOptions[0]; export const CollectionSortOption: SortDropdownOption = SortOptions[4]; +export const HAS_DATE_TAKEN = "has:dateTaken"; +export const HASNT_DATE_TAKEN = "-has:dateTaken"; +export const COLLECTION_SORT_VALUE = SortOptions[4].value; diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index 653b42a2ae..717d744a62 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -54,6 +54,7 @@ const SortControl: React.FC = ({ props }) => { const handleQueryChange = (e: any) => { const newQuery = e.detail.query ? (" " + e.detail.query) : ""; const curHasCollec = e.detail.hasCollection ? e.detail.hasCollection : false; + const orderBy = e.detail.orderBy ? e.detail.orderBy : DefaultSortOption.value; const newHasCollec = checkForCollection(newQuery); setHasCollection(newHasCollec); if (!curHasCollec && newHasCollec) { @@ -63,7 +64,8 @@ const SortControl: React.FC = ({ props }) => { } setSortOption(collecSortOption); } - if(!newHasCollec && curHasCollec && selSortOption.isCollection) { + const eventOrderOpt = (sortOptions.filter(o => o.value == orderBy))[0]; + if (!newHasCollec && curHasCollec && eventOrderOpt.isCollection) { setSortOption(DefaultSortOption); } }; diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 33926abdce..bbfb2e62b2 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -30,7 +30,7 @@ import panelTemplate from '../components/gr-info-panel/gr-info-panel.html import collectionsPanelTemplate from '../components/gr-collections-panel/gr-collections-panel.html'; import {cropUtil} from '../util/crop'; - +import { HAS_DATE_TAKEN, HASNT_DATE_TAKEN, COLLECTION_SORT_VALUE } from '../components/gr-sort-control/gr-sort-control-config'; export var search = angular.module('kahuna.search', [ 'ct.ui.router.extras.dsr', @@ -332,24 +332,18 @@ search.run(['$rootScope', '$state', function($rootScope, $state) { //allows sorting by newest first if set by user. Need to account for 'With Taken Date' tab impacts on query const checkForCollection = (query) => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); if (toParams.query && checkForCollection(toParams.query)) { - const toQuery = toParams.query ? toParams.query.replace('-has:dateTaken', '').replace('has:dateTaken', '').trim() : ""; - const toHasTaken = toParams.query ? toParams.query.includes('has:dateTaken') : false; - const fromQuery = fromParams.query ? fromParams.query.replace('-has:dateTaken', '').replace('has:dateTaken', '').trim() : ""; - const fromHasTaken = fromParams.query ? fromParams.query.includes('has:dateTaken') : false; + const toQuery = toParams.query ? toParams.query.replace(HASNT_DATE_TAKEN, '').replace(HAS_DATE_TAKEN, '').trim() : ""; + const toHasTaken = toParams.query ? toParams.query.includes(HAS_DATE_TAKEN) : false; + const fromQuery = fromParams.query ? fromParams.query.replace(HASNT_DATE_TAKEN, '').replace(HAS_DATE_TAKEN, '').trim() : ""; + const fromHasTaken = fromParams.query ? fromParams.query.includes(HAS_DATE_TAKEN) : false; if (fromHasTaken || toHasTaken) { - toParams.orderBy = (toQuery === fromQuery) ? toParams.orderBy : 'dateAddedToCollection'; + toParams.orderBy = (toQuery === fromQuery) ? toParams.orderBy : COLLECTION_SORT_VALUE; } } //If moving from a collection to a non-collection, reset order to default. - else if (toParams.orderBy === 'dateAddedToCollection') { + else if (toParams.orderBy === COLLECTION_SORT_VALUE) { delete toParams.orderBy; } - - // handle clear hasTaken chip from search - //if ( (toParams.orderBy && toParams.orderBy.includes('taken')) && - // (!toParams.query || !toParams.query.includes('has:dateTaken')) ) { - // delete toParams.orderBy; - //} } }); }]); diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index a0cbdcb71c..67d69658a8 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -87,9 +87,9 @@ query.controller('SearchQueryCtrl', [ window.dispatchEvent(customEvent); } - function raiseQueryChangeEvent(query, prevHasCollec) { + function raiseQueryChangeEvent(query, prevHasCollec, orderBy) { const customEvent = new CustomEvent('queryChangeEvent', { - detail: {query: query, hasCollection: prevHasCollec}, + detail: {query: query, hasCollection: prevHasCollec, orderBy: orderBy}, bubbles: true }); window.dispatchEvent(customEvent); @@ -223,16 +223,15 @@ query.controller('SearchQueryCtrl', [ nonFreeCheck = undefined; } ctrl.filter.nonFree = nonFreeCheck; - raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch); sendTelemetryForQuery(ctrl.filter.query, nonFreeCheck, uploadedByMe); - console.log(" ==> Old OrderBy=" + oldOrderBy + ", Current OrderBy=" + ctrl.ordering["orderBy"] + ", State OrderBy=" + $stateParams.orderBy); - console.log(" ==> curCollectionSearch=" + curCollectionSearch + ", Control Collection Search=" + ctrl.collectionSearch); if (ctrl.collectionSearch && !curCollectionSearch) { storage.setJs("orderBy", 'dateAddedToCollection'); ctrl.ordering["orderBy"] = 'dateAddedToCollection'; + raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, 'dateAddedToCollection'); $state.go('search.results', {...ctrl.filter, ...{orderBy: 'dateAddedToCollection'}}); } else { + raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, ctrl.ordering["orderBy"]); $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering["orderBy"]}}); } } diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index bf87f027b6..491e781e8e 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -135,7 +135,8 @@ -
-
Too many results to display
Please refine your search to limit the number of results
diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index 7a59bb0b40..43112ce34f 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -1230,6 +1230,13 @@ textarea.ng-invalid { ========================================================================== */ .results { + display: flex; + flex-wrap: wrap; + top: 35px; + position: relative; +} + +.results--alternate { display: flex; flex-wrap: wrap; top: 96px; From d234588551264692e27c16acb874f67fa1778848 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Sat, 23 Aug 2025 21:45:04 +0100 Subject: [PATCH 12/66] fixing dual control issues --- kahuna/public/js/search/query.js | 9 +++++++-- kahuna/public/stylesheets/main.css | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 67d69658a8..8945cd338e 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -200,8 +200,13 @@ query.controller('SearchQueryCtrl', [ // check for taken date sort contradiction const oldOrderBy = storage.getJs("orderBy"); - if (ctrl.usePermissionsFilter && $stateParams.orderBy && $stateParams.orderBy.includes("taken") && (!newFilter.query || !newFilter.query.includes("has:dateTaken"))) { - ctrl.ordering["orderBy"] = "newest"; + if (ctrl.usePermissionsFilter) { + if (sender && sender == "filterChange" && ctrl.ordering["orderBy"] != $stateParams.orderBy) { + ctrl.ordering["orderBy"] = $stateParams.orderBy; + } + if ($stateParams.orderBy && $stateParams.orderBy.includes("taken") && (!newFilter.query || !newFilter.query.includes("has:dateTaken"))) { + ctrl.ordering["orderBy"] = "newest"; + } } let sortBy = ctrl.ordering["orderBy"] ? ctrl.ordering["orderBy"] : "newest"; storage.setJs("orderBy", sortBy); diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index 43112ce34f..fcd87515b0 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -1232,7 +1232,7 @@ textarea.ng-invalid { .results { display: flex; flex-wrap: wrap; - top: 35px; + top: 96px; position: relative; } @@ -3079,7 +3079,7 @@ FIXME: what to do with touch devices text-align: left; /* above thumbnail and overlay */ - z-index: 2; + z-index: 1; } .result__select__overlay__text.alert { background-color: red; From 324caacf8d094f7b4950336503a02ea9fec8f0e6 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Wed, 27 Aug 2025 20:19:28 +0100 Subject: [PATCH 13/66] fixing BBC config bugs --- .../gr-sort-control/gr-sort-control-config.ts | 1 + .../js/components/gr-tab-swap/gr-tab-swap.tsx | 2 ++ kahuna/public/js/search/index.js | 8 +++--- kahuna/public/js/search/query.js | 26 ++++++++++++------- kahuna/public/js/search/results.js | 20 +++++++++----- 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts index 631ff72905..4c85c46be6 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control-config.ts @@ -62,4 +62,5 @@ export const DefaultSortOption: SortDropdownOption = SortOptions[0]; export const CollectionSortOption: SortDropdownOption = SortOptions[4]; export const HAS_DATE_TAKEN = "has:dateTaken"; export const HASNT_DATE_TAKEN = "-has:dateTaken"; +export const TAKEN_SORT = "taken"; export const COLLECTION_SORT_VALUE = SortOptions[4].value; diff --git a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx index efbe9c1abf..bb4a63de43 100644 --- a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx +++ b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx @@ -11,6 +11,8 @@ export interface TabSwapProps { panelVisible: boolean; } +export const TAB_WITH: string = "with"; + export const TabControl: React.FC = ({ onSelect, query, showTakenTab, noTakenDateCount, panelVisible }) => { const withLabel = "With taken date"; diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index bbfb2e62b2..24993e908a 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -331,12 +331,12 @@ search.run(['$rootScope', '$state', function($rootScope, $state) { //If moving to a collection, sorts images by time added to a collection by default //allows sorting by newest first if set by user. Need to account for 'With Taken Date' tab impacts on query const checkForCollection = (query) => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); - if (toParams.query && checkForCollection(toParams.query)) { - const toQuery = toParams.query ? toParams.query.replace(HASNT_DATE_TAKEN, '').replace(HAS_DATE_TAKEN, '').trim() : ""; - const toHasTaken = toParams.query ? toParams.query.includes(HAS_DATE_TAKEN) : false; + const toQuery = toParams.query ? toParams.query.replace(HASNT_DATE_TAKEN, '').replace(HAS_DATE_TAKEN, '').trim() : ""; + if (checkForCollection(toQuery)) { const fromQuery = fromParams.query ? fromParams.query.replace(HASNT_DATE_TAKEN, '').replace(HAS_DATE_TAKEN, '').trim() : ""; + const toHasTaken = toParams.query ? toParams.query.includes(HAS_DATE_TAKEN) : false; const fromHasTaken = fromParams.query ? fromParams.query.includes(HAS_DATE_TAKEN) : false; - if (fromHasTaken || toHasTaken) { + if ((fromHasTaken || toHasTaken) && !checkForCollection(fromQuery)) { toParams.orderBy = (toQuery === fromQuery) ? toParams.orderBy : COLLECTION_SORT_VALUE; } } diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 8945cd338e..308d8de672 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -22,6 +22,9 @@ import {updateFilterChips} from "../components/gr-permissions-filter/gr-permissi import { manageSortSelection, DefaultSortOption, + CollectionSortOption, + HAS_DATE_TAKEN, + TAKEN_SORT, SortOptions } from "../components/gr-sort-control/gr-sort-control-config"; @@ -200,20 +203,23 @@ query.controller('SearchQueryCtrl', [ // check for taken date sort contradiction const oldOrderBy = storage.getJs("orderBy"); + const curCollectionSearch = ctrl.collectionSearch; + ctrl.collectionSearch = newFilter.query ? checkForCollection(newFilter.query) : false; if (ctrl.usePermissionsFilter) { if (sender && sender == "filterChange" && ctrl.ordering["orderBy"] != $stateParams.orderBy) { ctrl.ordering["orderBy"] = $stateParams.orderBy; } - if ($stateParams.orderBy && $stateParams.orderBy.includes("taken") && (!newFilter.query || !newFilter.query.includes("has:dateTaken"))) { - ctrl.ordering["orderBy"] = "newest"; + if ($stateParams.orderBy && $stateParams.orderBy.includes(TAKEN_SORT) && (!newFilter.query || !newFilter.query.includes(HAS_DATE_TAKEN))) { + if (ctrl.collectionSearch) { + ctrl.ordering["orderBy"] = CollectionSortOption.value; + } else { + ctrl.ordering["orderBy"] = DefaultSortOption.value; + } } } - let sortBy = ctrl.ordering["orderBy"] ? ctrl.ordering["orderBy"] : "newest"; + let sortBy = ctrl.ordering["orderBy"] ? ctrl.ordering["orderBy"] : DefaultSortOption.value; storage.setJs("orderBy", sortBy); - const curCollectionSearch = ctrl.collectionSearch; - ctrl.collectionSearch = newFilter.query ? checkForCollection(newFilter.query) : false; - //--update filter elements-- manageUploadedBy(newFilter, sender); manageDefaultNonFree(newFilter); @@ -231,10 +237,10 @@ query.controller('SearchQueryCtrl', [ sendTelemetryForQuery(ctrl.filter.query, nonFreeCheck, uploadedByMe); if (ctrl.collectionSearch && !curCollectionSearch) { - storage.setJs("orderBy", 'dateAddedToCollection'); - ctrl.ordering["orderBy"] = 'dateAddedToCollection'; - raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, 'dateAddedToCollection'); - $state.go('search.results', {...ctrl.filter, ...{orderBy: 'dateAddedToCollection'}}); + storage.setJs("orderBy", CollectionSortOption.value); + ctrl.ordering["orderBy"] = CollectionSortOption.value; + raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, CollectionSortOption.value); + $state.go('search.results', {...ctrl.filter, ...{orderBy: CollectionSortOption.value}}); } else { raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, ctrl.ordering["orderBy"]); $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering["orderBy"]}}); diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 6c7360f78f..a09d18bfd6 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -23,8 +23,14 @@ import '../components/gr-sort-control/gr-sort-control'; import '../components/gr-sort-control/gr-extended-sort-control'; import { manageSortSelection, - DefaultSortOption + DefaultSortOption, + TAKEN_SORT, + HAS_DATE_TAKEN, + HASNT_DATE_TAKEN } from "../components/gr-sort-control/gr-sort-control-config"; +import { + TAB_WITH +} from "../components/gr-tab-swap/gr-tab-swap"; import { INVALIDIMAGES, sendToCaptureAllValid, sendToCaptureCancelBtnTxt, sendToCaptureConfirmBtnTxt, sendToCaptureInvalid, @@ -123,9 +129,9 @@ results.controller('SearchResultsCtrl', [ ctrl.collectionsPanel = panels.collectionsPanel; //-taken and sort controls- - const hasTakenDateClause = "has:dateTaken"; - const noTakenDateClause = "-has:dateTaken"; - const takenSort = "taken"; + const hasTakenDateClause = HAS_DATE_TAKEN; + const noTakenDateClause = HASNT_DATE_TAKEN; + const takenSort = TAKEN_SORT; ctrl.setTakenVisible = (isVisible) => storage.setJs("takenTabVisible", isVisible ? "visible" : "hidden", true); ctrl.getTakenVisible = () => { const vis = storage.getJs("takenTabVisible", true) ? storage.getJs("takenTabVisible", true) : "hidden"; @@ -146,7 +152,7 @@ results.controller('SearchResultsCtrl', [ ctrl.setLastTakenSort(orderBy); } if (userSelectedTaken) { - if (tabSelected === 'with') { + if (tabSelected === TAB_WITH) { curQuery = `${curQuery} ${hasTakenDateClause}`.trim(); orderBy = ctrl.getLastTakenSort(); } else { // without @@ -325,8 +331,8 @@ results.controller('SearchResultsCtrl', [ if (!ctrl.usePermissionsFilter()) { return; } - if (extendedProps.orderBy.includes('taken')) { - if (imagesTotal === 0) { // no images with taken date + if (extendedProps.orderBy.includes(takenSort) && extendedProps.query.includes(hasTakenDateClause)) { + if (imagesTotal === 0 && extendedProps.noTakenDateCount > 0) { // no images with taken date updateSortChange(DefaultSortOption, 'with', false, extendedProps.noTakenDateCount); const noMatchesStr = "There are no matching images with a taken date"; const notificationEvent = new CustomEvent("newNotification", { From 824aa5465ee4583948cc18c53c2e0f93c78ed6a0 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Thu, 28 Aug 2025 10:29:19 +0100 Subject: [PATCH 14/66] correcting GNM config css --- kahuna/public/stylesheets/main.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index fcd87515b0..8a4c73ea04 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -1232,7 +1232,7 @@ textarea.ng-invalid { .results { display: flex; flex-wrap: wrap; - top: 96px; + top: 35px; position: relative; } From de3a5e7581684e00f42c2d9174f0c3a3c1c210a0 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Mon, 1 Sep 2025 11:51:22 +0100 Subject: [PATCH 15/66] correcting bug in order by after filter change --- kahuna/public/js/search/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 308d8de672..c6e7799b4b 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -206,7 +206,7 @@ query.controller('SearchQueryCtrl', [ const curCollectionSearch = ctrl.collectionSearch; ctrl.collectionSearch = newFilter.query ? checkForCollection(newFilter.query) : false; if (ctrl.usePermissionsFilter) { - if (sender && sender == "filterChange" && ctrl.ordering["orderBy"] != $stateParams.orderBy) { + if (sender && ctrl.ordering["orderBy"] != $stateParams.orderBy) { ctrl.ordering["orderBy"] = $stateParams.orderBy; } if ($stateParams.orderBy && $stateParams.orderBy.includes(TAKEN_SORT) && (!newFilter.query || !newFilter.query.includes(HAS_DATE_TAKEN))) { From 6a7e827a76ae5bb034b7d55f7b1eafca7a0a2f8f Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Tue, 2 Sep 2025 10:26:35 +0100 Subject: [PATCH 16/66] enabling correct order by reset on collection change --- kahuna/public/js/search/query.js | 39 +++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index c6e7799b4b..016756da06 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -193,6 +193,27 @@ query.controller('SearchQueryCtrl', [ return /~"[a-zA-Z0-9 #-_.://]+"/.test(query) }; + function storeCollection(query) { + const match = query ? query.match(/~"[a-zA-Z0-9 #-_.://]+"/) : undefined; + const collection = match ? match[0] : ""; + storage.setJs("currentCollection", collection); + return collection; + } + + function getCollection() { + const collection = storage.getJs("currentCollection") ? storage.getJs("currentCollection") : ""; + return collection; + } + + function getPiorOrderBy() { + const prior = storage.getJs("priorOrderBy") ? storage.getJs("priorOrderBy") : ""; + return prior; + } + + function setPriorOrderBy(priorOrderBy) { + storage.setJs("priorOrderBy", priorOrderBy); + } + // eslint-disable-next-line complexity function watchSearchChange(newFilter, sender) { let showPaid = newFilter.nonFree ? newFilter.nonFree : false; @@ -205,6 +226,9 @@ query.controller('SearchQueryCtrl', [ const oldOrderBy = storage.getJs("orderBy"); const curCollectionSearch = ctrl.collectionSearch; ctrl.collectionSearch = newFilter.query ? checkForCollection(newFilter.query) : false; + const oldCollection = getCollection(); + const newCollection = storeCollection(newFilter.query); + if (ctrl.usePermissionsFilter) { if (sender && ctrl.ordering["orderBy"] != $stateParams.orderBy) { ctrl.ordering["orderBy"] = $stateParams.orderBy; @@ -215,6 +239,17 @@ query.controller('SearchQueryCtrl', [ } else { ctrl.ordering["orderBy"] = DefaultSortOption.value; } + } else { + const priorOrderBy = getPiorOrderBy(); + if (ctrl.collectionSearch && ((oldCollection !== newCollection) || ("" !== priorOrderBy))) { + if (priorOrderBy != "") { + ctrl.ordering["orderBy"] = priorOrderBy; + setPriorOrderBy(""); + } else { + //console.log("Old collection="+oldCollection + ", New collection=" + newCollection + ", PriorOrderBy=" +priorOrderBy); + setPriorOrderBy(CollectionSortOption.value); + } + } } } let sortBy = ctrl.ordering["orderBy"] ? ctrl.ordering["orderBy"] : DefaultSortOption.value; @@ -351,13 +386,11 @@ query.controller('SearchQueryCtrl', [ } ctrl.collectionSearch = ctrl.filter.query ? checkForCollection(ctrl.filter.query) : false; + storeCollection(ctrl.filter.query); $scope.$watch(() => $stateParams[key], onValChange(newVal => { // FIXME: broken for 'your uploads' // FIXME: + they triggers filter $watch and $state.go (breaks history) - //if (key === 'orderBy') { - // ctrl.ordering[key] = valOrUndefined(newVal); - //} else { if (key !== 'orderBy') { ctrl.filter[key] = valOrUndefined(newVal); } From 5eed78ccc210adf31833fa835f58eb1526e30d6c Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Tue, 2 Sep 2025 15:45:59 +0100 Subject: [PATCH 17/66] correcting issue when page refresh mis-sets order by --- kahuna/public/js/search/index.js | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 24993e908a..29f42b4f5b 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -331,18 +331,9 @@ search.run(['$rootScope', '$state', function($rootScope, $state) { //If moving to a collection, sorts images by time added to a collection by default //allows sorting by newest first if set by user. Need to account for 'With Taken Date' tab impacts on query const checkForCollection = (query) => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); - const toQuery = toParams.query ? toParams.query.replace(HASNT_DATE_TAKEN, '').replace(HAS_DATE_TAKEN, '').trim() : ""; - if (checkForCollection(toQuery)) { - const fromQuery = fromParams.query ? fromParams.query.replace(HASNT_DATE_TAKEN, '').replace(HAS_DATE_TAKEN, '').trim() : ""; - const toHasTaken = toParams.query ? toParams.query.includes(HAS_DATE_TAKEN) : false; - const fromHasTaken = fromParams.query ? fromParams.query.includes(HAS_DATE_TAKEN) : false; - if ((fromHasTaken || toHasTaken) && !checkForCollection(fromQuery)) { - toParams.orderBy = (toQuery === fromQuery) ? toParams.orderBy : COLLECTION_SORT_VALUE; - } - } - //If moving from a collection to a non-collection, reset order to default. - else if (toParams.orderBy === COLLECTION_SORT_VALUE) { - delete toParams.orderBy; + const toQuery = toParams.query ? toParams.query : ""; + if (!checkForCollection(toQuery) && toParams.orderBy === COLLECTION_SORT_VALUE) { + delete toParams.orderBy; } } }); From bf0c6f4359a1b521fc1345029baef6c2076b69cf Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Wed, 3 Sep 2025 12:10:24 +0100 Subject: [PATCH 18/66] correcting css error --- kahuna/public/js/search/results.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index 491e781e8e..fc1e9bd1e7 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -136,7 +136,7 @@
    Date: Wed, 3 Sep 2025 13:57:38 +0100 Subject: [PATCH 19/66] correcting js syntax errors --- .../js/components/gr-tab-swap/gr-tab-swap.tsx | 2 +- kahuna/public/js/search/index.js | 4 +- kahuna/public/js/search/query.js | 47 +++++++++++-------- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx index bb4a63de43..d2ea64f018 100644 --- a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx +++ b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.tsx @@ -11,7 +11,7 @@ export interface TabSwapProps { panelVisible: boolean; } -export const TAB_WITH: string = "with"; +export const TAB_WITH = "with"; export const TabControl: React.FC = ({ onSelect, query, showTakenTab, noTakenDateCount, panelVisible }) => { diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 29f42b4f5b..f8abe4df30 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -30,7 +30,7 @@ import panelTemplate from '../components/gr-info-panel/gr-info-panel.html import collectionsPanelTemplate from '../components/gr-collections-panel/gr-collections-panel.html'; import {cropUtil} from '../util/crop'; -import { HAS_DATE_TAKEN, HASNT_DATE_TAKEN, COLLECTION_SORT_VALUE } from '../components/gr-sort-control/gr-sort-control-config'; +import { COLLECTION_SORT_VALUE } from '../components/gr-sort-control/gr-sort-control-config'; export var search = angular.module('kahuna.search', [ 'ct.ui.router.extras.dsr', @@ -326,7 +326,7 @@ search.run(['$rootScope', '$state', function($rootScope, $state) { $state.go('search.results', null, {reload: true}); } }); - $rootScope.$on('$stateChangeStart', (_, toState, toParams, fromState, fromParams) => { + $rootScope.$on('$stateChangeStart', (_, toState, toParams) => { if (toState.name === 'search.results') { //If moving to a collection, sorts images by time added to a collection by default //allows sorting by newest first if set by user. Need to account for 'With Taken Date' tab impacts on query diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 016756da06..bad8574929 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -24,8 +24,7 @@ import { DefaultSortOption, CollectionSortOption, HAS_DATE_TAKEN, - TAKEN_SORT, - SortOptions + TAKEN_SORT } from "../components/gr-sort-control/gr-sort-control-config"; export var query = angular.module('kahuna.search.query', [ @@ -190,7 +189,7 @@ query.controller('SearchQueryCtrl', [ } function checkForCollection(query) { - return /~"[a-zA-Z0-9 #-_.://]+"/.test(query) + return /~"[a-zA-Z0-9 #-_.://]+"/.test(query); }; function storeCollection(query) { @@ -214,6 +213,29 @@ query.controller('SearchQueryCtrl', [ storage.setJs("priorOrderBy", priorOrderBy); } + function revisedOrderBy(collectionSearch) { + if (collectionSearch) { + return CollectionSortOption.value; + } else { + return DefaultSortOption.value; + } + } + + function priorRevisedOrderBy(collectionSearch, newCollection, oldCollection) { + const priorOrderBy = getPiorOrderBy(); + if (collectionSearch && ((oldCollection !== newCollection) || ("" !== priorOrderBy))) { + if (priorOrderBy != "") { + setPriorOrderBy(""); + return priorOrderBy; + } else { + setPriorOrderBy(CollectionSortOption.value); + return null; + } + } else { + return null; + } + } + // eslint-disable-next-line complexity function watchSearchChange(newFilter, sender) { let showPaid = newFilter.nonFree ? newFilter.nonFree : false; @@ -223,7 +245,6 @@ query.controller('SearchQueryCtrl', [ storage.setJs("isNonFree", showPaid, true); // check for taken date sort contradiction - const oldOrderBy = storage.getJs("orderBy"); const curCollectionSearch = ctrl.collectionSearch; ctrl.collectionSearch = newFilter.query ? checkForCollection(newFilter.query) : false; const oldCollection = getCollection(); @@ -234,22 +255,10 @@ query.controller('SearchQueryCtrl', [ ctrl.ordering["orderBy"] = $stateParams.orderBy; } if ($stateParams.orderBy && $stateParams.orderBy.includes(TAKEN_SORT) && (!newFilter.query || !newFilter.query.includes(HAS_DATE_TAKEN))) { - if (ctrl.collectionSearch) { - ctrl.ordering["orderBy"] = CollectionSortOption.value; - } else { - ctrl.ordering["orderBy"] = DefaultSortOption.value; - } + ctrl.ordering["orderBy"] = revisedOrderBy(ctrl.collectionSearch); } else { - const priorOrderBy = getPiorOrderBy(); - if (ctrl.collectionSearch && ((oldCollection !== newCollection) || ("" !== priorOrderBy))) { - if (priorOrderBy != "") { - ctrl.ordering["orderBy"] = priorOrderBy; - setPriorOrderBy(""); - } else { - //console.log("Old collection="+oldCollection + ", New collection=" + newCollection + ", PriorOrderBy=" +priorOrderBy); - setPriorOrderBy(CollectionSortOption.value); - } - } + const prior = priorRevisedOrderBy(ctrl.collectionSearch, newCollection, oldCollection); + ctrl.ordering["orderBy"] = prior ? prior : ctrl.ordering["orderBy"]; } } let sortBy = ctrl.ordering["orderBy"] ? ctrl.ordering["orderBy"] : DefaultSortOption.value; From 8d4c7e727ad840c895ea85b566ac49c2fbc42938 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Thu, 11 Sep 2025 15:41:44 +0100 Subject: [PATCH 20/66] correcting css errors and adding use of has:dateTaken chip to basic sort control --- kahuna/public/js/components/gr-tab-swap/gr-tab-swap.css | 2 +- kahuna/public/js/search/query.js | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.css b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.css index beffcae482..b4762cd15c 100644 --- a/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.css +++ b/kahuna/public/js/components/gr-tab-swap/gr-tab-swap.css @@ -7,7 +7,7 @@ } .gr-tab-container { - font-family: Arial, sans-serif; + font-family: "Open Sans", sans-serif; display: flex; width: fit-content; } diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index bad8574929..ec583b5f95 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -24,6 +24,7 @@ import { DefaultSortOption, CollectionSortOption, HAS_DATE_TAKEN, + HASNT_DATE_TAKEN, TAKEN_SORT } from "../components/gr-sort-control/gr-sort-control-config"; @@ -307,6 +308,12 @@ query.controller('SearchQueryCtrl', [ function updateSortChips (sortSel) { ctrl.ordering['orderBy'] = manageSortSelection(sortSel.value); storage.setJs("orderBy", ctrl.ordering["orderBy"]); + var curQuery = ctrl.filter.query ? ctrl.filter.query : ''; + curQuery = curQuery.replace(HASNT_DATE_TAKEN, "").replace(HAS_DATE_TAKEN, "").trim(); + if (sortSel.isTaken) { + curQuery = `${curQuery} ${HAS_DATE_TAKEN}`.trim(); + } + ctrl.filter.query = curQuery; $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering['orderBy']}}); } From 6aebd1c59cb00f1a08987aebf4670cc03f5f3809 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Thu, 11 Sep 2025 16:15:03 +0100 Subject: [PATCH 21/66] removing use of taken date chip in basic operation --- kahuna/public/js/search/query.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index ec583b5f95..938f84845b 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -308,12 +308,6 @@ query.controller('SearchQueryCtrl', [ function updateSortChips (sortSel) { ctrl.ordering['orderBy'] = manageSortSelection(sortSel.value); storage.setJs("orderBy", ctrl.ordering["orderBy"]); - var curQuery = ctrl.filter.query ? ctrl.filter.query : ''; - curQuery = curQuery.replace(HASNT_DATE_TAKEN, "").replace(HAS_DATE_TAKEN, "").trim(); - if (sortSel.isTaken) { - curQuery = `${curQuery} ${HAS_DATE_TAKEN}`.trim(); - } - ctrl.filter.query = curQuery; $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering['orderBy']}}); } From a9a6c6ae848d64892847a47e5a97bb186aa81631 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Thu, 11 Sep 2025 16:21:24 +0100 Subject: [PATCH 22/66] removing unused vars --- kahuna/public/js/search/query.js | 1 - 1 file changed, 1 deletion(-) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 938f84845b..bad8574929 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -24,7 +24,6 @@ import { DefaultSortOption, CollectionSortOption, HAS_DATE_TAKEN, - HASNT_DATE_TAKEN, TAKEN_SORT } from "../components/gr-sort-control/gr-sort-control-config"; From bbe606fbc06eede82652dc2010cd7d18ff0092b1 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 27 Aug 2025 15:45:00 +0100 Subject: [PATCH 23/66] add bedrock class --- build.sbt | 5 ++ .../com/gu/mediaservice/lib/aws/Bedrock.scala | 52 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala diff --git a/build.sbt b/build.sbt index 68f81a84c4..6808763ff8 100644 --- a/build.sbt +++ b/build.sbt @@ -107,6 +107,8 @@ lazy val commonLib = project("common-lib").settings( "org.scanamo" %% "scanamo" % "2.0.0", // declare explicit dependency on desired version of aws sdk v2 dynamo "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, + // declare explicit dependency on desired version of aws sdk v2 bedrock runtime + "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, ws, "org.testcontainers" % "elasticsearch" % "1.19.2" % Test ), @@ -163,6 +165,7 @@ lazy val thrall = playProject("thrall", 9002) // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, + "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, "com.gu" %% "kcl-pekko-stream" % "0.1.0", "org.testcontainers" % "elasticsearch" % "1.19.2" % Test, "com.google.protobuf" % "protobuf-java" % "3.19.6" @@ -186,6 +189,7 @@ lazy val usage = playProject("usage", 9009).settings( // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, + "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, "com.google.protobuf" % "protobuf-java" % "3.19.6" ), dependencyOverrides ++= Seq( @@ -201,6 +205,7 @@ lazy val scripts = project("scripts") // V2 of the AWS SDK as it's easier to use for scripts and won't leak to the rest of the project from here "software.amazon.awssdk" % "s3" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, + "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, // bump jcommander explicitly as AWS SDK is pulling in a vulnerable version "com.beust" % "jcommander" % "1.75", "org.apache.commons" % "commons-compress" % "1.27.1", diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala new file mode 100644 index 0000000000..3a421a0794 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -0,0 +1,52 @@ +package com.gu.mediaservice.lib.aws + +import software.amazon.awssdk.services.bedrockruntime.model._ +import software.amazon.awssdk.services.bedrockruntime._ +import com.amazonaws.auth.AWSCredentialsProvider +import com.gu.mediaservice.lib.config.CommonConfig +import java.util.concurrent.CompletableFuture +import play.api.libs.json.Json +import software.amazon.awssdk.core.SdkBytes +import java.net.URI +import com.gu.mediaservice.lib.logging.LogMarker + +class Bedrock(imageBase64: String) extends AwsClientV2BuilderUtils { + + override def awsLocalEndpointUri: Option[URI] = ??? + + override def isDev: Boolean = ??? + + val client = + withAWSCredentialsV2(BedrockRuntimeAsyncClient.builder()) + .build() + + private def createRequestBody( + images: List[String] + ): String = { + + val body = Map( + "input_type" -> "image", + "embedding_types" -> "float", + "images" -> s"data:image/jpg;base64,$images" + ) + + Json.toJson(body).toString() + } + + val request: InvokeModelRequest = + InvokeModelRequest + .builder() + .accept("*/*") + .body(SdkBytes.fromUtf8String(createRequestBody(List(imageBase64)))) + .contentType("application/json") + .modelId("cohere.embed-english-v3") + .build() + + def fetchEmbedding()(implicit + logMarker: LogMarker + ): CompletableFuture[InvokeModelResponse] = { + logger.info(logMarker, s"Going to fetch embedding now") + client.invokeModel(request) + } + +} From 4c90556b7d1b34b0f42d1c73f673e862ca7d476e Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 1 Oct 2025 10:16:32 +0100 Subject: [PATCH 24/66] copy over useful bits from old branch --- .../com/gu/mediaservice/lib/aws/Bedrock.scala | 120 ++++++++++++++---- .../mediaservice/model/ImageEmbedding.scala | 12 ++ image-loader/app/ImageLoaderComponents.scala | 10 +- image-loader/app/model/Projector.scala | 15 ++- image-loader/app/model/Uploader.scala | 3 +- 5 files changed, 123 insertions(+), 37 deletions(-) create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/model/ImageEmbedding.scala diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index 3a421a0794..461a111a88 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -4,49 +4,117 @@ import software.amazon.awssdk.services.bedrockruntime.model._ import software.amazon.awssdk.services.bedrockruntime._ import com.amazonaws.auth.AWSCredentialsProvider import com.gu.mediaservice.lib.config.CommonConfig + import java.util.concurrent.CompletableFuture import play.api.libs.json.Json import software.amazon.awssdk.core.SdkBytes + import java.net.URI import com.gu.mediaservice.lib.logging.LogMarker +import com.gu.mediaservice.model.ImageEmbedding +import play.api.libs.json.Format.GenericFormat +import play.api.libs.json.OFormat.oFormatFromReadsAndOWrites +import play.api.libs.json._ + +object Bedrock { + private case class BedrockRequest( + input_type: String, + embedding_types: List[String], + images: List[String] + ) + private implicit val bedrockRequestFormat: OFormat[BedrockRequest] = Json.format[BedrockRequest] +} + +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.FutureConverters.CompletionStageOps -class Bedrock(imageBase64: String) extends AwsClientV2BuilderUtils { +class Bedrock(config: CommonConfig) + extends AwsClientV2BuilderUtils { - override def awsLocalEndpointUri: Option[URI] = ??? + // TODO: figure out what the more usual pattern for turning off localstack behaviour is + override def awsLocalEndpointUri: Option[URI] = None - override def isDev: Boolean = ??? + override def isDev: Boolean = config.isDev - val client = + val client: BedrockRuntimeAsyncClient = { + logger.info("Creating Bedrock client") withAWSCredentialsV2(BedrockRuntimeAsyncClient.builder()) .build() + } - private def createRequestBody( - images: List[String] - ): String = { - - val body = Map( - "input_type" -> "image", - "embedding_types" -> "float", - "images" -> s"data:image/jpg;base64,$images" + private def createRequestBody(base64EncodedImage: String): InvokeModelRequest = { + logger.info("Creating request body") + val body = Bedrock.BedrockRequest( + input_type = "image", + embedding_types = List("float"), + images = List(s"data:image/jpg;base64,$base64EncodedImage") ) + val jsonBody = Json.toJson(body).toString() + logger.info(s"Request body created, length: ${jsonBody.length}") - Json.toJson(body).toString() + val request: InvokeModelRequest = { + logger.info("Building request") + InvokeModelRequest + .builder() + .accept("*/*") + .body(SdkBytes.fromUtf8String(jsonBody)) + .contentType("application/json") + .modelId("cohere.embed-english-v3") + .build() + } + request } - val request: InvokeModelRequest = - InvokeModelRequest - .builder() - .accept("*/*") - .body(SdkBytes.fromUtf8String(createRequestBody(List(imageBase64)))) - .contentType("application/json") - .modelId("cohere.embed-english-v3") - .build() - - def fetchEmbedding()(implicit - logMarker: LogMarker + def sendBedrockEmbeddingRequest(base64EncodedImage: String)( + implicit logMarker: LogMarker ): CompletableFuture[InvokeModelResponse] = { - logger.info(logMarker, s"Going to fetch embedding now") - client.invokeModel(request) + logger.info(logMarker, "Starting fetchEmbedding call") + try { + val future = client.invokeModel(createRequestBody(base64EncodedImage)) + logger.info(logMarker, "Bedrock API call initiated") + future.whenComplete((response, error) => { + if (error != null) { + logger.error(logMarker, "Bedrock API call failed", error) + } else { + logger.info( + logMarker, + s"Bedrock API call completed with status: ${response.sdkHttpResponse().statusCode()}" + ) + } + }) + future + } catch { + case e: Exception => + logger.error(logMarker, "Exception during Bedrock API call", e) + throw e + } } + def createImageEmbedding(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[ImageEmbedding] = { + logger.info( + logMarker, + s"Starting image embedding creation" + ) + val bedrockFuture = sendBedrockEmbeddingRequest(base64EncodedImage) + + bedrockFuture.asScala + .map { response => + logger.info( + logMarker, + s"Received Bedrock response. Status: ${response.sdkHttpResponse().statusCode()}" + ) + // Now we can safely access response.body() + val responseBody = response.body().asUtf8String() + logger.info(logMarker, s"Response body: $responseBody") + val json = Json.parse(responseBody) + logger.info(logMarker, s"Parsed JSON response: $json") + // Extract the embeddings array (first element since it's an array of arrays) + val embeddings = (json \ "embeddings" \ "float")(0).as[List[Double]] + logger.info( + logMarker, + s"Successfully extracted embeddings. Vector size: ${embeddings.size}" + ) + ImageEmbedding(cohereEmbedEnglishV3 = embeddings) + } + } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/ImageEmbedding.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/ImageEmbedding.scala new file mode 100644 index 0000000000..5ba3765d24 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/ImageEmbedding.scala @@ -0,0 +1,12 @@ +package com.gu.mediaservice.model + +import play.api.libs.json.Json +import play.api.libs.json.OFormat + +case class ImageEmbedding( + cohereEmbedEnglishV3: List[Double] +) + +object ImageEmbedding { + implicit val format: OFormat[ImageEmbedding] = Json.format[ImageEmbedding] +} \ No newline at end of file diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index ebe3ef69ee..89caf0aefc 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -1,5 +1,5 @@ import com.gu.mediaservice.GridClient -import com.gu.mediaservice.lib.aws.SimpleSqsMessageConsumer +import com.gu.mediaservice.lib.aws.{Bedrock, SimpleSqsMessageConsumer} import com.gu.mediaservice.lib.config.Services import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.GridLogging @@ -27,8 +27,12 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath) val notifications = new Notifications(config) val downloader = new Downloader()(ec,wsClient) - val uploader = new Uploader(store, config, imageOperations, notifications, imageProcessor) - val projector = Projector(config, imageOperations, imageProcessor, auth) + + val bedrock = new Bedrock(config) + logger.info(s"bedrock client: ${bedrock.client}") + + val uploader = new Uploader(store, config, imageOperations, notifications, bedrock, imageProcessor) + val projector = Projector(config, imageOperations, imageProcessor, auth, bedrock) val quarantineUploader: Option[QuarantineUploader] = (config.uploadToQuarantineEnabled, config.quarantineBucket) match { case (true, Some(bucketName)) =>{ val quarantineStore = new QuarantineStore(config) diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index c776d664f3..9c534970e4 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -8,8 +8,7 @@ import com.gu.mediaservice.lib.auth.Authentication import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object} import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.S3Ops -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{Bedrock, S3Object, S3Ops} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} @@ -31,8 +30,8 @@ object Projector { import Uploader.toImageUploadOpsCfg - def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication)(implicit ec: ExecutionContext): Projector - = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth) + def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, bedrock: Bedrock)(implicit ec: ExecutionContext): Projector + = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, bedrock) } case class S3FileExtractedMetadata( @@ -85,9 +84,10 @@ class Projector(config: ImageUploadOpsCfg, s3: AmazonS3, imageOps: ImageOperations, processor: ImageProcessor, - auth: Authentication) extends GridLogging { + auth: Authentication, + bedrock: Bedrock) extends GridLogging { - private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3) + private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, bedrock) def projectS3ImageById(imageId: String, tempFile: File, gridClient: GridClient, onBehalfOfFn: WSRequest => WSRequest) (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[Image]] = { @@ -159,7 +159,8 @@ class Projector(config: ImageUploadOpsCfg, class ImageUploadProjectionOps(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, - s3: AmazonS3 + s3: AmazonS3, + bedrock: Bedrock, ) extends GridLogging { import Uploader.{fromUploadRequestShared, toMetaMap} diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index 38e1574435..fa9dd5ba3c 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -12,7 +12,7 @@ import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.auth.Authentication import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{S3Object, UpdateMessage} +import com.gu.mediaservice.lib.aws.{Bedrock, S3Object, UpdateMessage} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.formatting._ import com.gu.mediaservice.lib.imaging.ImageOperations @@ -327,6 +327,7 @@ class Uploader(val store: ImageLoaderStore, val config: ImageLoaderConfig, val imageOps: ImageOperations, val notifications: Notifications, + val bedrock: Bedrock, imageProcessor: ImageProcessor) (implicit val ec: ExecutionContext) extends MessageSubjects with ArgoHelpers { From 1987034d54d6fff161b57c62b5b4c12099b067d9 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 1 Oct 2025 13:34:02 +0100 Subject: [PATCH 25/66] start to s3 vectors class --- .../main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala new file mode 100644 index 0000000000..94418f34d8 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -0,0 +1,5 @@ +package com.gu.mediaservice.lib.aws + +class S3Vectors { + +} From f2fa09e0e3b503076bbe0abac0e94aa1b8853edb Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 1 Oct 2025 14:07:07 +0100 Subject: [PATCH 26/66] create putVector function --- build.sbt | 6 +- .../com/gu/mediaservice/lib/aws/Bedrock.scala | 10 ++- .../gu/mediaservice/lib/aws/S3Vectors.scala | 69 ++++++++++++++++++- 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index 6808763ff8..eb1332cf76 100644 --- a/build.sbt +++ b/build.sbt @@ -63,7 +63,7 @@ Global / concurrentRestrictions := Seq( ) val awsSdkVersion = "1.12.470" -val awsSdkV2Version = "2.31.12" +val awsSdkV2Version = "2.32.33" val elastic4sVersion = "8.3.0" val okHttpVersion = "3.12.1" @@ -109,6 +109,7 @@ lazy val commonLib = project("common-lib").settings( "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, // declare explicit dependency on desired version of aws sdk v2 bedrock runtime "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, + "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, ws, "org.testcontainers" % "elasticsearch" % "1.19.2" % Test ), @@ -166,6 +167,7 @@ lazy val thrall = playProject("thrall", 9002) "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, + "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, "com.gu" %% "kcl-pekko-stream" % "0.1.0", "org.testcontainers" % "elasticsearch" % "1.19.2" % Test, "com.google.protobuf" % "protobuf-java" % "3.19.6" @@ -190,6 +192,7 @@ lazy val usage = playProject("usage", 9009).settings( "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, + "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, "com.google.protobuf" % "protobuf-java" % "3.19.6" ), dependencyOverrides ++= Seq( @@ -206,6 +209,7 @@ lazy val scripts = project("scripts") "software.amazon.awssdk" % "s3" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, + "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, // bump jcommander explicitly as AWS SDK is pulling in a vulnerable version "com.beust" % "jcommander" % "1.75", "org.apache.commons" % "commons-compress" % "1.27.1", diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index 461a111a88..685ccf9631 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -2,7 +2,6 @@ package com.gu.mediaservice.lib.aws import software.amazon.awssdk.services.bedrockruntime.model._ import software.amazon.awssdk.services.bedrockruntime._ -import com.amazonaws.auth.AWSCredentialsProvider import com.gu.mediaservice.lib.config.CommonConfig import java.util.concurrent.CompletableFuture @@ -11,7 +10,6 @@ import software.amazon.awssdk.core.SdkBytes import java.net.URI import com.gu.mediaservice.lib.logging.LogMarker -import com.gu.mediaservice.model.ImageEmbedding import play.api.libs.json.Format.GenericFormat import play.api.libs.json.OFormat.oFormatFromReadsAndOWrites import play.api.libs.json._ @@ -65,7 +63,7 @@ class Bedrock(config: CommonConfig) request } - def sendBedrockEmbeddingRequest(base64EncodedImage: String)( + private def sendBedrockEmbeddingRequest(base64EncodedImage: String)( implicit logMarker: LogMarker ): CompletableFuture[InvokeModelResponse] = { logger.info(logMarker, "Starting fetchEmbedding call") @@ -90,7 +88,7 @@ class Bedrock(config: CommonConfig) } } - def createImageEmbedding(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[ImageEmbedding] = { + def createImageEmbedding(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] = { logger.info( logMarker, s"Starting image embedding creation" @@ -109,12 +107,12 @@ class Bedrock(config: CommonConfig) val json = Json.parse(responseBody) logger.info(logMarker, s"Parsed JSON response: $json") // Extract the embeddings array (first element since it's an array of arrays) - val embeddings = (json \ "embeddings" \ "float")(0).as[List[Double]] + val embeddings = (json \ "embeddings" \ "float")(0).as[List[Float]] logger.info( logMarker, s"Successfully extracted embeddings. Vector size: ${embeddings.size}" ) - ImageEmbedding(cohereEmbedEnglishV3 = embeddings) + embeddings } } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index 94418f34d8..369a898ce6 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -1,5 +1,72 @@ package com.gu.mediaservice.lib.aws +import com.gu.mediaservice.lib.config.CommonConfig +import com.gu.mediaservice.lib.logging.LogMarker +import software.amazon.awssdk.services.s3vectors._ +import software.amazon.awssdk.services.s3vectors.model.{PutInputVector, PutVectorsRequest, PutVectorsResponse, VectorData} -class S3Vectors { +import java.net.URI +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ +class S3Vectors(config: CommonConfig) + extends AwsClientV2BuilderUtils { + + // TODO: figure out what the more usual pattern for turning off localstack behaviour is + override def awsLocalEndpointUri: Option[URI] = None + + override def isDev: Boolean = config.isDev + + val client: S3VectorsClient = { + logger.info("Creating Bedrock client") + withAWSCredentialsV2(S3VectorsClient.builder()) + .build() + } + + private def createRequestBody(embedding: List[Float], imageId: String) = { + logger.info("Creating request body") + + val vectorData: VectorData = VectorData + .builder() +// TODO find out if we can do something less upsetting than this float conversion + .float32(embedding.map(_.asInstanceOf[java.lang.Float]).asJava) + .build() + + val inputVector: PutInputVector = PutInputVector + .builder() + .data(vectorData) + .key(imageId) + .build() + + val request: PutVectorsRequest = PutVectorsRequest + .builder() + .indexName("cohereEmbedEnglishV3") + .vectorBucketName(s"image-embeddings-${config.stage}") + .vectors(inputVector) + .build() + + request + } + + def putVector(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker + ): Future[PutVectorsResponse] = { + logger.info("Starting embedding call") + + try { + + val bedrock = new Bedrock(config) + + val embedding = bedrock.createImageEmbedding(base64EncodedImage) + + val vectorInput = embedding.map { data => + client.putVectors(createRequestBody(data, imageId)) + } + + vectorInput + } catch { + case e: Exception => + logger.error(logMarker, "Exception during Bedrock API call", e) + throw e + } + } +// private def searchS3VectorStore() = {} } From e9459bff2154a699ba42ea629d4f3eb2628eed65 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Tue, 7 Oct 2025 13:37:23 +0100 Subject: [PATCH 27/66] wip commit --- .../gu/mediaservice/lib/aws/S3Vectors.scala | 20 ++++++++++------- image-loader/app/ImageLoaderComponents.scala | 10 ++++----- image-loader/app/model/Projector.scala | 18 ++++++++++----- image-loader/app/model/Uploader.scala | 22 +++++++++++++++---- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index 369a898ce6..62afb64edf 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -1,6 +1,7 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.LogMarker +import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3vectors._ import software.amazon.awssdk.services.s3vectors.model.{PutInputVector, PutVectorsRequest, PutVectorsResponse, VectorData} @@ -16,18 +17,20 @@ class S3Vectors(config: CommonConfig) override def isDev: Boolean = config.isDev + override def awsRegionV2: Region = Region.EU_CENTRAL_1 + val client: S3VectorsClient = { logger.info("Creating Bedrock client") withAWSCredentialsV2(S3VectorsClient.builder()) .build() } - private def createRequestBody(embedding: List[Float], imageId: String) = { + private def createRequestBody(embedding: List[Float], imageId: String): PutVectorsRequest = { logger.info("Creating request body") val vectorData: VectorData = VectorData .builder() -// TODO find out if we can do something less upsetting than this float conversion + // TODO find out if we can do something less upsetting than this float conversion .float32(embedding.map(_.asInstanceOf[java.lang.Float]).asJava) .build() @@ -39,8 +42,8 @@ class S3Vectors(config: CommonConfig) val request: PutVectorsRequest = PutVectorsRequest .builder() - .indexName("cohereEmbedEnglishV3") - .vectorBucketName(s"image-embeddings-${config.stage}") + .indexName("cohere-embed-english-v3") + .vectorBucketName(s"image-embeddings-${config.stage.toLowerCase}") .vectors(inputVector) .build() @@ -49,14 +52,15 @@ class S3Vectors(config: CommonConfig) def putVector(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker ): Future[PutVectorsResponse] = { - logger.info("Starting embedding call") + logger.info("Starting putVector") try { - val bedrock = new Bedrock(config) - + logger.info("Created bedrock class") + logger.info("image string: ", base64EncodedImage) val embedding = bedrock.createImageEmbedding(base64EncodedImage) - + logger.info("Created embedding, ", embedding) + logger.info("Now we're going to call the putVectors function...") val vectorInput = embedding.map { data => client.putVectors(createRequestBody(data, imageId)) } diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index 89caf0aefc..f9d4824bcd 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -1,5 +1,5 @@ import com.gu.mediaservice.GridClient -import com.gu.mediaservice.lib.aws.{Bedrock, SimpleSqsMessageConsumer} +import com.gu.mediaservice.lib.aws.{Bedrock, S3Vectors, SimpleSqsMessageConsumer} import com.gu.mediaservice.lib.config.Services import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.GridLogging @@ -28,11 +28,11 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val notifications = new Notifications(config) val downloader = new Downloader()(ec,wsClient) - val bedrock = new Bedrock(config) - logger.info(s"bedrock client: ${bedrock.client}") + val s3vectors = new S3Vectors(config) + logger.info(s"new S3 vectors client: ${s3vectors.client}") - val uploader = new Uploader(store, config, imageOperations, notifications, bedrock, imageProcessor) - val projector = Projector(config, imageOperations, imageProcessor, auth, bedrock) + val uploader = new Uploader(store, config, imageOperations, notifications, s3vectors, imageProcessor) + val projector = Projector(config, imageOperations, imageProcessor, auth, s3vectors) val quarantineUploader: Option[QuarantineUploader] = (config.uploadToQuarantineEnabled, config.quarantineBucket) match { case (true, Some(bucketName)) =>{ val quarantineStore = new QuarantineStore(config) diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 9c534970e4..8dec36cc3f 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -8,7 +8,7 @@ import com.gu.mediaservice.lib.auth.Authentication import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object} import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Bedrock, S3Object, S3Ops} +import com.gu.mediaservice.lib.aws.{S3Object, S3Ops, S3Vectors} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} @@ -21,6 +21,7 @@ import org.apache.tika.io.IOUtils import org.joda.time.{DateTime, DateTimeZone} import play.api.Logger import play.api.libs.ws.WSRequest +import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import scala.jdk.CollectionConverters._ import scala.concurrent.duration.Duration @@ -30,8 +31,8 @@ object Projector { import Uploader.toImageUploadOpsCfg - def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, bedrock: Bedrock)(implicit ec: ExecutionContext): Projector - = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, bedrock) + def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, s3vectors: S3Vectors)(implicit ec: ExecutionContext): Projector + = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, s3vectors) } case class S3FileExtractedMetadata( @@ -85,9 +86,9 @@ class Projector(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, - bedrock: Bedrock) extends GridLogging { + s3vectors: S3Vectors) extends GridLogging { - private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, bedrock) + private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, s3vectors) def projectS3ImageById(imageId: String, tempFile: File, gridClient: GridClient, onBehalfOfFn: WSRequest => WSRequest) (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[Image]] = { @@ -160,7 +161,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, s3: AmazonS3, - bedrock: Bedrock, + s3vectors: S3Vectors, ) extends GridLogging { import Uploader.{fromUploadRequestShared, toMetaMap} @@ -176,11 +177,16 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, projectOptimisedPNGFileAsS3Model, tryFetchThumbFile = fetchThumbFile, tryFetchOptimisedFile = fetchOptimisedFile, + fetchImageEmbedding = imageEmbedding, ) fromUploadRequestShared(uploadRequest, dependenciesWithProjectionsOnly, processor) } + private def imageEmbedding(base64EncodedImage: String, imageId: String) (implicit ec: ExecutionContext, logMarker: LogMarker): Future[PutVectorsResponse] = { + s3vectors.putVector(base64EncodedImage, imageId) + } + private def projectOriginalFileAsS3Model(storableOriginalImage: StorableOriginalImage) = Future.successful(storableOriginalImage.toProjectedS3Object(config.originalFileBucket)) diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index fa9dd5ba3c..fef768d9f5 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -7,12 +7,12 @@ import java.io.File import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} -import java.util.UUID +import java.util.{Base64, UUID} import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.auth.Authentication import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Bedrock, S3Object, UpdateMessage} +import com.gu.mediaservice.lib.aws.{S3Object, S3Vectors, UpdateMessage} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.formatting._ import com.gu.mediaservice.lib.imaging.ImageOperations @@ -30,6 +30,7 @@ import model.upload.{OptimiseOps, OptimiseWithPngQuant, UploadRequest} import org.joda.time.DateTime import play.api.libs.json.{JsObject, Json} import play.api.libs.ws.WSRequest +import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import scala.collection.compat._ import scala.concurrent.{ExecutionContext, Future} @@ -81,6 +82,7 @@ case class ImageUploadOpsDependencies( storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object], tryFetchThumbFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), + fetchImageEmbedding: (String, String) => Future[PutVectorsResponse] ) case class UploadStatusUri (uri: String) extends AnyVal { @@ -116,6 +118,7 @@ object Uploader extends GridLogging { storeOrProjectOriginalFile, storeOrProjectThumbFile, storeOrProjectOptimisedImage, + fetchImageEmbedding, OptimiseWithPngQuant, uploadRequest, deps, @@ -127,6 +130,7 @@ object Uploader extends GridLogging { private[model] def uploadAndStoreImage(storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], storeOrProjectOptimisedFile: StorableOptimisedImage => Future[S3Object], + fetchImageEmbedding: (String, String) => Future[PutVectorsResponse], optimiseOps: OptimiseOps, uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies, @@ -156,6 +160,12 @@ object Uploader extends GridLogging { val sourceStoreFuture = storeOrProjectOriginalFile(storableOriginalImage) val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, tempDirForRequest, deps) + val imageBase64: String = Base64.getEncoder().encodeToString(Files.readAllBytes(uploadRequest.tempFile.toPath)) + logger.info("imageBase64 from uploader: ", Files.readAllBytes(uploadRequest.tempFile.toPath)) +// TODO: how do I get the imageid? + val imageId: String = "abcdef" + val eventualImageEmbedding = fetchImageEmbedding(imageBase64, imageId) + val eventualImage = for { browserViewableImage <- eventualBrowserViewableImage s3Source <- sourceStoreFuture @@ -171,6 +181,7 @@ object Uploader extends GridLogging { case Some(storableOptimisedImage) => storeOrProjectOptimisedFile(storableOptimisedImage).map(a=>Some(a)) case None => Future.successful(None) } + imageEmbedding <- eventualImageEmbedding thumbDimensions <- FileMetadataReader.dimensions(thumbViewableImage.file, Some(thumbViewableImage.mimeType)) colourModel <- colourModelFuture } yield { @@ -327,20 +338,23 @@ class Uploader(val store: ImageLoaderStore, val config: ImageLoaderConfig, val imageOps: ImageOperations, val notifications: Notifications, - val bedrock: Bedrock, + val s3vectors: S3Vectors, imageProcessor: ImageProcessor) (implicit val ec: ExecutionContext) extends MessageSubjects with ArgoHelpers { def fromUploadRequest(uploadRequest: UploadRequest) (implicit logMarker: LogMarker): Future[ImageUpload] = { val sideEffectDependencies = ImageUploadOpsDependencies(toImageUploadOpsCfg(config), imageOps, - storeSource, storeThumbnail, storeOptimisedImage) + storeSource, storeThumbnail, storeOptimisedImage, fetchImageEmbedding = fetchImageEmbedding) Stopwatch.async("finalImage") { val finalImage = fromUploadRequestShared(uploadRequest, sideEffectDependencies, imageProcessor) finalImage.map(img => ImageUpload(uploadRequest, img)) } } + private def fetchImageEmbedding(base64EncodedImage: String, imageId: String)(implicit logMarker: LogMarker): Future[PutVectorsResponse] = + s3vectors.putVector(base64EncodedImage, imageId) + private def storeSource(storableOriginalImage: StorableOriginalImage) (implicit logMarker: LogMarker) = store.store(storableOriginalImage) From 9e512293382b403cff4a5db1cbc12ca1fa958d86 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Tue, 7 Oct 2025 15:43:25 +0100 Subject: [PATCH 28/66] successful write to s3 vector store --- .../gu/mediaservice/lib/aws/S3Vectors.scala | 20 +++++++++------- image-loader/app/model/Uploader.scala | 24 ++++++++++++++----- vectorcommand.sh | 8 +++++++ 3 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 vectorcommand.sh diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index 62afb64edf..a4c78dd88b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -43,13 +43,16 @@ class S3Vectors(config: CommonConfig) val request: PutVectorsRequest = PutVectorsRequest .builder() .indexName("cohere-embed-english-v3") - .vectorBucketName(s"image-embeddings-${config.stage.toLowerCase}") + // TODO EM: create buckets for -dev and -prod + .vectorBucketName("image-embeddings-test") +// (s"image-embeddings-${config.stage.toLowerCase}") .vectors(inputVector) .build() request } + // TODO EM: separate out creating the embedding from putting the vector def putVector(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker ): Future[PutVectorsResponse] = { logger.info("Starting putVector") @@ -57,14 +60,15 @@ class S3Vectors(config: CommonConfig) try { val bedrock = new Bedrock(config) logger.info("Created bedrock class") - logger.info("image string: ", base64EncodedImage) - val embedding = bedrock.createImageEmbedding(base64EncodedImage) - logger.info("Created embedding, ", embedding) - logger.info("Now we're going to call the putVectors function...") - val vectorInput = embedding.map { data => - client.putVectors(createRequestBody(data, imageId)) + logger.info(s"image string length: ${base64EncodedImage.length}") + val embeddingFuture = bedrock.createImageEmbedding(base64EncodedImage) + val vectorInput = embeddingFuture.map { embedding => + logger.info(s"Created embedding with length: ${embedding.length}") + val input = createRequestBody(embedding, imageId) + logger.info(s"vector request body ${input}") + logger.info("Now we're going to call the putVectors function...") + client.putVectors(input) } - vectorInput } catch { case e: Exception => diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index fef768d9f5..7704929194 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -160,11 +160,21 @@ object Uploader extends GridLogging { val sourceStoreFuture = storeOrProjectOriginalFile(storableOriginalImage) val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, tempDirForRequest, deps) - val imageBase64: String = Base64.getEncoder().encodeToString(Files.readAllBytes(uploadRequest.tempFile.toPath)) - logger.info("imageBase64 from uploader: ", Files.readAllBytes(uploadRequest.tempFile.toPath)) -// TODO: how do I get the imageid? - val imageId: String = "abcdef" - val eventualImageEmbedding = fetchImageEmbedding(imageBase64, imageId) +// TODO why is this not working???? + val filePath = uploadRequest.tempFile.toPath + logger.info(s"filePath: ${filePath}") + + val fileBytes = Files.readAllBytes(filePath) + logger.info(s"fileBytes length: ${fileBytes.length}") + + val base64EncodedString = Base64.getEncoder().encodeToString(fileBytes) + logger.info(s"base64EncodedString: ${base64EncodedString.length}") + +// val imageBase64: String = Base64.getEncoder().encodeToString(Files.readAllBytes(uploadRequest.tempFile.toPath)) +// logger.info("imageBase64 from uploader: ", Files.readAllBytes(uploadRequest.tempFile.toPath)) + + // TODO: check that uploadRequest.imageId is the eventual image id in elasticsearch etc + val eventualImageEmbedding = fetchImageEmbedding(base64EncodedString, uploadRequest.imageId) val eventualImage = for { browserViewableImage <- eventualBrowserViewableImage @@ -352,8 +362,10 @@ class Uploader(val store: ImageLoaderStore, } } - private def fetchImageEmbedding(base64EncodedImage: String, imageId: String)(implicit logMarker: LogMarker): Future[PutVectorsResponse] = + // TODO EM: this name feels wrong + private def fetchImageEmbedding(base64EncodedImage: String, imageId: String)(implicit logMarker: LogMarker): Future[PutVectorsResponse] = { s3vectors.putVector(base64EncodedImage, imageId) + } private def storeSource(storableOriginalImage: StorableOriginalImage) (implicit logMarker: LogMarker) = store.store(storableOriginalImage) diff --git a/vectorcommand.sh b/vectorcommand.sh new file mode 100644 index 0000000000..72d50b78c8 --- /dev/null +++ b/vectorcommand.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +aws s3vectors list-vectors \ + --vector-bucket-name image-embeddings-test \ + --index-name cohere-embed-english-v3 \ + --profile media-service \ + --region eu-central-1 + From 721a6be9d99b20c3dea8d562fe2b18c99de724f3 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Tue, 7 Oct 2025 16:16:10 +0100 Subject: [PATCH 29/66] rename and tidy up --- .../gu/mediaservice/lib/aws/S3Vectors.scala | 57 +++++++++++-------- image-loader/app/ImageLoaderComponents.scala | 3 - image-loader/app/model/Projector.scala | 6 +- image-loader/app/model/Uploader.scala | 29 +++++----- 4 files changed, 50 insertions(+), 45 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index a4c78dd88b..e2fa2a70a8 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -1,6 +1,7 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.LogMarker +import org.bouncycastle.util.encoders.Base64Encoder import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3vectors._ import software.amazon.awssdk.services.s3vectors.model.{PutInputVector, PutVectorsRequest, PutVectorsResponse, VectorData} @@ -26,7 +27,7 @@ class S3Vectors(config: CommonConfig) } private def createRequestBody(embedding: List[Float], imageId: String): PutVectorsRequest = { - logger.info("Creating request body") + logger.info("Creating request body for S3 Vector Store...") val vectorData: VectorData = VectorData .builder() @@ -43,38 +44,46 @@ class S3Vectors(config: CommonConfig) val request: PutVectorsRequest = PutVectorsRequest .builder() .indexName("cohere-embed-english-v3") - // TODO EM: create buckets for -dev and -prod - .vectorBucketName("image-embeddings-test") -// (s"image-embeddings-${config.stage.toLowerCase}") + .vectorBucketName(s"image-embeddings-${config.stage.toLowerCase}") .vectors(inputVector) .build() request } - // TODO EM: separate out creating the embedding from putting the vector - def putVector(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker - ): Future[PutVectorsResponse] = { - logger.info("Starting putVector") + private def fetchEmbeddingFromBedrock(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker + ): Future[List[Float]] = { + val bedrock = new Bedrock(config) + bedrock.createImageEmbedding(base64EncodedImage) + } + private def storeEmbeddingInS3VectorStore(bedrockEmbedding: List[Float], imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker + ): PutVectorsResponse = { try { - val bedrock = new Bedrock(config) - logger.info("Created bedrock class") - logger.info(s"image string length: ${base64EncodedImage.length}") - val embeddingFuture = bedrock.createImageEmbedding(base64EncodedImage) - val vectorInput = embeddingFuture.map { embedding => - logger.info(s"Created embedding with length: ${embedding.length}") - val input = createRequestBody(embedding, imageId) - logger.info(s"vector request body ${input}") - logger.info("Now we're going to call the putVectors function...") - client.putVectors(input) - } - vectorInput - } catch { - case e: Exception => - logger.error(logMarker, "Exception during Bedrock API call", e) - throw e + val input = createRequestBody(bedrockEmbedding, imageId) + logger.info("Creating S3 Vector ") + val response = client.putVectors(input) + logger.info( + logMarker, + s"S3 Vector Store API call completed with status: ${response.sdkHttpResponse().statusCode()}" + ) + response + } + catch { + case e: Exception => + logger.error(logMarker, "Exception during S3 Vector Store API call", e) + throw e + } + } + + def fetchEmbeddingAndStore(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker + ): Future[PutVectorsResponse] = { + val embeddingFuture = fetchEmbeddingFromBedrock(base64EncodedImage: String) + val vectorInput = embeddingFuture.map { embedding => + storeEmbeddingInS3VectorStore(embedding, imageId) } + vectorInput } + // private def searchS3VectorStore() = {} } diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index f9d4824bcd..219337ad51 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -27,10 +27,7 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath) val notifications = new Notifications(config) val downloader = new Downloader()(ec,wsClient) - val s3vectors = new S3Vectors(config) - logger.info(s"new S3 vectors client: ${s3vectors.client}") - val uploader = new Uploader(store, config, imageOperations, notifications, s3vectors, imageProcessor) val projector = Projector(config, imageOperations, imageProcessor, auth, s3vectors) val quarantineUploader: Option[QuarantineUploader] = (config.uploadToQuarantineEnabled, config.quarantineBucket) match { diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 8dec36cc3f..2cc3a50c3e 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -177,14 +177,14 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, projectOptimisedPNGFileAsS3Model, tryFetchThumbFile = fetchThumbFile, tryFetchOptimisedFile = fetchOptimisedFile, - fetchImageEmbedding = imageEmbedding, + fetchEmbeddingAndStore = fetchEmbeddingAndStore, ) fromUploadRequestShared(uploadRequest, dependenciesWithProjectionsOnly, processor) } - private def imageEmbedding(base64EncodedImage: String, imageId: String) (implicit ec: ExecutionContext, logMarker: LogMarker): Future[PutVectorsResponse] = { - s3vectors.putVector(base64EncodedImage, imageId) + private def fetchEmbeddingAndStore(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[PutVectorsResponse] = { + s3vectors.fetchEmbeddingAndStore(base64EncodedImage, imageId) } private def projectOriginalFileAsS3Model(storableOriginalImage: StorableOriginalImage) = diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index 7704929194..ded62ecaa7 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -75,14 +75,14 @@ case class ImageUploadOpsCfg( ) case class ImageUploadOpsDependencies( - config: ImageUploadOpsCfg, - imageOps: ImageOperations, - storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], - storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], - storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object], - tryFetchThumbFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), - tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), - fetchImageEmbedding: (String, String) => Future[PutVectorsResponse] + config: ImageUploadOpsCfg, + imageOps: ImageOperations, + storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], + storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], + storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object], + tryFetchThumbFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), + tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), + fetchEmbeddingAndStore: (String, String) => Future[PutVectorsResponse] ) case class UploadStatusUri (uri: String) extends AnyVal { @@ -118,7 +118,7 @@ object Uploader extends GridLogging { storeOrProjectOriginalFile, storeOrProjectThumbFile, storeOrProjectOptimisedImage, - fetchImageEmbedding, + fetchEmbeddingAndStore, OptimiseWithPngQuant, uploadRequest, deps, @@ -130,7 +130,7 @@ object Uploader extends GridLogging { private[model] def uploadAndStoreImage(storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], storeOrProjectOptimisedFile: StorableOptimisedImage => Future[S3Object], - fetchImageEmbedding: (String, String) => Future[PutVectorsResponse], + fetchEmbeddingAndStore: (String, String) => Future[PutVectorsResponse], optimiseOps: OptimiseOps, uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies, @@ -174,7 +174,7 @@ object Uploader extends GridLogging { // logger.info("imageBase64 from uploader: ", Files.readAllBytes(uploadRequest.tempFile.toPath)) // TODO: check that uploadRequest.imageId is the eventual image id in elasticsearch etc - val eventualImageEmbedding = fetchImageEmbedding(base64EncodedString, uploadRequest.imageId) + val eventualImageEmbedding = fetchEmbeddingAndStore(base64EncodedString, uploadRequest.imageId) val eventualImage = for { browserViewableImage <- eventualBrowserViewableImage @@ -355,16 +355,15 @@ class Uploader(val store: ImageLoaderStore, def fromUploadRequest(uploadRequest: UploadRequest) (implicit logMarker: LogMarker): Future[ImageUpload] = { val sideEffectDependencies = ImageUploadOpsDependencies(toImageUploadOpsCfg(config), imageOps, - storeSource, storeThumbnail, storeOptimisedImage, fetchImageEmbedding = fetchImageEmbedding) + storeSource, storeThumbnail, storeOptimisedImage, fetchEmbeddingAndStore = fetchEmbeddingAndStore) Stopwatch.async("finalImage") { val finalImage = fromUploadRequestShared(uploadRequest, sideEffectDependencies, imageProcessor) finalImage.map(img => ImageUpload(uploadRequest, img)) } } - // TODO EM: this name feels wrong - private def fetchImageEmbedding(base64EncodedImage: String, imageId: String)(implicit logMarker: LogMarker): Future[PutVectorsResponse] = { - s3vectors.putVector(base64EncodedImage, imageId) + private def fetchEmbeddingAndStore(base64EncodedImage: String, imageId: String)(implicit logMarker: LogMarker): Future[PutVectorsResponse] = { + s3vectors.fetchEmbeddingAndStore(base64EncodedImage, imageId) } private def storeSource(storableOriginalImage: StorableOriginalImage) From bff866c233788a672f60ad8617be023fcec72279 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Tue, 7 Oct 2025 16:40:18 +0100 Subject: [PATCH 30/66] tidy up more logging --- .../com/gu/mediaservice/lib/aws/Bedrock.scala | 17 ++--------------- .../com/gu/mediaservice/lib/aws/S3Vectors.scala | 4 ---- image-loader/app/model/Uploader.scala | 15 +-------------- 3 files changed, 3 insertions(+), 33 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index 685ccf9631..b4ca445b52 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -35,23 +35,19 @@ class Bedrock(config: CommonConfig) override def isDev: Boolean = config.isDev val client: BedrockRuntimeAsyncClient = { - logger.info("Creating Bedrock client") withAWSCredentialsV2(BedrockRuntimeAsyncClient.builder()) .build() } private def createRequestBody(base64EncodedImage: String): InvokeModelRequest = { - logger.info("Creating request body") val body = Bedrock.BedrockRequest( input_type = "image", embedding_types = List("float"), images = List(s"data:image/jpg;base64,$base64EncodedImage") ) val jsonBody = Json.toJson(body).toString() - logger.info(s"Request body created, length: ${jsonBody.length}") val request: InvokeModelRequest = { - logger.info("Building request") InvokeModelRequest .builder() .accept("*/*") @@ -66,13 +62,12 @@ class Bedrock(config: CommonConfig) private def sendBedrockEmbeddingRequest(base64EncodedImage: String)( implicit logMarker: LogMarker ): CompletableFuture[InvokeModelResponse] = { - logger.info(logMarker, "Starting fetchEmbedding call") try { val future = client.invokeModel(createRequestBody(base64EncodedImage)) - logger.info(logMarker, "Bedrock API call initiated") future.whenComplete((response, error) => { if (error != null) { - logger.error(logMarker, "Bedrock API call failed", error) + logger.error(logMarker, "Exception during Bedrock API call", error) + throw error } else { logger.info( logMarker, @@ -89,23 +84,15 @@ class Bedrock(config: CommonConfig) } def createImageEmbedding(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] = { - logger.info( - logMarker, - s"Starting image embedding creation" - ) val bedrockFuture = sendBedrockEmbeddingRequest(base64EncodedImage) - bedrockFuture.asScala .map { response => logger.info( logMarker, s"Received Bedrock response. Status: ${response.sdkHttpResponse().statusCode()}" ) - // Now we can safely access response.body() val responseBody = response.body().asUtf8String() - logger.info(logMarker, s"Response body: $responseBody") val json = Json.parse(responseBody) - logger.info(logMarker, s"Parsed JSON response: $json") // Extract the embeddings array (first element since it's an array of arrays) val embeddings = (json \ "embeddings" \ "float")(0).as[List[Float]] logger.info( diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index e2fa2a70a8..63c9e73b05 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -21,14 +21,11 @@ class S3Vectors(config: CommonConfig) override def awsRegionV2: Region = Region.EU_CENTRAL_1 val client: S3VectorsClient = { - logger.info("Creating Bedrock client") withAWSCredentialsV2(S3VectorsClient.builder()) .build() } private def createRequestBody(embedding: List[Float], imageId: String): PutVectorsRequest = { - logger.info("Creating request body for S3 Vector Store...") - val vectorData: VectorData = VectorData .builder() // TODO find out if we can do something less upsetting than this float conversion @@ -61,7 +58,6 @@ class S3Vectors(config: CommonConfig) ): PutVectorsResponse = { try { val input = createRequestBody(bedrockEmbedding, imageId) - logger.info("Creating S3 Vector ") val response = client.putVectors(input) logger.info( logMarker, diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index ded62ecaa7..7c5ea1f085 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -160,20 +160,7 @@ object Uploader extends GridLogging { val sourceStoreFuture = storeOrProjectOriginalFile(storableOriginalImage) val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, tempDirForRequest, deps) -// TODO why is this not working???? - val filePath = uploadRequest.tempFile.toPath - logger.info(s"filePath: ${filePath}") - - val fileBytes = Files.readAllBytes(filePath) - logger.info(s"fileBytes length: ${fileBytes.length}") - - val base64EncodedString = Base64.getEncoder().encodeToString(fileBytes) - logger.info(s"base64EncodedString: ${base64EncodedString.length}") - -// val imageBase64: String = Base64.getEncoder().encodeToString(Files.readAllBytes(uploadRequest.tempFile.toPath)) -// logger.info("imageBase64 from uploader: ", Files.readAllBytes(uploadRequest.tempFile.toPath)) - - // TODO: check that uploadRequest.imageId is the eventual image id in elasticsearch etc + val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(uploadRequest.tempFile.toPath)) val eventualImageEmbedding = fetchEmbeddingAndStore(base64EncodedString, uploadRequest.imageId) val eventualImage = for { From 333854c952a6374b8c882e966533b08bcbffb7d1 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 8 Oct 2025 11:48:00 +0100 Subject: [PATCH 31/66] tidy up error handling --- .../com/gu/mediaservice/lib/aws/Bedrock.scala | 57 ++++++++----------- .../gu/mediaservice/lib/aws/S3Vectors.scala | 3 +- image-loader/app/model/Uploader.scala | 8 ++- vectorcommand.sh | 2 +- 4 files changed, 32 insertions(+), 38 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index b4ca445b52..a5511b3a1d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -34,8 +34,8 @@ class Bedrock(config: CommonConfig) override def isDev: Boolean = config.isDev - val client: BedrockRuntimeAsyncClient = { - withAWSCredentialsV2(BedrockRuntimeAsyncClient.builder()) + val client: BedrockRuntimeClient = { + withAWSCredentialsV2(BedrockRuntimeClient.builder()) .build() } @@ -61,22 +61,16 @@ class Bedrock(config: CommonConfig) private def sendBedrockEmbeddingRequest(base64EncodedImage: String)( implicit logMarker: LogMarker - ): CompletableFuture[InvokeModelResponse] = { + ): InvokeModelResponse = { try { - val future = client.invokeModel(createRequestBody(base64EncodedImage)) - future.whenComplete((response, error) => { - if (error != null) { - logger.error(logMarker, "Exception during Bedrock API call", error) - throw error - } else { - logger.info( - logMarker, - s"Bedrock API call completed with status: ${response.sdkHttpResponse().statusCode()}" - ) - } - }) - future - } catch { + val response = client.invokeModel(createRequestBody(base64EncodedImage)) + logger.info( + logMarker, + s"Bedrock API call completed with status: ${response.sdkHttpResponse().statusCode()}" + ) + response + } + catch { case e: Exception => logger.error(logMarker, "Exception during Bedrock API call", e) throw e @@ -84,22 +78,17 @@ class Bedrock(config: CommonConfig) } def createImageEmbedding(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] = { - val bedrockFuture = sendBedrockEmbeddingRequest(base64EncodedImage) - bedrockFuture.asScala - .map { response => - logger.info( - logMarker, - s"Received Bedrock response. Status: ${response.sdkHttpResponse().statusCode()}" - ) - val responseBody = response.body().asUtf8String() - val json = Json.parse(responseBody) - // Extract the embeddings array (first element since it's an array of arrays) - val embeddings = (json \ "embeddings" \ "float")(0).as[List[Float]] - logger.info( - logMarker, - s"Successfully extracted embeddings. Vector size: ${embeddings.size}" - ) - embeddings - } + val bedrockFuture = Future { sendBedrockEmbeddingRequest(base64EncodedImage) } + bedrockFuture.map { response => + val responseBody = response.body().asUtf8String() + val json = Json.parse(responseBody) + // Extract the embeddings array (first element since it's an array of arrays) + val embeddings = (json \ "embeddings" \ "float")(0).as[List[Float]] + logger.info( + logMarker, + s"Successfully extracted embeddings. Vector size: ${embeddings.size}" + ) + embeddings + } } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index 63c9e73b05..f8a9791fdf 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -18,6 +18,7 @@ class S3Vectors(config: CommonConfig) override def isDev: Boolean = config.isDev +// The S3 Vector Store is not yet available in eu-west-1, so we are using eu-central-1 because it's closest to us. override def awsRegionV2: Region = Region.EU_CENTRAL_1 val client: S3VectorsClient = { @@ -67,7 +68,7 @@ class S3Vectors(config: CommonConfig) } catch { case e: Exception => - logger.error(logMarker, "Exception during S3 Vector Store API call", e) + logger.error(logMarker, s"Exception during S3 Vector Store API call for ImageId $imageId: ", e) throw e } } diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index 7c5ea1f085..d00886401a 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -161,7 +161,12 @@ object Uploader extends GridLogging { val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, tempDirForRequest, deps) val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(uploadRequest.tempFile.toPath)) - val eventualImageEmbedding = fetchEmbeddingAndStore(base64EncodedString, uploadRequest.imageId) + +// We are fetching the embedding and storing it in the S3Vectors bucket +// This should not block the image upload process on failure + fetchEmbeddingAndStore(base64EncodedString, uploadRequest.imageId).failed.foreach { failure => + logger.error(logMarker, s"Failed to fetch embedding for ${uploadRequest.imageId} and store", failure) + } val eventualImage = for { browserViewableImage <- eventualBrowserViewableImage @@ -178,7 +183,6 @@ object Uploader extends GridLogging { case Some(storableOptimisedImage) => storeOrProjectOptimisedFile(storableOptimisedImage).map(a=>Some(a)) case None => Future.successful(None) } - imageEmbedding <- eventualImageEmbedding thumbDimensions <- FileMetadataReader.dimensions(thumbViewableImage.file, Some(thumbViewableImage.mimeType)) colourModel <- colourModelFuture } yield { diff --git a/vectorcommand.sh b/vectorcommand.sh index 72d50b78c8..60dc3d3ef9 100644 --- a/vectorcommand.sh +++ b/vectorcommand.sh @@ -1,7 +1,7 @@ #!/bin/bash aws s3vectors list-vectors \ - --vector-bucket-name image-embeddings-test \ + --vector-bucket-name image-embeddings-dev \ --index-name cohere-embed-english-v3 \ --profile media-service \ --region eu-central-1 From a6179893ed89f0b09313cea2b58ae6954cc96d0b Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 8 Oct 2025 16:48:24 +0100 Subject: [PATCH 32/66] update tests --- image-loader/test/scala/model/ImageUploadTest.scala | 13 +++++++++++-- image-loader/test/scala/model/ProjectorTest.scala | 12 ++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index 35859abf4b..0c2b36618e 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -17,6 +17,7 @@ import org.scalatest.Assertion import org.scalatest.funsuite.AsyncFunSuite import org.scalatest.matchers.should.Matchers import org.scalatestplus.mockito.MockitoSugar +import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import test.lib.ResourceHelpers import scala.concurrent.{ExecutionContext, Future} @@ -56,16 +57,23 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None) ) + def mockVectorStore = (imageBase64: String, imageId: String) => + Future.successful( + PutVectorsResponse + ) + def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore def storeOrProjectThumbFile: StorableThumbImage => Future[S3Object] = mockStore def storeOrProjectOptimisedPNG: StorableOptimisedImage => Future[S3Object] = mockStore + def fetchEmbeddingAndStore: (String, String) => Future[PutVectorsResponse] = mockVectorStore val mockDependencies = ImageUploadOpsDependencies( mockConfig, imageOps, storeOrProjectOriginalFile, storeOrProjectThumbFile, - storeOrProjectOptimisedPNG + storeOrProjectOptimisedPNG, + fetchEmbeddingAndStore = fetchEmbeddingAndStore ) val tempFile = ResourceHelpers.fileAt(fileName) @@ -85,11 +93,12 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { mockDependencies.storeOrProjectOriginalFile, mockDependencies.storeOrProjectThumbFile, mockDependencies.storeOrProjectOptimisedImage, + fetchEmbeddingAndStore, OptimiseWithPngQuant, uploadRequest, mockDependencies, FileMetadata(), - ImageProcessor.identity + ImageProcessor.identity, ) // Assertions; Failure will auto-fail diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index 3779c6ff90..5ba8a5eeb4 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -7,6 +7,7 @@ import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.model.ObjectMetadata import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication +import com.gu.mediaservice.lib.aws.S3Vectors import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} @@ -14,6 +15,7 @@ import com.gu.mediaservice.model._ import com.gu.mediaservice.model.leases.LeasesByMedia import lib.DigestedFile import org.joda.time.{DateTime, DateTimeZone} +import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{times, verify, when} import org.scalatest.concurrent.ScalaFutures import org.scalatest.freespec.AnyFreeSpec @@ -21,11 +23,12 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Span} import org.scalatestplus.mockito.MockitoSugar import play.api.libs.json.{JsArray, JsString} +import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import test.lib.ResourceHelpers import scala.concurrent.ExecutionContext.Implicits.global import scala.jdk.CollectionConverters._ -import scala.concurrent.Future +import scala.concurrent.{ExecutionContext, Future} class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with MockitoSugar { @@ -39,9 +42,14 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket") + private val mockPutVectorsResponse = mock[PutVectorsResponse] + private val s3vectors = mock[S3Vectors] + when(s3vectors.fetchEmbeddingAndStore(any[String], any[String])(any[ExecutionContext], any[LogMarker])) + .thenReturn(Future.successful(mockPutVectorsResponse)) + private val s3 = mock[AmazonS3] private val auth = mock[Authentication] - private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth) + private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, s3vectors) // FIXME temporary ignored as test is not executable in CI/CD machine // because graphic lib files like srgb.icc, cmyk.icc are in root directory instead of resources From 7da7c0dd250ece293929f97bc5ae0d514a0c4716 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Thu, 9 Oct 2025 11:29:26 +0100 Subject: [PATCH 33/66] sort out mock (hopefully?) --- image-loader/test/scala/model/ImageUploadTest.scala | 3 ++- image-loader/test/scala/model/ProjectorTest.scala | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index 0c2b36618e..67b09dff84 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -57,9 +57,10 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None) ) + val mockPutVectorsResponse = mock[PutVectorsResponse] def mockVectorStore = (imageBase64: String, imageId: String) => Future.successful( - PutVectorsResponse + mockPutVectorsResponse ) def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index 5ba8a5eeb4..c8b1014a50 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -42,7 +42,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket") - private val mockPutVectorsResponse = mock[PutVectorsResponse] + private val mockPutVectorsResponse = PutVectorsResponse.builder().build() private val s3vectors = mock[S3Vectors] when(s3vectors.fetchEmbeddingAndStore(any[String], any[String])(any[ExecutionContext], any[LogMarker])) .thenReturn(Future.successful(mockPutVectorsResponse)) From 828f8ca7dc32122e23302d4f3c593b5250d8188e Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 9 Oct 2025 12:33:09 +0100 Subject: [PATCH 34/66] Fix an issue where typing e.g. -sport{Enter} would not-dechip the input to produce the plain query -sport Caused by an import that was inadvertently removed without error --- kahuna/public/js/search/structured-query/structured-query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kahuna/public/js/search/structured-query/structured-query.js b/kahuna/public/js/search/structured-query/structured-query.js index 9aeb0a08ef..4824d7f361 100644 --- a/kahuna/public/js/search/structured-query/structured-query.js +++ b/kahuna/public/js/search/structured-query/structured-query.js @@ -51,7 +51,7 @@ grStructuredQuery.controller("grStructuredQueryCtrl", [ .debounce(500); ctrl.getSuggestions = querySuggestions.getChipSuggestions; - ctrl.filterFields = querySuggestions.filterFields; + ctrl.filterFields = querySuggestions.typeaheadFields.map(_ => _.name); function valOrUndefined(str) { // Watch out for `false`, but we know it's a string here.. From d1a3401cadee2a89f34032edc867944251f4cd01 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 9 Oct 2025 13:47:03 +0100 Subject: [PATCH 35/66] Correct name -> fieldName --- kahuna/public/js/search/structured-query/structured-query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kahuna/public/js/search/structured-query/structured-query.js b/kahuna/public/js/search/structured-query/structured-query.js index 4824d7f361..6583d4defc 100644 --- a/kahuna/public/js/search/structured-query/structured-query.js +++ b/kahuna/public/js/search/structured-query/structured-query.js @@ -51,7 +51,7 @@ grStructuredQuery.controller("grStructuredQueryCtrl", [ .debounce(500); ctrl.getSuggestions = querySuggestions.getChipSuggestions; - ctrl.filterFields = querySuggestions.typeaheadFields.map(_ => _.name); + ctrl.filterFields = querySuggestions.typeaheadFields.map(_ => _.fieldName); function valOrUndefined(str) { // Watch out for `false`, but we know it's a string here.. From 7f91ce9e6b48ab81c9e95a59328e15e839fff1ed Mon Sep 17 00:00:00 2001 From: Mateusz Date: Fri, 10 Oct 2025 13:14:52 +0100 Subject: [PATCH 36/66] Persist state of Apply circular mask square crop checkbox in localStorage --- kahuna/public/js/crop/controller.js | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/kahuna/public/js/crop/controller.js b/kahuna/public/js/crop/controller.js index 70a7299cf8..b976e3fedf 100644 --- a/kahuna/public/js/crop/controller.js +++ b/kahuna/public/js/crop/controller.js @@ -4,11 +4,13 @@ import '../components/gr-keyboard-shortcut/gr-keyboard-shortcut'; import {radioList} from '../components/gr-radio-list/gr-radio-list'; import {cropUtil} from "../util/crop"; import {cropOptions} from "../util/constants/cropOptions"; +import {storage as storageModule} from "../util/storage"; const crop = angular.module('kahuna.crop.controller', [ 'gr.keyboardShortcut', radioList.name, - cropUtil.name + cropUtil.name, + storageModule.name ]); crop.controller('ImageCropCtrl', [ @@ -26,6 +28,7 @@ crop.controller('ImageCropCtrl', [ 'square', 'freeform', 'pollUntilCropCreated', + 'storage', function( $scope, $rootScope, @@ -40,11 +43,27 @@ crop.controller('ImageCropCtrl', [ cropSettings, square, freeform, - pollUntilCropCreated) { + pollUntilCropCreated, + storage) { const ctrl = this; const imageId = $stateParams.imageId; + const circularMaskKey = 'crop.shouldUseCircularMask'; + try { + const stored = storage.getJs(circularMaskKey); + if (typeof stored === 'boolean') { + ctrl.shouldUseCircularMask = stored; + } + } catch (_) { + } + + $scope.$watch(() => ctrl.shouldUseCircularMask, (val) => { + if (typeof val === 'boolean') { + storage.setJs(circularMaskKey, val); + } + }); + cropSettings.set($stateParams); const allCropOptions = cropSettings.getCropOptions(); @@ -231,4 +250,3 @@ crop.controller('ImageCropCtrl', [ }); }); }]); - From d0ad13eb0dd92d7b93dd2e8014a0eeba7f7bca56 Mon Sep 17 00:00:00 2001 From: Mateusz Date: Fri, 10 Oct 2025 17:22:53 +0100 Subject: [PATCH 37/66] Catch error in console Co-authored-by: Tom Richards --- kahuna/public/js/crop/controller.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kahuna/public/js/crop/controller.js b/kahuna/public/js/crop/controller.js index b976e3fedf..168cdd0245 100644 --- a/kahuna/public/js/crop/controller.js +++ b/kahuna/public/js/crop/controller.js @@ -55,7 +55,8 @@ crop.controller('ImageCropCtrl', [ if (typeof stored === 'boolean') { ctrl.shouldUseCircularMask = stored; } - } catch (_) { + } catch (e) { + console.error(`failed to store '${circularMaskKey}' to local storage', e); } $scope.$watch(() => ctrl.shouldUseCircularMask, (val) => { From 342235ced964ccb3dc73a9caf1a250f5de82904e Mon Sep 17 00:00:00 2001 From: Mateusz Date: Fri, 10 Oct 2025 17:23:39 +0100 Subject: [PATCH 38/66] Clearer code Co-authored-by: Tom Richards --- kahuna/public/js/crop/controller.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kahuna/public/js/crop/controller.js b/kahuna/public/js/crop/controller.js index 168cdd0245..10c38b8dc2 100644 --- a/kahuna/public/js/crop/controller.js +++ b/kahuna/public/js/crop/controller.js @@ -59,9 +59,9 @@ crop.controller('ImageCropCtrl', [ console.error(`failed to store '${circularMaskKey}' to local storage', e); } - $scope.$watch(() => ctrl.shouldUseCircularMask, (val) => { - if (typeof val === 'boolean') { - storage.setJs(circularMaskKey, val); + $scope.$watch(() => ctrl.shouldUseCircularMask, (shouldUseCircularMask) => { + if (typeof shouldUseCircularMask === 'boolean') { + storage.setJs(circularMaskKey, shouldUseCircularMask); } }); From a0c86a8faa3d24fe9495286c4910a146017b435e Mon Sep 17 00:00:00 2001 From: Mateusz Date: Fri, 10 Oct 2025 17:37:44 +0100 Subject: [PATCH 39/66] =?UTF-8?q?Don=E2=80=99t=20trust=20anyone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kahuna/public/js/crop/controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kahuna/public/js/crop/controller.js b/kahuna/public/js/crop/controller.js index 10c38b8dc2..370278f599 100644 --- a/kahuna/public/js/crop/controller.js +++ b/kahuna/public/js/crop/controller.js @@ -56,7 +56,7 @@ crop.controller('ImageCropCtrl', [ ctrl.shouldUseCircularMask = stored; } } catch (e) { - console.error(`failed to store '${circularMaskKey}' to local storage', e); + console.error(`failed to store '${circularMaskKey}' to local storage`, e); } $scope.$watch(() => ctrl.shouldUseCircularMask, (shouldUseCircularMask) => { From 35e252b22b6c6550763984e6f3879072bbf78f30 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 13 Oct 2025 10:53:12 +0100 Subject: [PATCH 40/66] Bump CQL to 1.8.2 To fix critical vulnerability https://github.com/guardian/cql/security/dependabot/35 --- kahuna/package-lock.json | 788 +++++++-------------------------------- kahuna/package.json | 2 +- 2 files changed, 129 insertions(+), 661 deletions(-) diff --git a/kahuna/package-lock.json b/kahuna/package-lock.json index 33fd57b829..9a6893d467 100644 --- a/kahuna/package-lock.json +++ b/kahuna/package-lock.json @@ -5,11 +5,11 @@ "packages": { "": { "dependencies": { - "@babel/polyfill": "^7.8.7", - "@guardian/cql": "^1.8.1", - "@guardian/user-telemetry-client": "^1.1.0", - "@sentry/browser": "^6.11.0", - "@sentry/integrations": "^6.11.0", + "@babel/polyfill": "7.8.7", + "@guardian/cql": "1.8.2", + "@guardian/user-telemetry-client": "1.1.0", + "@sentry/browser": "6.11.0", + "@sentry/integrations": "6.11.0", "angular": "1.8.3", "angular-animate": "1.8.3", "angular-cookies": "1.8.3", @@ -2103,72 +2103,11 @@ "node": ">=10.0.0" } }, - "node_modules/@eslint-community/eslint-utils": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.7.0.tgz", - "integrity": "sha512-dyybb3AcajC7uha6CvhdVRJqaKyn7w2YKqKyAN37NKYgZT36w+iRb0Dymmc5qEJ549c/S31cMMSFd75bteCpCw==", - "license": "MIT", - "dependencies": { - "eslint-visitor-keys": "^3.4.3" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - }, - "peerDependencies": { - "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" - } - }, - "node_modules/@eslint-community/regexpp": { - "version": "4.12.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", - "integrity": "sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==", - "license": "MIT", - "engines": { - "node": "^12.0.0 || ^14.0.0 || >=16.0.0" - } - }, - "node_modules/@eslint/config-array": { - "version": "0.21.0", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.0.tgz", - "integrity": "sha512-ENIdc4iLu0d93HeYirvKmrzshzofPw6VkZRKQGe9Nv46ZnWUzcF1xV01dcvEg/1wXUR61OmmlSfyeyO7EvjLxQ==", - "license": "Apache-2.0", - "dependencies": { - "@eslint/object-schema": "^2.1.6", - "debug": "^4.3.1", - "minimatch": "^3.1.2" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/config-helpers": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.3.1.tgz", - "integrity": "sha512-xR93k9WhrDYpXHORXpxVL5oHj3Era7wo6k/Wd8/IsQNnZUTzkGS29lyn3nAT05v6ltUuTFVCCYDEGfy2Or/sPA==", - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/core": { - "version": "0.15.2", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.15.2.tgz", - "integrity": "sha512-78Md3/Rrxh83gCxoUc0EiciuOHsIITzLy53m3d9UyiW8y9Dj2D29FeETqyKA+BRK76tnTp6RXWb3pCay8Oyomg==", - "license": "Apache-2.0", - "dependencies": { - "@types/json-schema": "^7.0.15" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, "node_modules/@eslint/eslintrc": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.0.5.tgz", "integrity": "sha512-BLxsnmK3KyPunz5wmCCpqy0YelEoxxGmH73Is+Z74oOTMtExcjkr3dDR6quwrjh1YspA8DH9gnX1o069KiS9AQ==", + "dev": true, "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", @@ -2187,12 +2126,14 @@ "node_modules/@eslint/eslintrc/node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true }, "node_modules/@eslint/eslintrc/node_modules/globals": { "version": "13.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.12.0.tgz", "integrity": "sha512-uS8X6lSKN2JumVoXrbUz+uG4BYG+eiawqm3qFcT7ammfbUHeCBoJMlHcec/S3krSk73/AE/f0szYFmgAA3kYZg==", + "dev": true, "dependencies": { "type-fest": "^0.20.2" }, @@ -2207,6 +2148,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, "dependencies": { "argparse": "^2.0.1" }, @@ -2214,40 +2156,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/@eslint/js": { - "version": "9.33.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.33.0.tgz", - "integrity": "sha512-5K1/mKhWaMfreBGJTwval43JJmkip0RmM+3+IuqupeSKNC/Th2Kc7ucaq5ovTSra/OOKB9c58CGSz3QMVbWt0A==", - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" - } - }, - "node_modules/@eslint/object-schema": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.6.tgz", - "integrity": "sha512-RBMg5FRL0I0gs51M/guSAj5/e14VQ4tpZnQNWwuDT66P14I43ItmPfIZRhO9fUVIPOAQXU47atlywZ/czoqFPA==", - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/plugin-kit": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.3.5.tgz", - "integrity": "sha512-Z5kJ+wU3oA7MMIqVR9tyZRtjYPr4OC004Q4Rw7pgOKUOKkJfZ3O24nz3WYfGRpMDNmcOi3TwQOmgm7B7Tpii0w==", - "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^0.15.2", - "levn": "^0.4.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, "node_modules/@floating-ui/core": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.3.tgz", @@ -2274,17 +2182,12 @@ "license": "MIT" }, "node_modules/@guardian/cql": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@guardian/cql/-/cql-1.8.1.tgz", - "integrity": "sha512-eb4eJiK0o98C5RLVXAT1IA/hBdanHya1/GoyW3dh5bJL61iPKxrc+x9x9CB+wql/xeOfF4ZC4DisH/dr2g1vrg==", + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@guardian/cql/-/cql-1.8.2.tgz", + "integrity": "sha512-SnWJ7zmUZgOHt2+MSQcN/md1XlGU8x4XUmIG6ZOZtcg96LOvTEu858VEjaL9bV7A325WxJISkjGK5S2yrkdmEQ==", "dependencies": { "@floating-ui/dom": "^1.6.11", - "eslint": "^9.14.0", - "eslint-config-prettier": "^9.1.0", - "eslint-plugin-prettier": "^5.2.1", - "happy-dom": "^15.11.7", "preact": "^10.25.4", - "prettier": "^3.5.3", "prosemirror-commands": "^1.6.0", "prosemirror-history": "^1.4.1", "prosemirror-keymap": "^1.2.2", @@ -2295,338 +2198,6 @@ "prosemirror-view": "^1.34.3" } }, - "node_modules/@guardian/cql/node_modules/@eslint/eslintrc": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.1.tgz", - "integrity": "sha512-gtF186CXhIl1p4pJNGZw8Yc6RlshoePRvE0X91oPGb3vZ8pM3qOS9W9NGPat9LziaBV7XrJWGylNQXkGcnM3IQ==", - "license": "MIT", - "dependencies": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^10.0.1", - "globals": "^14.0.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@guardian/cql/node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", - "license": "MIT" - }, - "node_modules/@guardian/cql/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@guardian/cql/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0" - }, - "node_modules/@guardian/cql/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/@guardian/cql/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/@guardian/cql/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/@guardian/cql/node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@guardian/cql/node_modules/eslint": { - "version": "9.33.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.33.0.tgz", - "integrity": "sha512-TS9bTNIryDzStCpJN93aC5VRSW3uTx9sClUn4B87pwiCaJh220otoI0X8mJKr+VcPtniMdN8GKjlwgWGUv5ZKA==", - "license": "MIT", - "dependencies": { - "@eslint-community/eslint-utils": "^4.2.0", - "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.21.0", - "@eslint/config-helpers": "^0.3.1", - "@eslint/core": "^0.15.2", - "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.33.0", - "@eslint/plugin-kit": "^0.3.5", - "@humanfs/node": "^0.16.6", - "@humanwhocodes/module-importer": "^1.0.1", - "@humanwhocodes/retry": "^0.4.2", - "@types/estree": "^1.0.6", - "@types/json-schema": "^7.0.15", - "ajv": "^6.12.4", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.6", - "debug": "^4.3.2", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.4.0", - "eslint-visitor-keys": "^4.2.1", - "espree": "^10.4.0", - "esquery": "^1.5.0", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^8.0.0", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3" - }, - "bin": { - "eslint": "bin/eslint.js" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" - }, - "peerDependencies": { - "jiti": "*" - }, - "peerDependenciesMeta": { - "jiti": { - "optional": true - } - } - }, - "node_modules/@guardian/cql/node_modules/eslint-scope": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", - "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", - "license": "BSD-2-Clause", - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@guardian/cql/node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@guardian/cql/node_modules/espree": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", - "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", - "license": "BSD-2-Clause", - "dependencies": { - "acorn": "^8.15.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@guardian/cql/node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/@guardian/cql/node_modules/file-entry-cache": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", - "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", - "license": "MIT", - "dependencies": { - "flat-cache": "^4.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@guardian/cql/node_modules/find-up": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", - "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", - "license": "MIT", - "dependencies": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@guardian/cql/node_modules/flat-cache": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", - "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", - "license": "MIT", - "dependencies": { - "flatted": "^3.2.9", - "keyv": "^4.5.4" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/@guardian/cql/node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@guardian/cql/node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/@guardian/cql/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/@guardian/cql/node_modules/locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", - "license": "MIT", - "dependencies": { - "p-locate": "^5.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@guardian/cql/node_modules/p-locate": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", - "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", - "license": "MIT", - "dependencies": { - "p-limit": "^3.0.2" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@guardian/cql/node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@guardian/cql/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/@guardian/user-telemetry-client": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@guardian/user-telemetry-client/-/user-telemetry-client-1.1.0.tgz", @@ -2635,45 +2206,11 @@ "lodash": "^4.17.20" } }, - "node_modules/@humanfs/core": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", - "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", - "license": "Apache-2.0", - "engines": { - "node": ">=18.18.0" - } - }, - "node_modules/@humanfs/node": { - "version": "0.16.6", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.6.tgz", - "integrity": "sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw==", - "license": "Apache-2.0", - "dependencies": { - "@humanfs/core": "^0.19.1", - "@humanwhocodes/retry": "^0.3.0" - }, - "engines": { - "node": ">=18.18.0" - } - }, - "node_modules/@humanfs/node/node_modules/@humanwhocodes/retry": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.3.1.tgz", - "integrity": "sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==", - "license": "Apache-2.0", - "engines": { - "node": ">=18.18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, "node_modules/@humanwhocodes/config-array": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.2.tgz", "integrity": "sha512-UXOuFCGcwciWckOpmfKDq/GyhlTf9pN/BzG//x8p8zTOFEcGuA68ANXheFS0AGvy3qgZqLBUkMs7hqzqCKOVwA==", + "dev": true, "dependencies": { "@humanwhocodes/object-schema": "^1.2.1", "debug": "^4.1.1", @@ -2683,36 +2220,11 @@ "node": ">=10.10.0" } }, - "node_modules/@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "license": "Apache-2.0", - "engines": { - "node": ">=12.22" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, "node_modules/@humanwhocodes/object-schema": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", - "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==" - }, - "node_modules/@humanwhocodes/retry": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", - "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=18.18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } + "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", + "dev": true }, "node_modules/@isaacs/cliui": { "version": "8.0.2", @@ -3773,6 +3285,7 @@ "version": "0.2.7", "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.7.tgz", "integrity": "sha512-YLT9Zo3oNPJoBjBc4q8G2mjU4tqIbf5CEOORbUUr48dCD9q3umJ3IPlVqOqDakPfd2HuwccBaqlGhN4Gmr5OWg==", + "dev": true, "license": "MIT", "engines": { "node": "^12.20.0 || ^14.18.0 || >=16.0.0" @@ -3962,7 +3475,7 @@ "version": "8.21.2", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.21.2.tgz", "integrity": "sha512-EMpxUyystd3uZVByZap1DACsMXvb82ypQnGn89e1Y0a+LYu3JJscUd/gqhRsVFDkaD2MIiWo0MT8EfXr3DGRKw==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "@types/estree": "*", @@ -3983,7 +3496,7 @@ "version": "0.0.51", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.51.tgz", "integrity": "sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ==", - "devOptional": true + "dev": true }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", @@ -4026,6 +3539,7 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, "license": "MIT" }, "node_modules/@types/lodash": { @@ -4572,6 +4086,7 @@ "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, "bin": { "acorn": "bin/acorn" }, @@ -4592,6 +4107,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } @@ -4609,6 +4125,7 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -4736,6 +4253,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "dev": true, "engines": { "node": ">=6" } @@ -4772,6 +4290,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, "engines": { "node": ">=8" } @@ -5426,7 +4945,8 @@ "node_modules/balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true }, "node_modules/base": { "version": "0.11.2", @@ -5521,6 +5041,7 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -5648,6 +5169,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, "engines": { "node": ">=6" } @@ -5974,7 +5496,8 @@ "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true }, "node_modules/convert-source-map": { "version": "2.0.0", @@ -6073,6 +5596,7 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -6086,6 +5610,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -6250,6 +5775,7 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -6305,7 +5831,8 @@ "node_modules/deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", - "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true }, "node_modules/deepmerge": { "version": "4.3.1", @@ -6414,6 +5941,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, "license": "MIT", "dependencies": { "esutils": "^2.0.2" @@ -6605,6 +6133,7 @@ "version": "2.3.6", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", + "dev": true, "dependencies": { "ansi-colors": "^4.1.1" }, @@ -6685,6 +6214,7 @@ "version": "8.6.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.6.0.tgz", "integrity": "sha512-UvxdOJ7mXFlw7iuHZA4jmzPaUqIw54mZrv+XPYKNbKdLR0et4rf60lIZUU9kiNtnzzMzGWxMV+tQ7uG7JG8DPw==", + "dev": true, "license": "MIT", "dependencies": { "@eslint/eslintrc": "^1.0.5", @@ -6736,48 +6266,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/eslint-config-prettier": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.2.tgz", - "integrity": "sha512-iI1f+D2ViGn+uvv5HuHVUamg8ll4tN+JRHGc6IJi4TP9Kl976C57fzPXgseXNs8v0iA8aSJpHsTWjDb9QJamGQ==", - "license": "MIT", - "bin": { - "eslint-config-prettier": "bin/cli.js" - }, - "peerDependencies": { - "eslint": ">=7.0.0" - } - }, - "node_modules/eslint-plugin-prettier": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.5.4.tgz", - "integrity": "sha512-swNtI95SToIz05YINMA6Ox5R057IMAmWZ26GqPxusAp1TZzj+IdY9tXNWWD3vkF/wEqydCONcwjTFpxybBqZsg==", - "license": "MIT", - "dependencies": { - "prettier-linter-helpers": "^1.0.0", - "synckit": "^0.11.7" - }, - "engines": { - "node": "^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint-plugin-prettier" - }, - "peerDependencies": { - "@types/eslint": ">=8.0.0", - "eslint": ">=8.0.0", - "eslint-config-prettier": ">= 7.0.0 <10.0.0 || >=10.1.0", - "prettier": ">=3.0.0" - }, - "peerDependenciesMeta": { - "@types/eslint": { - "optional": true - }, - "eslint-config-prettier": { - "optional": true - } - } - }, "node_modules/eslint-plugin-react": { "version": "7.28.0", "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.28.0.tgz", @@ -6855,6 +6343,7 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.0.tgz", "integrity": "sha512-aWwkhnS0qAXqNOgKOK0dJ2nvzEbhEvpy8OlJ9kZ0FeZnA6zpjv1/Vei+puGFFX7zkPCkHHXb7IDX3A+7yPrRWg==", + "dev": true, "license": "MIT", "dependencies": { "esrecurse": "^4.3.0", @@ -6868,6 +6357,7 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, "engines": { "node": ">=4.0" } @@ -6876,6 +6366,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", + "dev": true, "license": "MIT", "dependencies": { "eslint-visitor-keys": "^2.0.0" @@ -6894,6 +6385,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", + "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -6903,6 +6395,7 @@ "version": "3.4.3", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, "license": "Apache-2.0", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -6915,6 +6408,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "color-convert": "^2.0.1" @@ -6929,12 +6423,14 @@ "node_modules/eslint/node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true }, "node_modules/eslint/node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -6950,6 +6446,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "color-name": "~1.1.4" @@ -6961,12 +6458,14 @@ "node_modules/eslint/node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "node_modules/eslint/node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -6979,6 +6478,7 @@ "version": "13.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.12.0.tgz", "integrity": "sha512-uS8X6lSKN2JumVoXrbUz+uG4BYG+eiawqm3qFcT7ammfbUHeCBoJMlHcec/S3krSk73/AE/f0szYFmgAA3kYZg==", + "dev": true, "license": "MIT", "dependencies": { "type-fest": "^0.20.2" @@ -6994,6 +6494,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, "engines": { "node": ">=8" } @@ -7002,6 +6503,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -7014,6 +6516,7 @@ "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "prelude-ls": "^1.2.1", @@ -7027,6 +6530,7 @@ "version": "7.6.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", + "dev": true, "bin": { "semver": "bin/semver.js" }, @@ -7038,6 +6542,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -7050,6 +6555,7 @@ "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1" @@ -7062,6 +6568,7 @@ "version": "9.3.0", "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.0.tgz", "integrity": "sha512-d/5nCsb0JcqsSEeQzFZ8DH1RmxPcglRWh24EFTlUEmCKoehXGdpsx0RkHDubqUI8LSAIKMQp4r9SzQ3n+sm4HQ==", + "dev": true, "dependencies": { "acorn": "^8.7.0", "acorn-jsx": "^5.3.1", @@ -7088,6 +6595,7 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "estraverse": "^5.1.0" @@ -7100,6 +6608,7 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, "license": "MIT", "engines": { "node": ">=4.0" @@ -7109,6 +6618,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, "license": "MIT", "dependencies": { "estraverse": "^5.2.0" @@ -7121,6 +6631,7 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, "engines": { "node": ">=4.0" } @@ -7138,6 +6649,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, "engines": { "node": ">=0.10.0" } @@ -7441,14 +6953,9 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, "license": "MIT" }, - "node_modules/fast-diff": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", - "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", - "license": "Apache-2.0" - }, "node_modules/fast-glob": { "version": "3.2.10", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.10.tgz", @@ -7481,12 +6988,14 @@ "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true }, "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true }, "node_modules/fast-xml-parser": { "version": "4.4.0", @@ -7540,6 +7049,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, "license": "MIT", "dependencies": { "flat-cache": "^3.0.4" @@ -7833,6 +7343,7 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", + "dev": true, "license": "MIT", "dependencies": { "flatted": "^3.1.0", @@ -7846,6 +7357,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, "dependencies": { "glob": "^7.1.3" }, @@ -7860,6 +7372,7 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "dev": true, "license": "ISC" }, "node_modules/for-in": { @@ -8076,7 +7589,8 @@ "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true }, "node_modules/fsevents": { "version": "2.3.3", @@ -8106,7 +7620,8 @@ "node_modules/functional-red-black-tree": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=" + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true }, "node_modules/gensync": { "version": "1.0.0-beta.2", @@ -8187,6 +7702,7 @@ "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, "license": "MIT", "dependencies": { "fs.realpath": "^1.0.0", @@ -8207,6 +7723,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, "license": "MIT", "dependencies": { "is-glob": "^4.0.3" @@ -8219,6 +7736,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -8276,18 +7794,6 @@ "which": "bin/which" } }, - "node_modules/globals": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", - "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/globby": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", @@ -8323,32 +7829,6 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true }, - "node_modules/happy-dom": { - "version": "15.11.7", - "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-15.11.7.tgz", - "integrity": "sha512-KyrFvnl+J9US63TEzwoiJOQzZBJY7KgBushJA8X61DMbNsH+2ONkDuLDnCnwUiPTF42tLoEmrPyoqbenVA5zrg==", - "license": "MIT", - "dependencies": { - "entities": "^4.5.0", - "webidl-conversions": "^7.0.0", - "whatwg-mimetype": "^3.0.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/happy-dom/node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -8753,6 +8233,7 @@ "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true, "engines": { "node": ">= 4" } @@ -8942,6 +8423,7 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, "license": "MIT", "dependencies": { "parent-module": "^1.0.0", @@ -8958,6 +8440,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -8987,6 +8470,7 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true, "engines": { "node": ">=0.8.19" } @@ -8995,6 +8479,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, "license": "MIT", "dependencies": { "once": "^1.3.0", @@ -9221,6 +8706,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9249,6 +8735,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -9429,7 +8916,8 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true }, "node_modules/isobject": { "version": "3.0.1", @@ -11679,12 +11167,6 @@ "node": ">=6" } }, - "node_modules/json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "license": "MIT" - }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", @@ -11703,12 +11185,14 @@ "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=" + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true }, "node_modules/json5": { "version": "2.2.3", @@ -11816,15 +11300,6 @@ "node": ">=8" } }, - "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "license": "MIT", - "dependencies": { - "json-buffer": "3.0.1" - } - }, "node_modules/kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -11860,6 +11335,7 @@ "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1", @@ -11982,7 +11458,8 @@ "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true }, "node_modules/loose-envify": { "version": "1.4.0", @@ -12177,6 +11654,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -12239,6 +11717,7 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, "license": "MIT" }, "node_modules/nanoid": { @@ -12300,7 +11779,8 @@ "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=" + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true }, "node_modules/neo-async": { "version": "2.6.2", @@ -12988,6 +12468,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, "dependencies": { "wrappy": "1" } @@ -13012,6 +12493,7 @@ "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, "license": "MIT", "dependencies": { "deep-is": "^0.1.3", @@ -13075,6 +12557,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, "dependencies": { "yocto-queue": "^0.1.0" }, @@ -13204,6 +12687,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, "dependencies": { "callsites": "^3.0.0" }, @@ -13317,6 +12801,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true, "engines": { "node": ">=0.10.0" } @@ -13589,38 +13074,12 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.8.0" } }, - "node_modules/prettier": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", - "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", - "license": "MIT", - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - }, - "node_modules/prettier-linter-helpers": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", - "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", - "license": "MIT", - "dependencies": { - "fast-diff": "^1.1.2" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/pretty-format": { "version": "30.0.2", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.2.tgz", @@ -13665,6 +13124,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true, "engines": { "node": ">=0.4.0" } @@ -13806,6 +13266,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, "engines": { "node": ">=6" } @@ -14020,6 +13481,7 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", + "dev": true, "engines": { "node": ">=8" }, @@ -14413,6 +13875,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -14424,6 +13887,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, "engines": { "node": ">=8" } @@ -15006,6 +14470,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -15059,6 +14524,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, "engines": { "node": ">=8" }, @@ -15171,6 +14637,7 @@ "version": "0.11.8", "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.8.tgz", "integrity": "sha512-+XZ+r1XGIJGeQk3VvXhT6xx/VpbHsRzsTkGgF6E5RX9TTXD0118l87puaEBZ566FhqblC6U0d4XnubznJDm30A==", + "dev": true, "license": "MIT", "dependencies": { "@pkgr/core": "^0.2.4" @@ -15351,7 +14818,8 @@ "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=" + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "dev": true }, "node_modules/theseus": { "version": "0.5.2", @@ -15705,6 +15173,7 @@ "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1" @@ -15727,6 +15196,7 @@ "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, "engines": { "node": ">=10" }, @@ -15975,6 +15445,7 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "dev": true, "dependencies": { "punycode": "^2.1.0" } @@ -16016,7 +15487,8 @@ "node_modules/v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", - "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==" + "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", + "dev": true }, "node_modules/v8-to-istanbul": { "version": "9.3.0", @@ -16079,6 +15551,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dev": true, "engines": { "node": ">=12" } @@ -16349,15 +15822,6 @@ "node": ">=18" } }, - "node_modules/whatwg-mimetype": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", - "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", - "license": "MIT", - "engines": { - "node": ">=12" - } - }, "node_modules/whatwg-url": { "version": "14.2.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", @@ -16376,6 +15840,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "dependencies": { "isexe": "^2.0.0" }, @@ -16418,6 +15883,7 @@ "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, "engines": { "node": ">=0.10.0" } @@ -16572,7 +16038,8 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true }, "node_modules/write-file-atomic": { "version": "5.0.1", @@ -16787,6 +16254,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=10" diff --git a/kahuna/package.json b/kahuna/package.json index de73fba305..72cafb3f83 100644 --- a/kahuna/package.json +++ b/kahuna/package.json @@ -1,7 +1,7 @@ { "dependencies": { "@babel/polyfill": "7.8.7", - "@guardian/cql": "1.8.1", + "@guardian/cql": "1.8.2", "@guardian/user-telemetry-client": "1.1.0", "@sentry/browser": "6.11.0", "@sentry/integrations": "6.11.0", From 4484f21f1adc93cdba196bec7c93fde4e714bf25 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Tue, 14 Oct 2025 12:13:42 +0100 Subject: [PATCH 41/66] add image embed config setting --- image-loader/app/ImageLoaderComponents.scala | 4 ++++ image-loader/app/lib/ImageLoaderConfig.scala | 2 ++ image-loader/app/lib/ImageLoaderStore.scala | 2 ++ 3 files changed, 8 insertions(+) diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index 219337ad51..8d747c721d 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -27,6 +27,10 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath) val notifications = new Notifications(config) val downloader = new Downloader()(ec,wsClient) + + val embedImage: Boolean = store.embedImage() + logger.info(s"embedImage = $embedImage") + val s3vectors = new S3Vectors(config) val uploader = new Uploader(store, config, imageOperations, notifications, s3vectors, imageProcessor) val projector = Projector(config, imageOperations, imageProcessor, auth, s3vectors) diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala index 8c85037fe9..ae683260c6 100644 --- a/image-loader/app/lib/ImageLoaderConfig.scala +++ b/image-loader/app/lib/ImageLoaderConfig.scala @@ -33,6 +33,8 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res val uploadStatusTable: String = string("dynamo.table.upload.status") val uploadStatusExpiry: FiniteDuration = configuration.get[FiniteDuration]("uploadStatus.recordExpiry") + val embedImages: Boolean = boolean("image.embed.images") + /** * Load in the chain of image processors from config. This can be a list of * companion objects, class names, both with and without config. diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 1c509f3712..db7adb5efe 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -24,6 +24,8 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati } } + def embedImage(): Boolean = config.embedImages + def getS3Object(key: String)(implicit logMarker: LogMarker): S3Object = handleNotFound(key) { client.getObject(config.maybeIngestBucket.get, key) } { From 1878a4f19436fd372ef6bcda7e88d0bd5946c4b7 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Tue, 14 Oct 2025 16:32:09 +0100 Subject: [PATCH 42/66] refactor into embedding class and rename fetch to create --- .../gu/mediaservice/lib/aws/Embedding.scala | 22 +++++++++++++ .../gu/mediaservice/lib/aws/S3Vectors.scala | 30 +++++++++--------- image-loader/app/ImageLoaderComponents.scala | 11 +++---- image-loader/app/lib/ImageLoaderConfig.scala | 2 +- image-loader/app/lib/ImageLoaderStore.scala | 2 -- image-loader/app/model/Projector.scala | 23 ++++++++------ image-loader/app/model/Uploader.scala | 31 +++++++++---------- .../test/scala/model/ImageUploadTest.scala | 6 ++-- .../test/scala/model/ProjectorTest.scala | 2 +- 9 files changed, 74 insertions(+), 55 deletions(-) create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedding.scala diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedding.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedding.scala new file mode 100644 index 0000000000..1e99e35a46 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedding.scala @@ -0,0 +1,22 @@ +package com.gu.mediaservice.lib.aws +import com.gu.mediaservice.lib.logging.LogMarker +import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse + +import java.nio.file.{Files, Path} +import java.util.Base64 +import scala.concurrent.{ExecutionContext, Future} + +class Embedding(s3vectors: S3Vectors, bedrock: Bedrock) { + + def createEmbeddingAndStore(imageFilePath: Path, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker + ): Future[PutVectorsResponse] = { + // TODO construct the base64 string inside the fetch embedding option only if config is switched on + val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(imageFilePath)) + + val embeddingFuture = bedrock.createImageEmbedding(base64EncodedString) + val vectorInput = embeddingFuture.map { embedding => + s3vectors.storeEmbeddingInS3VectorStore(embedding, imageId) + } + vectorInput + } +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index f8a9791fdf..425c0c023c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -49,13 +49,13 @@ class S3Vectors(config: CommonConfig) request } - private def fetchEmbeddingFromBedrock(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker - ): Future[List[Float]] = { - val bedrock = new Bedrock(config) - bedrock.createImageEmbedding(base64EncodedImage) - } +// private def createImageEmbeddingWithBedrock(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker +// ): Future[List[Float]] = { +// val bedrock = new Bedrock(config) +// bedrock.createImageEmbedding(base64EncodedImage) +// } - private def storeEmbeddingInS3VectorStore(bedrockEmbedding: List[Float], imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker + def storeEmbeddingInS3VectorStore(bedrockEmbedding: List[Float], imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker ): PutVectorsResponse = { try { val input = createRequestBody(bedrockEmbedding, imageId) @@ -73,14 +73,12 @@ class S3Vectors(config: CommonConfig) } } - def fetchEmbeddingAndStore(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker - ): Future[PutVectorsResponse] = { - val embeddingFuture = fetchEmbeddingFromBedrock(base64EncodedImage: String) - val vectorInput = embeddingFuture.map { embedding => - storeEmbeddingInS3VectorStore(embedding, imageId) - } - vectorInput - } - -// private def searchS3VectorStore() = {} +// def createEmbeddingAndStore(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker +// ): Future[PutVectorsResponse] = { +// val embeddingFuture = createImageEmbeddingWithBedrock(base64EncodedImage: String) +// val vectorInput = embeddingFuture.map { embedding => +// storeEmbeddingInS3VectorStore(embedding, imageId) +// } +// vectorInput +// } } diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index 8d747c721d..c78c6be1c4 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -1,5 +1,5 @@ import com.gu.mediaservice.GridClient -import com.gu.mediaservice.lib.aws.{Bedrock, S3Vectors, SimpleSqsMessageConsumer} +import com.gu.mediaservice.lib.aws.{Bedrock, S3Vectors, SimpleSqsMessageConsumer, Embedding} import com.gu.mediaservice.lib.config.Services import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.GridLogging @@ -28,12 +28,11 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val notifications = new Notifications(config) val downloader = new Downloader()(ec,wsClient) - val embedImage: Boolean = store.embedImage() - logger.info(s"embedImage = $embedImage") + val embedder: Option[Embedding] = if (config.shouldEmbed) Some(new Embedding(new S3Vectors(config), new Bedrock(config))) else None - val s3vectors = new S3Vectors(config) - val uploader = new Uploader(store, config, imageOperations, notifications, s3vectors, imageProcessor) - val projector = Projector(config, imageOperations, imageProcessor, auth, s3vectors) + + val uploader = new Uploader(store, config, imageOperations, notifications, embedder, imageProcessor) + val projector = Projector(config, imageOperations, imageProcessor, auth, embedder) val quarantineUploader: Option[QuarantineUploader] = (config.uploadToQuarantineEnabled, config.quarantineBucket) match { case (true, Some(bucketName)) =>{ val quarantineStore = new QuarantineStore(config) diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala index ae683260c6..61964f9456 100644 --- a/image-loader/app/lib/ImageLoaderConfig.scala +++ b/image-loader/app/lib/ImageLoaderConfig.scala @@ -33,7 +33,7 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res val uploadStatusTable: String = string("dynamo.table.upload.status") val uploadStatusExpiry: FiniteDuration = configuration.get[FiniteDuration]("uploadStatus.recordExpiry") - val embedImages: Boolean = boolean("image.embed.images") + val shouldEmbed: Boolean = boolean("s3.vectors.shouldEmbed") /** * Load in the chain of image processors from config. This can be a list of diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index db7adb5efe..1c509f3712 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -24,8 +24,6 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati } } - def embedImage(): Boolean = config.embedImages - def getS3Object(key: String)(implicit logMarker: LogMarker): S3Object = handleNotFound(key) { client.getObject(config.maybeIngestBucket.get, key) } { diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 2cc3a50c3e..3b9ec0b0d0 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -8,7 +8,7 @@ import com.gu.mediaservice.lib.auth.Authentication import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object} import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{S3Object, S3Ops, S3Vectors} +import com.gu.mediaservice.lib.aws.{Embedding, S3Object, S3Ops, S3Vectors} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} @@ -23,6 +23,7 @@ import play.api.Logger import play.api.libs.ws.WSRequest import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse +import java.nio.file.Path import scala.jdk.CollectionConverters._ import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, Future} @@ -31,8 +32,8 @@ object Projector { import Uploader.toImageUploadOpsCfg - def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, s3vectors: S3Vectors)(implicit ec: ExecutionContext): Projector - = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, s3vectors) + def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbed: Option[Embedding])(implicit ec: ExecutionContext): Projector + = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, maybeEmbed) } case class S3FileExtractedMetadata( @@ -86,9 +87,9 @@ class Projector(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, - s3vectors: S3Vectors) extends GridLogging { + maybeEmbed: Option[Embedding]) extends GridLogging { - private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, s3vectors) + private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, maybeEmbed) def projectS3ImageById(imageId: String, tempFile: File, gridClient: GridClient, onBehalfOfFn: WSRequest => WSRequest) (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[Image]] = { @@ -161,7 +162,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, s3: AmazonS3, - s3vectors: S3Vectors, + maybeEmbed: Option[Embedding], ) extends GridLogging { import Uploader.{fromUploadRequestShared, toMetaMap} @@ -177,14 +178,18 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, projectOptimisedPNGFileAsS3Model, tryFetchThumbFile = fetchThumbFile, tryFetchOptimisedFile = fetchOptimisedFile, - fetchEmbeddingAndStore = fetchEmbeddingAndStore, + createEmbeddingAndStore = createEmbeddingAndStore, ) fromUploadRequestShared(uploadRequest, dependenciesWithProjectionsOnly, processor) } - private def fetchEmbeddingAndStore(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[PutVectorsResponse] = { - s3vectors.fetchEmbeddingAndStore(base64EncodedImage, imageId) + private def createEmbeddingAndStore(imageFilePath: Path, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[PutVectorsResponse]] = { + maybeEmbed match { + case Some(embedding) => + embedding.createEmbeddingAndStore(imageFilePath, imageId).map(Some(_)) + case None => Future.successful(None) + } } private def projectOriginalFileAsS3Model(storableOriginalImage: StorableOriginalImage) = diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index d00886401a..8b78a4da03 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -4,15 +4,10 @@ import com.gu.mediaservice.{GridClient, ImageDataMerger} import com.gu.mediaservice.lib.Files.createTempFile import java.io.File -import java.net.URLEncoder -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Paths} -import java.util.{Base64, UUID} +import java.nio.file.{Files, Path} import com.gu.mediaservice.lib.argo.ArgoHelpers -import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{S3Object, S3Vectors, UpdateMessage} +import com.gu.mediaservice.lib.aws.{Embedding, S3Object, S3Vectors, UpdateMessage} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.formatting._ import com.gu.mediaservice.lib.imaging.ImageOperations @@ -82,7 +77,7 @@ case class ImageUploadOpsDependencies( storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object], tryFetchThumbFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), - fetchEmbeddingAndStore: (String, String) => Future[PutVectorsResponse] + createEmbeddingAndStore: (Path, String) => Future[Option[PutVectorsResponse]] ) case class UploadStatusUri (uri: String) extends AnyVal { @@ -118,7 +113,7 @@ object Uploader extends GridLogging { storeOrProjectOriginalFile, storeOrProjectThumbFile, storeOrProjectOptimisedImage, - fetchEmbeddingAndStore, + createEmbeddingAndStore, OptimiseWithPngQuant, uploadRequest, deps, @@ -130,7 +125,7 @@ object Uploader extends GridLogging { private[model] def uploadAndStoreImage(storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], storeOrProjectOptimisedFile: StorableOptimisedImage => Future[S3Object], - fetchEmbeddingAndStore: (String, String) => Future[PutVectorsResponse], + createEmbeddingAndStore: (Path, String) => Future[Option[PutVectorsResponse]], optimiseOps: OptimiseOps, uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies, @@ -160,11 +155,9 @@ object Uploader extends GridLogging { val sourceStoreFuture = storeOrProjectOriginalFile(storableOriginalImage) val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, tempDirForRequest, deps) - val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(uploadRequest.tempFile.toPath)) - // We are fetching the embedding and storing it in the S3Vectors bucket // This should not block the image upload process on failure - fetchEmbeddingAndStore(base64EncodedString, uploadRequest.imageId).failed.foreach { failure => + createEmbeddingAndStore(uploadRequest.tempFile.toPath, uploadRequest.imageId).failed.foreach { failure => logger.error(logMarker, s"Failed to fetch embedding for ${uploadRequest.imageId} and store", failure) } @@ -339,22 +332,26 @@ class Uploader(val store: ImageLoaderStore, val config: ImageLoaderConfig, val imageOps: ImageOperations, val notifications: Notifications, - val s3vectors: S3Vectors, + val maybeEmbed: Option[Embedding], imageProcessor: ImageProcessor) (implicit val ec: ExecutionContext) extends MessageSubjects with ArgoHelpers { def fromUploadRequest(uploadRequest: UploadRequest) (implicit logMarker: LogMarker): Future[ImageUpload] = { val sideEffectDependencies = ImageUploadOpsDependencies(toImageUploadOpsCfg(config), imageOps, - storeSource, storeThumbnail, storeOptimisedImage, fetchEmbeddingAndStore = fetchEmbeddingAndStore) + storeSource, storeThumbnail, storeOptimisedImage, createEmbeddingAndStore = createEmbeddingAndStore) Stopwatch.async("finalImage") { val finalImage = fromUploadRequestShared(uploadRequest, sideEffectDependencies, imageProcessor) finalImage.map(img => ImageUpload(uploadRequest, img)) } } - private def fetchEmbeddingAndStore(base64EncodedImage: String, imageId: String)(implicit logMarker: LogMarker): Future[PutVectorsResponse] = { - s3vectors.fetchEmbeddingAndStore(base64EncodedImage, imageId) + private def createEmbeddingAndStore(imageFilePath: Path, imageId: String)(implicit logMarker: LogMarker): Future[Option[PutVectorsResponse]] = { + maybeEmbed match { + case Some(embedding) => + embedding.createEmbeddingAndStore(imageFilePath, imageId).map(Some(_)) + case None => Future.successful(None) + } } private def storeSource(storableOriginalImage: StorableOriginalImage) diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index 67b09dff84..8140f96957 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -66,7 +66,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore def storeOrProjectThumbFile: StorableThumbImage => Future[S3Object] = mockStore def storeOrProjectOptimisedPNG: StorableOptimisedImage => Future[S3Object] = mockStore - def fetchEmbeddingAndStore: (String, String) => Future[PutVectorsResponse] = mockVectorStore + def createEmbeddingAndStore: (String, String) => Future[PutVectorsResponse] = mockVectorStore val mockDependencies = ImageUploadOpsDependencies( mockConfig, @@ -74,7 +74,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { storeOrProjectOriginalFile, storeOrProjectThumbFile, storeOrProjectOptimisedPNG, - fetchEmbeddingAndStore = fetchEmbeddingAndStore + createEmbeddingAndStore = createEmbeddingAndStore ) val tempFile = ResourceHelpers.fileAt(fileName) @@ -94,7 +94,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { mockDependencies.storeOrProjectOriginalFile, mockDependencies.storeOrProjectThumbFile, mockDependencies.storeOrProjectOptimisedImage, - fetchEmbeddingAndStore, + createEmbeddingAndStore, OptimiseWithPngQuant, uploadRequest, mockDependencies, diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index c8b1014a50..e9f46ef0c0 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -44,7 +44,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val mockPutVectorsResponse = PutVectorsResponse.builder().build() private val s3vectors = mock[S3Vectors] - when(s3vectors.fetchEmbeddingAndStore(any[String], any[String])(any[ExecutionContext], any[LogMarker])) + when(s3vectors.createEmbeddingAndStore(any[String], any[String])(any[ExecutionContext], any[LogMarker])) .thenReturn(Future.successful(mockPutVectorsResponse)) private val s3 = mock[AmazonS3] From da90978132a9c20aebfcef564999b2cc19d41ef3 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Tue, 14 Oct 2025 17:40:11 +0100 Subject: [PATCH 43/66] Modifications in response to PR comments --- .../js/components/gr-sort-control/base-sort-control.tsx | 6 +----- .../components/gr-sort-control/gr-extended-sort-control.tsx | 2 +- .../js/components/gr-sort-control/gr-sort-control.tsx | 4 ++-- kahuna/public/js/search/query.js | 4 ++-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx index ed55bb55aa..8881a02635 100644 --- a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx @@ -75,17 +75,13 @@ export const BaseSortControl: React.FC = ({ }) => { const hasCollection = startHasCollection; - const startSort:SortDropdownOption = startSelectedOption ? startSelectedOption : options.filter(opt => opt.value == DEFAULT_OPTION)[0]; + const startSort:SortDropdownOption = startSelectedOption ? startSelectedOption : DefaultSortOption; const [isOpen, setIsOpen] = useState(false); const [selectedOption, setSelection] = useState(startSort); const [previousOption, setPrevious] = useState(startSort); const [currentIndex, setCurrentIndex] = useState(-1); const [isPanelVisible, setPanelVisible] = useState(panelVisible); - if (startSort.value !== selectedOption.value) { - setSelection(startSort); - } - const handleArrowKeys = (event:KeyboardEvent) => { if (event.key === 'ArrowDown' || event.key === 'ArrowUp' || diff --git a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx index c6c7d2c2cf..95e3115d28 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx @@ -34,7 +34,7 @@ const ExtendedSortControl: React.FC = ({ props }) => { let startSortOption = DefaultSortOption; if (!query.includes(noTakenDateClause) && (sortOptions.filter(o => o.value === orderBy)).length > 0) { - startSortOption = sortOptions.filter(o => o.value === orderBy)[0]; + startSortOption = sortOptions.find(o => o.value === orderBy); } const startHasCollection = checkForCollection(query); diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index 717d744a62..962565bae0 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -28,11 +28,11 @@ const SortControl: React.FC = ({ props }) => { let startSortOption = DefaultSortOption; if (startHasCollection) { if ((sortOptions.filter(o => o.isCollection)).length > 0) { - startSortOption = sortOptions.filter(o => o.isCollection)[0]; + startSortOption = sortOptions.find(o => o.isCollection); } } else { if ((sortOptions.filter(o => o.value === orderBy)).length > 0) { - startSortOption = sortOptions.filter(o => o.value === orderBy)[0]; + startSortOption = sortOptions.find(o => o.value === orderBy); } } diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index bad8574929..244677a1d1 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -204,7 +204,7 @@ query.controller('SearchQueryCtrl', [ return collection; } - function getPiorOrderBy() { + function getPriorOrderBy() { const prior = storage.getJs("priorOrderBy") ? storage.getJs("priorOrderBy") : ""; return prior; } @@ -222,7 +222,7 @@ query.controller('SearchQueryCtrl', [ } function priorRevisedOrderBy(collectionSearch, newCollection, oldCollection) { - const priorOrderBy = getPiorOrderBy(); + const priorOrderBy = getPriorOrderBy(); if (collectionSearch && ((oldCollection !== newCollection) || ("" !== priorOrderBy))) { if (priorOrderBy != "") { setPriorOrderBy(""); From fa69542014c96a4d16d7bea88843c924f34b03bf Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 15 Oct 2025 09:59:34 +0100 Subject: [PATCH 44/66] update tests --- image-loader/test/scala/model/ImageUploadTest.scala | 9 ++++----- image-loader/test/scala/model/ProjectorTest.scala | 10 ++++------ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index 8140f96957..f9c5684520 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -20,6 +20,7 @@ import org.scalatestplus.mockito.MockitoSugar import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import test.lib.ResourceHelpers +import java.nio.file.Path import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} @@ -58,15 +59,13 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { ) val mockPutVectorsResponse = mock[PutVectorsResponse] - def mockVectorStore = (imageBase64: String, imageId: String) => - Future.successful( - mockPutVectorsResponse - ) + def mockVectorStore = (imagePath: Path, imageId: String) => + Future.successful(Some(mockPutVectorsResponse)) def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore def storeOrProjectThumbFile: StorableThumbImage => Future[S3Object] = mockStore def storeOrProjectOptimisedPNG: StorableOptimisedImage => Future[S3Object] = mockStore - def createEmbeddingAndStore: (String, String) => Future[PutVectorsResponse] = mockVectorStore + def createEmbeddingAndStore: (Path, String) => Future[Option[PutVectorsResponse]] = mockVectorStore val mockDependencies = ImageUploadOpsDependencies( mockConfig, diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index e9f46ef0c0..ad9adde0ba 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -7,7 +7,7 @@ import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.model.ObjectMetadata import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.aws.S3Vectors +import com.gu.mediaservice.lib.aws.{Embedding, S3Vectors} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} @@ -26,6 +26,7 @@ import play.api.libs.json.{JsArray, JsString} import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import test.lib.ResourceHelpers +import java.nio.file.Path import scala.concurrent.ExecutionContext.Implicits.global import scala.jdk.CollectionConverters._ import scala.concurrent.{ExecutionContext, Future} @@ -42,14 +43,11 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket") - private val mockPutVectorsResponse = PutVectorsResponse.builder().build() - private val s3vectors = mock[S3Vectors] - when(s3vectors.createEmbeddingAndStore(any[String], any[String])(any[ExecutionContext], any[LogMarker])) - .thenReturn(Future.successful(mockPutVectorsResponse)) + private val maybeEmbed = None private val s3 = mock[AmazonS3] private val auth = mock[Authentication] - private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, s3vectors) + private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbed) // FIXME temporary ignored as test is not executable in CI/CD machine // because graphic lib files like srgb.icc, cmyk.icc are in root directory instead of resources From b05985ef21e549f080405c2aced82b08d7e89f42 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 15 Oct 2025 16:50:18 +0100 Subject: [PATCH 45/66] refactor to embedder --- .../com/gu/mediaservice/lib/aws/Bedrock.scala | 2 -- .../lib/aws/{Embedding.scala => Embedder.scala} | 4 ++-- .../com/gu/mediaservice/lib/aws/S3Vectors.scala | 16 +--------------- image-loader/app/ImageLoaderComponents.scala | 5 ++--- image-loader/app/model/Projector.scala | 10 ++++------ image-loader/app/model/Uploader.scala | 8 ++++---- .../test/scala/model/ProjectorTest.scala | 2 +- 7 files changed, 14 insertions(+), 33 deletions(-) rename common-lib/src/main/scala/com/gu/mediaservice/lib/aws/{Embedding.scala => Embedder.scala} (82%) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index a5511b3a1d..780d0e45f4 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -3,8 +3,6 @@ package com.gu.mediaservice.lib.aws import software.amazon.awssdk.services.bedrockruntime.model._ import software.amazon.awssdk.services.bedrockruntime._ import com.gu.mediaservice.lib.config.CommonConfig - -import java.util.concurrent.CompletableFuture import play.api.libs.json.Json import software.amazon.awssdk.core.SdkBytes diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedding.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala similarity index 82% rename from common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedding.scala rename to common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index 1e99e35a46..0b6a29ad76 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedding.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -1,16 +1,16 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.logging.LogMarker + import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import java.nio.file.{Files, Path} import java.util.Base64 import scala.concurrent.{ExecutionContext, Future} -class Embedding(s3vectors: S3Vectors, bedrock: Bedrock) { +class Embedder(s3vectors: S3Vectors, bedrock: Bedrock) { def createEmbeddingAndStore(imageFilePath: Path, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker ): Future[PutVectorsResponse] = { - // TODO construct the base64 string inside the fetch embedding option only if config is switched on val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(imageFilePath)) val embeddingFuture = bedrock.createImageEmbedding(base64EncodedString) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index 425c0c023c..a2278a2f52 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -1,7 +1,7 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.LogMarker -import org.bouncycastle.util.encoders.Base64Encoder + import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3vectors._ import software.amazon.awssdk.services.s3vectors.model.{PutInputVector, PutVectorsRequest, PutVectorsResponse, VectorData} @@ -49,12 +49,6 @@ class S3Vectors(config: CommonConfig) request } -// private def createImageEmbeddingWithBedrock(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker -// ): Future[List[Float]] = { -// val bedrock = new Bedrock(config) -// bedrock.createImageEmbedding(base64EncodedImage) -// } - def storeEmbeddingInS3VectorStore(bedrockEmbedding: List[Float], imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker ): PutVectorsResponse = { try { @@ -73,12 +67,4 @@ class S3Vectors(config: CommonConfig) } } -// def createEmbeddingAndStore(base64EncodedImage: String, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker -// ): Future[PutVectorsResponse] = { -// val embeddingFuture = createImageEmbeddingWithBedrock(base64EncodedImage: String) -// val vectorInput = embeddingFuture.map { embedding => -// storeEmbeddingInS3VectorStore(embedding, imageId) -// } -// vectorInput -// } } diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index c78c6be1c4..4075177ee8 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -1,5 +1,5 @@ import com.gu.mediaservice.GridClient -import com.gu.mediaservice.lib.aws.{Bedrock, S3Vectors, SimpleSqsMessageConsumer, Embedding} +import com.gu.mediaservice.lib.aws.{Bedrock, S3Vectors, SimpleSqsMessageConsumer, Embedder} import com.gu.mediaservice.lib.config.Services import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.GridLogging @@ -28,8 +28,7 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val notifications = new Notifications(config) val downloader = new Downloader()(ec,wsClient) - val embedder: Option[Embedding] = if (config.shouldEmbed) Some(new Embedding(new S3Vectors(config), new Bedrock(config))) else None - + val embedder: Option[Embedder] = if (config.shouldEmbed) Some(new Embedder(new S3Vectors(config), new Bedrock(config))) else None val uploader = new Uploader(store, config, imageOperations, notifications, embedder, imageProcessor) val projector = Projector(config, imageOperations, imageProcessor, auth, embedder) diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 3b9ec0b0d0..a3595a446f 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -1,14 +1,13 @@ package model import java.io.{File, FileOutputStream} -import java.util.UUID import com.amazonaws.services.s3.AmazonS3 import com.gu.mediaservice.{GridClient, ImageDataMerger} import com.gu.mediaservice.lib.auth.Authentication import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object} import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Embedding, S3Object, S3Ops, S3Vectors} +import com.gu.mediaservice.lib.aws.{Embedder, S3Ops} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} @@ -19,7 +18,6 @@ import lib.{DigestedFile, ImageLoaderConfig} import model.upload.UploadRequest import org.apache.tika.io.IOUtils import org.joda.time.{DateTime, DateTimeZone} -import play.api.Logger import play.api.libs.ws.WSRequest import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse @@ -32,7 +30,7 @@ object Projector { import Uploader.toImageUploadOpsCfg - def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbed: Option[Embedding])(implicit ec: ExecutionContext): Projector + def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbed: Option[Embedder])(implicit ec: ExecutionContext): Projector = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, maybeEmbed) } @@ -87,7 +85,7 @@ class Projector(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, - maybeEmbed: Option[Embedding]) extends GridLogging { + maybeEmbed: Option[Embedder]) extends GridLogging { private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, maybeEmbed) @@ -162,7 +160,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, s3: AmazonS3, - maybeEmbed: Option[Embedding], + maybeEmbed: Option[Embedder], ) extends GridLogging { import Uploader.{fromUploadRequestShared, toMetaMap} diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index 8b78a4da03..3b289fb56f 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -7,7 +7,7 @@ import java.io.File import java.nio.file.{Files, Path} import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Embedding, S3Object, S3Vectors, UpdateMessage} +import com.gu.mediaservice.lib.aws.{Embedder, S3Object, S3Vectors, UpdateMessage} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.formatting._ import com.gu.mediaservice.lib.imaging.ImageOperations @@ -332,7 +332,7 @@ class Uploader(val store: ImageLoaderStore, val config: ImageLoaderConfig, val imageOps: ImageOperations, val notifications: Notifications, - val maybeEmbed: Option[Embedding], + val maybeEmbed: Option[Embedder], imageProcessor: ImageProcessor) (implicit val ec: ExecutionContext) extends MessageSubjects with ArgoHelpers { @@ -348,8 +348,8 @@ class Uploader(val store: ImageLoaderStore, private def createEmbeddingAndStore(imageFilePath: Path, imageId: String)(implicit logMarker: LogMarker): Future[Option[PutVectorsResponse]] = { maybeEmbed match { - case Some(embedding) => - embedding.createEmbeddingAndStore(imageFilePath, imageId).map(Some(_)) + case Some(embedder) => + embedder.createEmbeddingAndStore(imageFilePath, imageId).map(Some(_)) case None => Future.successful(None) } } diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index ad9adde0ba..1ecd56b4c9 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -7,7 +7,7 @@ import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.model.ObjectMetadata import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.aws.{Embedding, S3Vectors} +import com.gu.mediaservice.lib.aws.{Embedder, S3Vectors} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} From 7bafc0ea416353529b9d1066a91b276811fc150f Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Thu, 16 Oct 2025 13:45:15 +0100 Subject: [PATCH 46/66] update script --- dev/script/get-s3-vector-store-records.sh | 21 +++++++++++++++++++++ vectorcommand.sh | 8 -------- 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100755 dev/script/get-s3-vector-store-records.sh delete mode 100644 vectorcommand.sh diff --git a/dev/script/get-s3-vector-store-records.sh b/dev/script/get-s3-vector-store-records.sh new file mode 100755 index 0000000000..ff63cb3b40 --- /dev/null +++ b/dev/script/get-s3-vector-store-records.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Check if environment argument is provided +if [ $# -ne 1 ]; then + echo "Usage: $0 " + echo "Environment options: dev, test, prod" + exit 1 +fi + +# Validate environment argument +env=$1 +if [[ ! "$env" =~ ^(dev|test|prod)$ ]]; then + echo "Invalid environment. Please use dev, test, or prod" + exit 1 +fi + +aws s3vectors list-vectors \ + --vector-bucket-name "image-embeddings-$env" \ + --index-name cohere-embed-english-v3 \ + --profile media-service \ + --region eu-central-1 \ No newline at end of file diff --git a/vectorcommand.sh b/vectorcommand.sh deleted file mode 100644 index 60dc3d3ef9..0000000000 --- a/vectorcommand.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -aws s3vectors list-vectors \ - --vector-bucket-name image-embeddings-dev \ - --index-name cohere-embed-english-v3 \ - --profile media-service \ - --region eu-central-1 - From e232b07c77bda19b80dd4e4660941ffd4c811e04 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Thu, 16 Oct 2025 14:24:17 +0100 Subject: [PATCH 47/66] update pacakge lock --- kahuna/package-lock.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kahuna/package-lock.json b/kahuna/package-lock.json index 33fd57b829..bc01884cb7 100644 --- a/kahuna/package-lock.json +++ b/kahuna/package-lock.json @@ -5,11 +5,11 @@ "packages": { "": { "dependencies": { - "@babel/polyfill": "^7.8.7", - "@guardian/cql": "^1.8.1", - "@guardian/user-telemetry-client": "^1.1.0", - "@sentry/browser": "^6.11.0", - "@sentry/integrations": "^6.11.0", + "@babel/polyfill": "7.8.7", + "@guardian/cql": "1.8.1", + "@guardian/user-telemetry-client": "1.1.0", + "@sentry/browser": "6.11.0", + "@sentry/integrations": "6.11.0", "angular": "1.8.3", "angular-animate": "1.8.3", "angular-cookies": "1.8.3", From a3ba79cbf6a0758ba51d9f005bb434f850c4d90a Mon Sep 17 00:00:00 2001 From: Mateusz Date: Fri, 17 Oct 2025 11:01:20 +0100 Subject: [PATCH 48/66] Propagate pgUp and pgDown when caret in searchbox --- kahuna/public/js/components/gr-cql-input/gr-cql-input.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kahuna/public/js/components/gr-cql-input/gr-cql-input.ts b/kahuna/public/js/components/gr-cql-input/gr-cql-input.ts index 03fcc2cef0..b8feac8c05 100644 --- a/kahuna/public/js/components/gr-cql-input/gr-cql-input.ts +++ b/kahuna/public/js/components/gr-cql-input/gr-cql-input.ts @@ -92,7 +92,9 @@ grCqlInput.directive< "ArrowUp", "ArrowDown", "ArrowLeft", - "ArrowRight" + "ArrowRight", + "PageUp", + "PageDown" ]; ["keydown", "keyup", "keypress"].forEach((eventType) => { cqlInput.addEventListener(eventType, (e) => { From e6e16bfdcc99e5d66b1d70ea8d5816e484312fea Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Mon, 20 Oct 2025 12:39:37 +0100 Subject: [PATCH 49/66] action comments --- .../scala/com/gu/mediaservice/lib/aws/Embedder.scala | 3 +-- .../com/gu/mediaservice/lib/aws/S3Vectors.scala | 5 ++--- .../com/gu/mediaservice/model/ImageEmbedding.scala | 12 ------------ 3 files changed, 3 insertions(+), 17 deletions(-) delete mode 100644 common-lib/src/main/scala/com/gu/mediaservice/model/ImageEmbedding.scala diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index 0b6a29ad76..e4f9ae856d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -14,9 +14,8 @@ class Embedder(s3vectors: S3Vectors, bedrock: Bedrock) { val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(imageFilePath)) val embeddingFuture = bedrock.createImageEmbedding(base64EncodedString) - val vectorInput = embeddingFuture.map { embedding => + embeddingFuture.map { embedding => s3vectors.storeEmbeddingInS3VectorStore(embedding, imageId) } - vectorInput } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index a2278a2f52..2bc6dadb41 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -29,8 +29,7 @@ class S3Vectors(config: CommonConfig) private def createRequestBody(embedding: List[Float], imageId: String): PutVectorsRequest = { val vectorData: VectorData = VectorData .builder() - // TODO find out if we can do something less upsetting than this float conversion - .float32(embedding.map(_.asInstanceOf[java.lang.Float]).asJava) + .float32(embedding.map(float2Float).asJava) .build() val inputVector: PutInputVector = PutInputVector @@ -49,7 +48,7 @@ class S3Vectors(config: CommonConfig) request } - def storeEmbeddingInS3VectorStore(bedrockEmbedding: List[Float], imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker + def storeEmbeddingInS3VectorStore(bedrockEmbedding: List[Float], imageId: String)(implicit logMarker: LogMarker ): PutVectorsResponse = { try { val input = createRequestBody(bedrockEmbedding, imageId) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/ImageEmbedding.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/ImageEmbedding.scala deleted file mode 100644 index 5ba3765d24..0000000000 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/ImageEmbedding.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.gu.mediaservice.model - -import play.api.libs.json.Json -import play.api.libs.json.OFormat - -case class ImageEmbedding( - cohereEmbedEnglishV3: List[Double] -) - -object ImageEmbedding { - implicit val format: OFormat[ImageEmbedding] = Json.format[ImageEmbedding] -} \ No newline at end of file From 3e9842eaff614298d389a8b6d5660dfc40df6496 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Mon, 20 Oct 2025 13:31:32 +0100 Subject: [PATCH 50/66] only add sdk to common lib --- build.sbt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/build.sbt b/build.sbt index b775f90138..d5949211f7 100644 --- a/build.sbt +++ b/build.sbt @@ -166,8 +166,6 @@ lazy val thrall = playProject("thrall", 9002) // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, - "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, - "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, "com.gu" %% "kcl-pekko-stream" % "0.1.0", "org.testcontainers" % "elasticsearch" % "1.19.2" % Test, "com.google.protobuf" % "protobuf-java" % "3.19.6" @@ -191,8 +189,6 @@ lazy val usage = playProject("usage", 9009).settings( // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, - "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, - "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, "com.google.protobuf" % "protobuf-java" % "3.19.6" ), dependencyOverrides ++= Seq( @@ -208,8 +204,6 @@ lazy val scripts = project("scripts") // V2 of the AWS SDK as it's easier to use for scripts and won't leak to the rest of the project from here "software.amazon.awssdk" % "s3" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, - "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, - "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, // bump jcommander explicitly as AWS SDK is pulling in a vulnerable version "com.beust" % "jcommander" % "1.75", "org.apache.commons" % "commons-compress" % "1.27.1", From 3355eba57fe19e8314d8040ef82d99f6d857b1e3 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Mon, 20 Oct 2025 14:23:09 +0100 Subject: [PATCH 51/66] update elasticsearch container version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a09aa8f7dd..f1da235edd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: pull-requests: write services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2 + image: docker.elastic.co/elasticsearch/elasticsearch:8.18.3 # Wait for elasticsearch to report healthy before continuing. # see https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml#L28 options: -e "discovery.type=single-node" --expose 9200 --health-cmd "curl localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10 From 23457ffff5c6027e815065e8f61355ee0e2e7aaf Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Mon, 20 Oct 2025 14:42:34 +0100 Subject: [PATCH 52/66] change ci elasticsearch image version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1da235edd..cc5bb8e379 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: pull-requests: write services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.18.3 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.29 # Wait for elasticsearch to report healthy before continuing. # see https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml#L28 options: -e "discovery.type=single-node" --expose 9200 --health-cmd "curl localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10 From 2ad5d77b781d853fef0e1fe168a1df70010a4970 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Tue, 21 Oct 2025 15:37:01 +0100 Subject: [PATCH 53/66] add check if > 5MB or not JPEG --- .../gu/mediaservice/lib/aws/Embedder.scala | 67 ++++++++++++++++--- image-loader/app/model/Projector.scala | 6 +- image-loader/app/model/Uploader.scala | 10 +-- .../test/scala/model/ImageUploadTest.scala | 4 +- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index e4f9ae856d..0636bf6383 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -1,21 +1,70 @@ package com.gu.mediaservice.lib.aws -import com.gu.mediaservice.lib.logging.LogMarker - +import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} +import com.gu.mediaservice.model.{Jpeg, MimeType} import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import java.nio.file.{Files, Path} import java.util.Base64 import scala.concurrent.{ExecutionContext, Future} -class Embedder(s3vectors: S3Vectors, bedrock: Bedrock) { +class Embedder(s3vectors: S3Vectors, bedrock: Bedrock) extends GridLogging { + + // We can't be sure that the image is JPEG or <5MB so we need to check and log + def meetsCohereRequirements(fileType: MimeType, imageFilePath: Path)(implicit logMarker: LogMarker): Boolean = { + val fileSize = Files.size(imageFilePath) + val fiveMB = 5_000_000 + + // Hi Ellen! This is Joe + // Here are some image ids that are between 5MB and 5MiB: + // [ + // "3bb95474d37f14d71ba38b63db324704c3947b00", + // 5241471, + // "image/jpeg" + // ] + // [ + // "3bf2ca04f570842fba77ba3a882fb53a59e9bc00", + // 5004231, + // "image/jpeg" + // ] + // [ + // "33b401db3a8846496bed4c3d12fa2ec35be5b900", + // 5071551, + // "image/jpeg" + // ] + // [ + // "402fe271245dcf994fbd90ff75b99f3eff763800", + // 5052613, + // "image/jpeg" + // ] + // [ + // "4034a0e1909866c5f3f64b23f94ddcf2cdd80400", + // 5189131, + // "image/jpeg" + // ] - def createEmbeddingAndStore(imageFilePath: Path, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker - ): Future[PutVectorsResponse] = { - val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(imageFilePath)) + // Step 1: check the file extension + if (fileType != Jpeg) { + logger.error(logMarker, s"Image file type is not JPEG. File type: $fileType") + false + } + // Step 2: check the file type + else if (fileSize > fiveMB) { + logger.error(logMarker, s"Image file is >5MB. File size: $fileSize") + false + } else true + } - val embeddingFuture = bedrock.createImageEmbedding(base64EncodedString) - embeddingFuture.map { embedding => - s3vectors.storeEmbeddingInS3VectorStore(embedding, imageId) + def createEmbeddingAndStore(fileType: MimeType, imageFilePath: Path, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker + ): Future[Option[PutVectorsResponse]] = { + if (!meetsCohereRequirements(fileType, imageFilePath)(logMarker)) { + logger.info(logMarker, s"Skipping image embedding for $imageId as it does not meet the requirements.") + Future.successful(None) + } else { + val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(imageFilePath)) + val embeddingFuture = bedrock.createImageEmbedding(base64EncodedString) + embeddingFuture.map { embedding => + Some(s3vectors.storeEmbeddingInS3VectorStore(embedding, imageId)) + } } } } diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index a3595a446f..307d384413 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -182,10 +182,10 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, fromUploadRequestShared(uploadRequest, dependenciesWithProjectionsOnly, processor) } - private def createEmbeddingAndStore(imageFilePath: Path, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[PutVectorsResponse]] = { + private def createEmbeddingAndStore(fileType: MimeType, imageFilePath: Path, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[PutVectorsResponse]] = { maybeEmbed match { - case Some(embedding) => - embedding.createEmbeddingAndStore(imageFilePath, imageId).map(Some(_)) + case Some(embedder) => + embedder.createEmbeddingAndStore(fileType, imageFilePath, imageId) case None => Future.successful(None) } } diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index 3b289fb56f..548cde7dad 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -77,7 +77,7 @@ case class ImageUploadOpsDependencies( storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object], tryFetchThumbFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), - createEmbeddingAndStore: (Path, String) => Future[Option[PutVectorsResponse]] + createEmbeddingAndStore: (MimeType, Path, String) => Future[Option[PutVectorsResponse]] ) case class UploadStatusUri (uri: String) extends AnyVal { @@ -125,7 +125,7 @@ object Uploader extends GridLogging { private[model] def uploadAndStoreImage(storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], storeOrProjectOptimisedFile: StorableOptimisedImage => Future[S3Object], - createEmbeddingAndStore: (Path, String) => Future[Option[PutVectorsResponse]], + createEmbeddingAndStore: (MimeType, Path, String) => Future[Option[PutVectorsResponse]], optimiseOps: OptimiseOps, uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies, @@ -157,7 +157,7 @@ object Uploader extends GridLogging { // We are fetching the embedding and storing it in the S3Vectors bucket // This should not block the image upload process on failure - createEmbeddingAndStore(uploadRequest.tempFile.toPath, uploadRequest.imageId).failed.foreach { failure => + createEmbeddingAndStore(originalMimeType, uploadRequest.tempFile.toPath, uploadRequest.imageId).failed.foreach { failure => logger.error(logMarker, s"Failed to fetch embedding for ${uploadRequest.imageId} and store", failure) } @@ -346,10 +346,10 @@ class Uploader(val store: ImageLoaderStore, } } - private def createEmbeddingAndStore(imageFilePath: Path, imageId: String)(implicit logMarker: LogMarker): Future[Option[PutVectorsResponse]] = { + private def createEmbeddingAndStore(fileType: MimeType, imageFilePath: Path, imageId: String)(implicit logMarker: LogMarker): Future[Option[PutVectorsResponse]] = { maybeEmbed match { case Some(embedder) => - embedder.createEmbeddingAndStore(imageFilePath, imageId).map(Some(_)) + embedder.createEmbeddingAndStore(fileType, imageFilePath, imageId) case None => Future.successful(None) } } diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index f9c5684520..ae6034ec7d 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -59,13 +59,13 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { ) val mockPutVectorsResponse = mock[PutVectorsResponse] - def mockVectorStore = (imagePath: Path, imageId: String) => + def mockVectorStore = (fileType: MimeType, imagePath: Path, imageId: String) => Future.successful(Some(mockPutVectorsResponse)) def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore def storeOrProjectThumbFile: StorableThumbImage => Future[S3Object] = mockStore def storeOrProjectOptimisedPNG: StorableOptimisedImage => Future[S3Object] = mockStore - def createEmbeddingAndStore: (Path, String) => Future[Option[PutVectorsResponse]] = mockVectorStore + def createEmbeddingAndStore: (MimeType, Path, String) => Future[Option[PutVectorsResponse]] = mockVectorStore val mockDependencies = ImageUploadOpsDependencies( mockConfig, From 706d882f853ace585f521d60358c80ad2959eef7 Mon Sep 17 00:00:00 2001 From: Mateusz Date: Tue, 21 Oct 2025 22:29:20 +0100 Subject: [PATCH 54/66] Spotted some more useless byline values Added new redundant tokens 'STR New' and '-STR' to the list. --- .../com/gu/mediaservice/lib/cleanup/RedundantTokenRemover.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/RedundantTokenRemover.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/RedundantTokenRemover.scala index 3c74625f90..d61b2b5b00 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/RedundantTokenRemover.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/RedundantTokenRemover.scala @@ -21,6 +21,8 @@ object RedundantTokenRemover extends MetadataCleaner { "Stringer", "Stringer .", "STR", + "STR New", + "-STR", "supplied", "Supplied", "SUPPLIED", From eb464e634279539da00cee230152c3731366e9d9 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 22 Oct 2025 11:27:11 +0100 Subject: [PATCH 55/66] add shouldembed to image loader config script --- dev/script/generate-config/service-config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/script/generate-config/service-config.js b/dev/script/generate-config/service-config.js index d969844af3..3c522da91a 100644 --- a/dev/script/generate-config/service-config.js +++ b/dev/script/generate-config/service-config.js @@ -74,6 +74,7 @@ function getImageLoaderConfig(config) { |metrics.request.enabled=false |transcoded.mime.types="image/tiff" |upload.quarantine.enabled=false + |s3.vectors.shouldEmbed=false |`; } From f210ffd5be3a7ae14899b8c267964cc10e231120 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 22 Oct 2025 13:38:55 +0100 Subject: [PATCH 56/66] check if image is compatible with cohere --- .../com/gu/mediaservice/lib/aws/Bedrock.scala | 19 ++--- .../gu/mediaservice/lib/aws/Embedder.scala | 71 ++++++------------- 2 files changed, 33 insertions(+), 57 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index 780d0e45f4..9ad2cfd78c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -8,7 +8,6 @@ import software.amazon.awssdk.core.SdkBytes import java.net.URI import com.gu.mediaservice.lib.logging.LogMarker -import play.api.libs.json.Format.GenericFormat import play.api.libs.json.OFormat.oFormatFromReadsAndOWrites import play.api.libs.json._ @@ -22,7 +21,6 @@ object Bedrock { } import scala.concurrent.{ExecutionContext, Future} -import scala.jdk.FutureConverters.CompletionStageOps class Bedrock(config: CommonConfig) extends AwsClientV2BuilderUtils { @@ -37,11 +35,16 @@ class Bedrock(config: CommonConfig) .build() } - private def createRequestBody(base64EncodedImage: String): InvokeModelRequest = { + private def createRequestBody(base64EncodedImage: String, fileType: CohereCompatibleMimeType): InvokeModelRequest = { + val images = fileType match { + case CohereJpeg => List(s"data:image/jpg;base64,$base64EncodedImage") + case CoherePng => List(s"data:image/png;base64,$base64EncodedImage") + } + val body = Bedrock.BedrockRequest( input_type = "image", embedding_types = List("float"), - images = List(s"data:image/jpg;base64,$base64EncodedImage") + images = images ) val jsonBody = Json.toJson(body).toString() @@ -57,11 +60,11 @@ class Bedrock(config: CommonConfig) request } - private def sendBedrockEmbeddingRequest(base64EncodedImage: String)( + private def sendBedrockEmbeddingRequest(base64EncodedImage: String, fileType: CohereCompatibleMimeType)( implicit logMarker: LogMarker ): InvokeModelResponse = { try { - val response = client.invokeModel(createRequestBody(base64EncodedImage)) + val response = client.invokeModel(createRequestBody(base64EncodedImage, fileType)) logger.info( logMarker, s"Bedrock API call completed with status: ${response.sdkHttpResponse().statusCode()}" @@ -75,8 +78,8 @@ class Bedrock(config: CommonConfig) } } - def createImageEmbedding(base64EncodedImage: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] = { - val bedrockFuture = Future { sendBedrockEmbeddingRequest(base64EncodedImage) } + def createImageEmbedding(base64EncodedImage: String, fileType: CohereCompatibleMimeType)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] = { + val bedrockFuture = Future { sendBedrockEmbeddingRequest(base64EncodedImage, fileType) } bedrockFuture.map { response => val responseBody = response.body().asUtf8String() val json = Json.parse(responseBody) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index 0636bf6383..3c764305a0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -1,69 +1,42 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} -import com.gu.mediaservice.model.{Jpeg, MimeType} +import com.gu.mediaservice.model.{Jpeg, MimeType, Png, Tiff} import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import java.nio.file.{Files, Path} import java.util.Base64 import scala.concurrent.{ExecutionContext, Future} -class Embedder(s3vectors: S3Vectors, bedrock: Bedrock) extends GridLogging { +sealed trait CohereCompatibleMimeType +case object CohereJpeg extends CohereCompatibleMimeType +case object CoherePng extends CohereCompatibleMimeType - // We can't be sure that the image is JPEG or <5MB so we need to check and log - def meetsCohereRequirements(fileType: MimeType, imageFilePath: Path)(implicit logMarker: LogMarker): Boolean = { +class Embedder(s3vectors: S3Vectors, bedrock: Bedrock) extends GridLogging { + def meetsCohereRequirements(fileType: MimeType, imageFilePath: Path)(implicit logMarker: LogMarker): Either[String, CohereCompatibleMimeType]= { val fileSize = Files.size(imageFilePath) val fiveMB = 5_000_000 - // Hi Ellen! This is Joe - // Here are some image ids that are between 5MB and 5MiB: - // [ - // "3bb95474d37f14d71ba38b63db324704c3947b00", - // 5241471, - // "image/jpeg" - // ] - // [ - // "3bf2ca04f570842fba77ba3a882fb53a59e9bc00", - // 5004231, - // "image/jpeg" - // ] - // [ - // "33b401db3a8846496bed4c3d12fa2ec35be5b900", - // 5071551, - // "image/jpeg" - // ] - // [ - // "402fe271245dcf994fbd90ff75b99f3eff763800", - // 5052613, - // "image/jpeg" - // ] - // [ - // "4034a0e1909866c5f3f64b23f94ddcf2cdd80400", - // 5189131, - // "image/jpeg" - // ] - - // Step 1: check the file extension - if (fileType != Jpeg) { - logger.error(logMarker, s"Image file type is not JPEG. File type: $fileType") - false + fileType match { + case _ if fileSize > fiveMB => Left(s"Image file is >5MB. File size: $fileSize") + case Jpeg => Right(CohereJpeg) + case Png => Right(CoherePng) + case Tiff => Left("Image file type is not supported. File type: Tiff") } - // Step 2: check the file type - else if (fileSize > fiveMB) { - logger.error(logMarker, s"Image file is >5MB. File size: $fileSize") - false - } else true } def createEmbeddingAndStore(fileType: MimeType, imageFilePath: Path, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker ): Future[Option[PutVectorsResponse]] = { - if (!meetsCohereRequirements(fileType, imageFilePath)(logMarker)) { - logger.info(logMarker, s"Skipping image embedding for $imageId as it does not meet the requirements.") - Future.successful(None) - } else { - val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(imageFilePath)) - val embeddingFuture = bedrock.createImageEmbedding(base64EncodedString) - embeddingFuture.map { embedding => - Some(s3vectors.storeEmbeddingInS3VectorStore(embedding, imageId)) + meetsCohereRequirements(fileType, imageFilePath)(logMarker) match { + case Left(error) => { + logger.info(logMarker, s"Skipping image embedding for $imageId as it does not meet the requirements: $error") + Future.successful(None) + } + case Right(imageType) => { + val base64EncodedString: String = Base64.getEncoder().encodeToString(Files.readAllBytes(imageFilePath)) + val embeddingFuture = bedrock.createImageEmbedding(base64EncodedString, imageType) + embeddingFuture.map { embedding => + Some(s3vectors.storeEmbeddingInS3VectorStore(embedding, imageId)) + } } } } From 449cb5d3ea17464b532d5239aa9d5327f94829ba Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 24 Oct 2025 12:01:39 +0100 Subject: [PATCH 57/66] Add link to docs in comment --- .../src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index 3c764305a0..1a00db5ed0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -12,6 +12,7 @@ case object CohereJpeg extends CohereCompatibleMimeType case object CoherePng extends CohereCompatibleMimeType class Embedder(s3vectors: S3Vectors, bedrock: Bedrock) extends GridLogging { + // https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-embed-v3.html#:~:text=The%20image%20must%20be%20in%20either%20image/jpeg%20or%20image/png%20format%20and%20has%20a%20maximum%20size%20of%205MB def meetsCohereRequirements(fileType: MimeType, imageFilePath: Path)(implicit logMarker: LogMarker): Either[String, CohereCompatibleMimeType]= { val fileSize = Files.size(imageFilePath) val fiveMB = 5_000_000 From 2886dc15d701f2d05a17c557978e47a4a4525026 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 24 Oct 2025 12:01:57 +0100 Subject: [PATCH 58/66] Indent comment to match surrounding indentation --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index 2bc6dadb41..d4a7a1737f 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -18,7 +18,7 @@ class S3Vectors(config: CommonConfig) override def isDev: Boolean = config.isDev -// The S3 Vector Store is not yet available in eu-west-1, so we are using eu-central-1 because it's closest to us. + // The S3 Vector Store is not yet available in eu-west-1, so we are using eu-central-1 because it's closest to us. override def awsRegionV2: Region = Region.EU_CENTRAL_1 val client: S3VectorsClient = { From c6bdf290cef0270e1c73a20ce0a75996098e7e8c Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 24 Oct 2025 12:02:37 +0100 Subject: [PATCH 59/66] Rename input => request to match type and function name --- .../main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index d4a7a1737f..934fa4e463 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -51,8 +51,8 @@ class S3Vectors(config: CommonConfig) def storeEmbeddingInS3VectorStore(bedrockEmbedding: List[Float], imageId: String)(implicit logMarker: LogMarker ): PutVectorsResponse = { try { - val input = createRequestBody(bedrockEmbedding, imageId) - val response = client.putVectors(input) + val request = createRequestBody(bedrockEmbedding, imageId) + val response = client.putVectors(request) logger.info( logMarker, s"S3 Vector Store API call completed with status: ${response.sdkHttpResponse().statusCode()}" From 97c27689f07c99858f3c2445dc3f544038008b75 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 24 Oct 2025 12:03:05 +0100 Subject: [PATCH 60/66] Back out formatting change to match existing style --- image-loader/app/model/Uploader.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index 548cde7dad..ed7c7c0bc4 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -70,14 +70,14 @@ case class ImageUploadOpsCfg( ) case class ImageUploadOpsDependencies( - config: ImageUploadOpsCfg, - imageOps: ImageOperations, - storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], - storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], - storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object], - tryFetchThumbFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), - tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), - createEmbeddingAndStore: (MimeType, Path, String) => Future[Option[PutVectorsResponse]] + config: ImageUploadOpsCfg, + imageOps: ImageOperations, + storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], + storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], + storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object], + tryFetchThumbFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), + tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None), + createEmbeddingAndStore: (MimeType, Path, String) => Future[Option[PutVectorsResponse]] ) case class UploadStatusUri (uri: String) extends AnyVal { From 6597603c84833f1269c734f8c4ac50c3f245d39e Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 24 Oct 2025 12:03:26 +0100 Subject: [PATCH 61/66] Indent comment to match surrounding indentation --- image-loader/app/model/Uploader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index ed7c7c0bc4..e547e649b1 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -155,7 +155,7 @@ object Uploader extends GridLogging { val sourceStoreFuture = storeOrProjectOriginalFile(storableOriginalImage) val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, tempDirForRequest, deps) -// We are fetching the embedding and storing it in the S3Vectors bucket + // We are fetching the embedding and storing it in the S3Vectors bucket // This should not block the image upload process on failure createEmbeddingAndStore(originalMimeType, uploadRequest.tempFile.toPath, uploadRequest.imageId).failed.foreach { failure => logger.error(logMarker, s"Failed to fetch embedding for ${uploadRequest.imageId} and store", failure) From 3aad8096eae8748db464317a1664ef9de626efdb Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 24 Oct 2025 12:03:58 +0100 Subject: [PATCH 62/66] embedder => maybeEmbedder to reflect optional type --- image-loader/app/ImageLoaderComponents.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index 4075177ee8..b4f2b48df7 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -28,7 +28,7 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val notifications = new Notifications(config) val downloader = new Downloader()(ec,wsClient) - val embedder: Option[Embedder] = if (config.shouldEmbed) Some(new Embedder(new S3Vectors(config), new Bedrock(config))) else None + val maybeEmbedder: Option[Embedder] = if (config.shouldEmbed) Some(new Embedder(new S3Vectors(config), new Bedrock(config))) else None val uploader = new Uploader(store, config, imageOperations, notifications, embedder, imageProcessor) val projector = Projector(config, imageOperations, imageProcessor, auth, embedder) From f76135e629158ddfd593b106d1040a37acc12398 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 24 Oct 2025 12:20:49 +0100 Subject: [PATCH 63/66] maybeEmbed => maybeEmbedder --- image-loader/app/ImageLoaderComponents.scala | 4 ++-- image-loader/app/model/Projector.scala | 12 ++++++------ image-loader/app/model/Uploader.scala | 4 ++-- image-loader/test/scala/model/ProjectorTest.scala | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index b4f2b48df7..b49ce974d8 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -30,8 +30,8 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val maybeEmbedder: Option[Embedder] = if (config.shouldEmbed) Some(new Embedder(new S3Vectors(config), new Bedrock(config))) else None - val uploader = new Uploader(store, config, imageOperations, notifications, embedder, imageProcessor) - val projector = Projector(config, imageOperations, imageProcessor, auth, embedder) + val uploader = new Uploader(store, config, imageOperations, notifications, maybeEmbedder, imageProcessor) + val projector = Projector(config, imageOperations, imageProcessor, auth, maybeEmbedder) val quarantineUploader: Option[QuarantineUploader] = (config.uploadToQuarantineEnabled, config.quarantineBucket) match { case (true, Some(bucketName)) =>{ val quarantineStore = new QuarantineStore(config) diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 307d384413..eb6a4ac43a 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -30,8 +30,8 @@ object Projector { import Uploader.toImageUploadOpsCfg - def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbed: Option[Embedder])(implicit ec: ExecutionContext): Projector - = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, maybeEmbed) + def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder])(implicit ec: ExecutionContext): Projector + = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, maybeEmbedder) } case class S3FileExtractedMetadata( @@ -85,9 +85,9 @@ class Projector(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, - maybeEmbed: Option[Embedder]) extends GridLogging { + maybeEmbedder: Option[Embedder]) extends GridLogging { - private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, maybeEmbed) + private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, maybeEmbedder) def projectS3ImageById(imageId: String, tempFile: File, gridClient: GridClient, onBehalfOfFn: WSRequest => WSRequest) (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[Image]] = { @@ -160,7 +160,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, s3: AmazonS3, - maybeEmbed: Option[Embedder], + maybeEmbedder: Option[Embedder], ) extends GridLogging { import Uploader.{fromUploadRequestShared, toMetaMap} @@ -183,7 +183,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, } private def createEmbeddingAndStore(fileType: MimeType, imageFilePath: Path, imageId: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[PutVectorsResponse]] = { - maybeEmbed match { + maybeEmbedder match { case Some(embedder) => embedder.createEmbeddingAndStore(fileType, imageFilePath, imageId) case None => Future.successful(None) diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index e547e649b1..3274e84db3 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -332,7 +332,7 @@ class Uploader(val store: ImageLoaderStore, val config: ImageLoaderConfig, val imageOps: ImageOperations, val notifications: Notifications, - val maybeEmbed: Option[Embedder], + val maybeEmbedder: Option[Embedder], imageProcessor: ImageProcessor) (implicit val ec: ExecutionContext) extends MessageSubjects with ArgoHelpers { @@ -347,7 +347,7 @@ class Uploader(val store: ImageLoaderStore, } private def createEmbeddingAndStore(fileType: MimeType, imageFilePath: Path, imageId: String)(implicit logMarker: LogMarker): Future[Option[PutVectorsResponse]] = { - maybeEmbed match { + maybeEmbedder match { case Some(embedder) => embedder.createEmbeddingAndStore(fileType, imageFilePath, imageId) case None => Future.successful(None) diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index 1ecd56b4c9..b0473127ee 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -43,11 +43,11 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket") - private val maybeEmbed = None + private val maybeEmbedder = None private val s3 = mock[AmazonS3] private val auth = mock[Authentication] - private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbed) + private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbedder) // FIXME temporary ignored as test is not executable in CI/CD machine // because graphic lib files like srgb.icc, cmyk.icc are in root directory instead of resources From ed7c5d6057550b9ddb82a43e3685907fe39f21dc Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 24 Oct 2025 12:28:43 +0100 Subject: [PATCH 64/66] Indent comment to match surrounding indentation --- image-loader/app/model/Uploader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index 3274e84db3..2a263d1f5e 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -156,7 +156,7 @@ object Uploader extends GridLogging { val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, tempDirForRequest, deps) // We are fetching the embedding and storing it in the S3Vectors bucket -// This should not block the image upload process on failure + // This should not block the image upload process on failure createEmbeddingAndStore(originalMimeType, uploadRequest.tempFile.toPath, uploadRequest.imageId).failed.foreach { failure => logger.error(logMarker, s"Failed to fetch embedding for ${uploadRequest.imageId} and store", failure) } From c7d09d53c1153b7144be5fcaeceabaf0f927d5b9 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 24 Oct 2025 16:02:59 +0100 Subject: [PATCH 65/66] Mention image embedding in all log message so we can find them easily --- .../scala/com/gu/mediaservice/lib/aws/Bedrock.scala | 12 ++++++------ .../com/gu/mediaservice/lib/aws/S3Vectors.scala | 4 ++-- image-loader/app/model/Uploader.scala | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index 9ad2cfd78c..f1d816466b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -67,13 +67,13 @@ class Bedrock(config: CommonConfig) val response = client.invokeModel(createRequestBody(base64EncodedImage, fileType)) logger.info( logMarker, - s"Bedrock API call completed with status: ${response.sdkHttpResponse().statusCode()}" + s"Bedrock API call to create image embedding completed with status: ${response.sdkHttpResponse().statusCode()}" ) response } catch { case e: Exception => - logger.error(logMarker, "Exception during Bedrock API call", e) + logger.error(logMarker, "Exception during Bedrock API call to create image embedding", e) throw e } } @@ -83,13 +83,13 @@ class Bedrock(config: CommonConfig) bedrockFuture.map { response => val responseBody = response.body().asUtf8String() val json = Json.parse(responseBody) - // Extract the embeddings array (first element since it's an array of arrays) - val embeddings = (json \ "embeddings" \ "float")(0).as[List[Float]] + // Extract the embedding array (first element since it's an array of arrays) + val embedding = (json \ "embeddings" \ "float")(0).as[List[Float]] logger.info( logMarker, - s"Successfully extracted embeddings. Vector size: ${embeddings.size}" + s"Successfully extracted image embedding. Vector size: ${embedding.size}" ) - embeddings + embedding } } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index 934fa4e463..4ea06a03cd 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -55,13 +55,13 @@ class S3Vectors(config: CommonConfig) val response = client.putVectors(request) logger.info( logMarker, - s"S3 Vector Store API call completed with status: ${response.sdkHttpResponse().statusCode()}" + s"S3 Vector Store API call to store image embedding completed with status: ${response.sdkHttpResponse().statusCode()}" ) response } catch { case e: Exception => - logger.error(logMarker, s"Exception during S3 Vector Store API call for ImageId $imageId: ", e) + logger.error(logMarker, s"Exception during S3 Vector Store API call to store image embedding for $imageId: ", e) throw e } } diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index 2a263d1f5e..92fbf901ca 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -158,7 +158,7 @@ object Uploader extends GridLogging { // We are fetching the embedding and storing it in the S3Vectors bucket // This should not block the image upload process on failure createEmbeddingAndStore(originalMimeType, uploadRequest.tempFile.toPath, uploadRequest.imageId).failed.foreach { failure => - logger.error(logMarker, s"Failed to fetch embedding for ${uploadRequest.imageId} and store", failure) + logger.error(logMarker, s"Failed to create and store image embedding for ${uploadRequest.imageId}", failure) } val eventualImage = for { From a7a5d7f802e0b026ee5493abf5dc686df049cf9f Mon Sep 17 00:00:00 2001 From: Mateusz Date: Fri, 31 Oct 2025 10:22:58 +0000 Subject: [PATCH 66/66] Add to PhotographerRenamer dictionary #36 --- .../gu/mediaservice/lib/cleanup/PhotographerRenamer.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala index 76244a4c34..c55b9cf1b1 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala @@ -234,6 +234,7 @@ object PhotographerRenamer extends MetadataCleaner { "Huseyin Demirci" -> "Hüseyin Demirci", "Huseyin Yildiz" -> "Hüseyin Yıldız", "Ian Macnicol" -> "Ian MacNicol", + "Igor Pavicevic" -> "Igor Pavićević", "Inti Ocon" -> "Inti Ocón", "Ints Kalnins" -> "Ints Kalniņš", "Irek Dorozanski" -> "Irek Dorożański", @@ -274,6 +275,7 @@ object PhotographerRenamer extends MetadataCleaner { "Jerome Prebois" -> "Jérôme Prébois", "Jerome Prevost" -> "Jérôme Prévost", "Jerome Sessini" -> "Jérôme Sessini", + "Jerzy Muszynski" -> "Jerzy Muszyński", "Jesus Bustamante" -> "Jesús Bustamante", "Jesus Diges" -> "Jesús Diges", "Jesus Merida" -> "Jesús Mérida", @@ -338,6 +340,7 @@ object PhotographerRenamer extends MetadataCleaner { "Klebher Vasquez" -> "Klebher Vásquez", "Koca Sulejmanovic" -> "Koca Sulejmanović", "Krisztian Elek" -> "Krisztián Elek", + "Krzysztof Cwik" -> "Krzysztof Ćwik", "Krzysztof Swiderski" -> "Krzysztof Świderski", "Kuba Stezycki" -> "Kuba Stężycki", "Laszlo Balogh" -> "László Balogh", @@ -378,6 +381,7 @@ object PhotographerRenamer extends MetadataCleaner { "Manu Fernandez" -> "Manu Fernández", "Manuel Vazquez" -> "Manuel Vázquez", "Manuel Velazquez" -> "Manuel Velázquez", + "Marek Antoni Iwanczuk" -> "Marek Antoni Iwańczuk", "Marc Mccormack" -> "Marc McCormack", "Marcelo Del Pozo" -> "Marcelo del Pozo", "Marcelo Hernandez" -> "Marcelo Hernández", @@ -419,6 +423,7 @@ object PhotographerRenamer extends MetadataCleaner { "Milos Bicanski" -> "Miloš Bičanski", "Milos Vujovic" -> "Miloš Vujović", "Miro Kuzmanovic" -> "Miro Kuzmanović", + "Mitar Mitrovic" -> "Mitar Mitrović", "Moises Castillo" -> "Moisés Castillo", "Morne de Klerk" -> "Morné de Klerk", "Murat Ozgur Guvendik" -> "Murat Özgür Güvendik", @@ -482,6 +487,7 @@ object PhotographerRenamer extends MetadataCleaner { "Radoslaw Jozwiak" -> "Radosław Jóźwiak", "Rafal Gaglewski" -> "Rafał Gąglewski", "Rafal Guz" -> "Rafał Guz", + "Raimonda Kulikauskiene" -> "Raimonda Kulikauskienė", "Ramon Buxo Martinez" -> "Ramon Buxó Martínez", "Ramon Costa" -> "Ramón Costa", "Ramon de la Rocha" -> "Ramón de la Rocha",